From 1ca66626630d27a34c4e145415027a9028352308 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 00:11:01 +1100 Subject: [PATCH 01/17] Fix 8 P2 bugs: p99 bias, inverted min/max, deep_merge aliasing, float truncation, preset race, CLI summary, fixture fields, TestClient leaks - metrics_store: use nearest-rank (math.ceil) for p50/p95/p99 percentile calculations - latency: reject inverted min_sec > max_sec in simulate_slow_response - config_loader: use copy.deepcopy(base) in deep_merge to prevent shared references - validators: reject fractional floats in parse_range (allow exact integer floats) - response_generator: double-checked locking for PresetBank lazy init thread safety - llm/web cli: dynamic _pct field discovery for serve startup error summary - chaosllm fixture: add forbidden_pct, not_found_pct, connection_failed_pct, connection_stall_pct - integration tests: wrap TestClient usage in context managers to prevent resource leaks Co-Authored-By: Claude Opus 4.6 (1M context) --- src/errorworks/engine/config_loader.py | 3 +- src/errorworks/engine/latency.py | 2 + src/errorworks/engine/metrics_store.py | 11 +-- src/errorworks/engine/validators.py | 3 + src/errorworks/llm/cli.py | 14 ++-- src/errorworks/llm/response_generator.py | 15 ++-- src/errorworks/web/cli.py | 16 ++-- tests/fixtures/chaosllm.py | 12 +++ tests/integration/test_llm_pipeline.py | 90 +++++++++++------------ tests/integration/test_web_pipeline.py | 88 ++++++++++------------ tests/unit/engine/test_config_loader.py | 8 ++ tests/unit/engine/test_metrics_store.py | 57 ++++++++++++++ tests/unit/engine/test_validators.py | 25 +++++++ tests/unit/llm/test_cli.py | 16 ++++ tests/unit/llm/test_fixture.py | 40 ++++++++++ tests/unit/llm/test_latency_simulator.py | 8 ++ tests/unit/llm/test_response_generator.py | 37 ++++++++++ tests/unit/web/test_cli.py | 16 ++++ 18 files changed, 337 insertions(+), 124 deletions(-) create mode 100644 tests/unit/engine/test_validators.py diff --git a/src/errorworks/engine/config_loader.py b/src/errorworks/engine/config_loader.py index 90a9069..58a9347 100644 --- a/src/errorworks/engine/config_loader.py +++ b/src/errorworks/engine/config_loader.py @@ -7,6 +7,7 @@ from __future__ import annotations +import copy import re import warnings from pathlib import Path @@ -30,7 +31,7 @@ def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any] Returns: Merged configuration dict (new dict, does not mutate inputs). """ - result = dict(base) + result = copy.deepcopy(base) for key, value in override.items(): if key in result and isinstance(result[key], dict) and isinstance(value, dict): result[key] = deep_merge(result[key], value) diff --git a/src/errorworks/engine/latency.py b/src/errorworks/engine/latency.py index 500eefb..69dca55 100644 --- a/src/errorworks/engine/latency.py +++ b/src/errorworks/engine/latency.py @@ -75,4 +75,6 @@ def simulate_slow_response(self, min_sec: int, max_sec: int) -> float: Returns: Delay in seconds (float). """ + if min_sec > max_sec: + raise ValueError(f"min_sec ({min_sec}) must be <= max_sec ({max_sec})") return self._rng.uniform(min_sec, max_sec) diff --git a/src/errorworks/engine/metrics_store.py b/src/errorworks/engine/metrics_store.py index 54cee1f..2d50fa5 100644 --- a/src/errorworks/engine/metrics_store.py +++ b/src/errorworks/engine/metrics_store.py @@ -8,6 +8,7 @@ for recording domain-specific request data. """ +import math import sqlite3 import threading import uuid @@ -345,7 +346,7 @@ def update_bucket_latency(self, bucket_utc: str, latency_ms: float | None) -> No count = row[1] # Compute p99 via LIMIT/OFFSET (SQLite handles the sort internally) - p99_offset = min(int(count * 0.99), count - 1) + p99_offset = max(0, min(math.ceil(count * 0.99) - 1, count - 1)) cursor = conn.execute( """ SELECT latency_ms FROM requests @@ -443,7 +444,7 @@ def rebuild_timeseries( if latencies: avg_latency = sum(latencies) / len(latencies) latencies.sort() - p99_index = min(int(len(latencies) * 0.99), len(latencies) - 1) + p99_index = max(0, min(math.ceil(len(latencies) * 0.99) - 1, len(latencies) - 1)) p99_latency = latencies[p99_index] conn.execute( "UPDATE timeseries SET avg_latency_ms = ?, p99_latency_ms = ? WHERE bucket_utc = ?", @@ -516,9 +517,9 @@ def get_stats(self) -> dict[str, Any]: p99_latency = None if latencies: - p50_latency = latencies[min(int(len(latencies) * 0.50), len(latencies) - 1)] - p95_latency = latencies[min(int(len(latencies) * 0.95), len(latencies) - 1)] - p99_latency = latencies[min(int(len(latencies) * 0.99), len(latencies) - 1)] + p50_latency = latencies[max(0, min(math.ceil(len(latencies) * 0.50) - 1, len(latencies) - 1))] + p95_latency = latencies[max(0, min(math.ceil(len(latencies) * 0.95) - 1, len(latencies) - 1))] + p99_latency = latencies[max(0, min(math.ceil(len(latencies) * 0.99) - 1, len(latencies) - 1))] stats["latency_stats"] = { "avg_ms": row[0], diff --git a/src/errorworks/engine/validators.py b/src/errorworks/engine/validators.py index ffd3c02..88859da 100644 --- a/src/errorworks/engine/validators.py +++ b/src/errorworks/engine/validators.py @@ -18,6 +18,9 @@ def parse_range(v: Any) -> tuple[int, int]: Use as a Pydantic field_validator(mode="before") for tuple[int, int] fields. """ if isinstance(v, (list, tuple)) and len(v) == 2: + for i, val in enumerate(v): + if isinstance(val, float) and not val.is_integer(): + raise ValueError(f"Range values must be integers, got float {val} at index {i}") lo, hi = int(v[0]), int(v[1]) if lo < 0 or hi < 0: raise ValueError(f"Range values must be non-negative, got [{lo}, {hi}]") diff --git a/src/errorworks/llm/cli.py b/src/errorworks/llm/cli.py index 78308b3..035d6ef 100644 --- a/src/errorworks/llm/cli.py +++ b/src/errorworks/llm/cli.py @@ -345,14 +345,12 @@ def serve( # Show error injection summary error_cfg = config.error_injection active_errors = [] - if error_cfg.rate_limit_pct > 0: - active_errors.append(f"429:{error_cfg.rate_limit_pct:.1f}%") - if error_cfg.capacity_529_pct > 0: - active_errors.append(f"529:{error_cfg.capacity_529_pct:.1f}%") - if error_cfg.service_unavailable_pct > 0: - active_errors.append(f"503:{error_cfg.service_unavailable_pct:.1f}%") - if error_cfg.internal_error_pct > 0: - active_errors.append(f"500:{error_cfg.internal_error_pct:.1f}%") + for name in type(error_cfg).model_fields: + if name.endswith("_pct"): + val = getattr(error_cfg, name) + if val > 0: + label = name.removesuffix("_pct") + active_errors.append(f"{label}:{val:.1f}%") if active_errors: typer.echo(f" Error injection: {', '.join(active_errors)}") diff --git a/src/errorworks/llm/response_generator.py b/src/errorworks/llm/response_generator.py index 1b25266..01063f1 100644 --- a/src/errorworks/llm/response_generator.py +++ b/src/errorworks/llm/response_generator.py @@ -234,6 +234,7 @@ def __init__( # Lazy-load preset bank self._preset_bank: PresetBank | None = None + self._preset_lock = threading.Lock() # Setup Jinja2 environment with custom helpers self._jinja_env = self._create_jinja_env() @@ -341,13 +342,15 @@ def _generate_echo_response(self, request: dict[str, Any]) -> str: return f"Echo: {last_content}" def _get_preset_bank(self) -> PresetBank: - """Get or create preset bank (lazy loading).""" + """Get or create preset bank (lazy loading, thread-safe).""" if self._preset_bank is None: - self._preset_bank = PresetBank.from_jsonl( - self._config.preset.file, - self._config.preset.selection, - rng=self._rng, - ) + with self._preset_lock: + if self._preset_bank is None: # double-check under lock + self._preset_bank = PresetBank.from_jsonl( + self._config.preset.file, + self._config.preset.selection, + rng=self._rng, + ) return self._preset_bank def _generate_preset_response(self) -> str: diff --git a/src/errorworks/web/cli.py b/src/errorworks/web/cli.py index e8d0e4d..56dbac5 100644 --- a/src/errorworks/web/cli.py +++ b/src/errorworks/web/cli.py @@ -266,16 +266,12 @@ def serve( # Error injection summary error_cfg = config.error_injection active_errors = [] - if error_cfg.rate_limit_pct > 0: - active_errors.append(f"429:{error_cfg.rate_limit_pct:.1f}%") - if error_cfg.forbidden_pct > 0: - active_errors.append(f"403:{error_cfg.forbidden_pct:.1f}%") - if error_cfg.not_found_pct > 0: - active_errors.append(f"404:{error_cfg.not_found_pct:.1f}%") - if error_cfg.service_unavailable_pct > 0: - active_errors.append(f"503:{error_cfg.service_unavailable_pct:.1f}%") - if error_cfg.ssrf_redirect_pct > 0: - active_errors.append(f"SSRF:{error_cfg.ssrf_redirect_pct:.1f}%") + for name in type(error_cfg).model_fields: + if name.endswith("_pct"): + val = getattr(error_cfg, name) + if val > 0: + label = name.removesuffix("_pct") + active_errors.append(f"{label}:{val:.1f}%") if active_errors: typer.echo(f" Error injection: {', '.join(active_errors)}") diff --git a/tests/fixtures/chaosllm.py b/tests/fixtures/chaosllm.py index 1f28114..da218f1 100644 --- a/tests/fixtures/chaosllm.py +++ b/tests/fixtures/chaosllm.py @@ -77,12 +77,16 @@ def update_config( internal_error_pct: float | None = None, timeout_pct: float | None = None, connection_reset_pct: float | None = None, + connection_failed_pct: float | None = None, + connection_stall_pct: float | None = None, slow_response_pct: float | None = None, invalid_json_pct: float | None = None, truncated_pct: float | None = None, empty_body_pct: float | None = None, missing_fields_pct: float | None = None, wrong_content_type_pct: float | None = None, + forbidden_pct: float | None = None, + not_found_pct: float | None = None, selection_mode: str | None = None, base_ms: int | None = None, jitter_ms: int | None = None, @@ -99,12 +103,16 @@ def update_config( ("internal_error_pct", internal_error_pct), ("timeout_pct", timeout_pct), ("connection_reset_pct", connection_reset_pct), + ("connection_failed_pct", connection_failed_pct), + ("connection_stall_pct", connection_stall_pct), ("slow_response_pct", slow_response_pct), ("invalid_json_pct", invalid_json_pct), ("truncated_pct", truncated_pct), ("empty_body_pct", empty_body_pct), ("missing_fields_pct", missing_fields_pct), ("wrong_content_type_pct", wrong_content_type_pct), + ("forbidden_pct", forbidden_pct), + ("not_found_pct", not_found_pct), ("selection_mode", selection_mode), ]: if val is not None: @@ -193,12 +201,16 @@ def _build_config_from_marker( "internal_error_pct", "timeout_pct", "connection_reset_pct", + "connection_failed_pct", + "connection_stall_pct", "slow_response_pct", "invalid_json_pct", "truncated_pct", "empty_body_pct", "missing_fields_pct", "wrong_content_type_pct", + "forbidden_pct", + "not_found_pct", "selection_mode", ]: if key in marker.kwargs: diff --git a/tests/integration/test_llm_pipeline.py b/tests/integration/test_llm_pipeline.py index 79157e4..2f8b2bb 100644 --- a/tests/integration/test_llm_pipeline.py +++ b/tests/integration/test_llm_pipeline.py @@ -31,25 +31,25 @@ def _make_client(*, preset: str | None = None, config_file: Path | None = None) def test_silent_preset_returns_200() -> None: """Silent preset should produce zero errors.""" - client = _make_client(preset="silent") - statuses = [client.post("/v1/chat/completions", json=CHAT_BODY).status_code for _ in range(50)] - assert all(s == 200 for s in statuses), f"Expected all 200s, got non-200 codes: {[s for s in statuses if s != 200]}" + with _make_client(preset="silent") as client: + statuses = [client.post("/v1/chat/completions", json=CHAT_BODY).status_code for _ in range(50)] + assert all(s == 200 for s in statuses), f"Expected all 200s, got non-200 codes: {[s for s in statuses if s != 200]}" def test_gentle_preset_injects_errors() -> None: """Gentle preset sums to 2% errors; verify within tolerance over 500 requests.""" - client = _make_client(preset="gentle") - n = 500 - errors = sum(1 for _ in range(n) if client.post("/v1/chat/completions", json=CHAT_BODY).status_code != 200) - assert_rate_near(errors, n, expected_pct=2.0, tolerance_pct=4.0) + with _make_client(preset="gentle") as client: + n = 500 + errors = sum(1 for _ in range(n) if client.post("/v1/chat/completions", json=CHAT_BODY).status_code != 200) + assert_rate_near(errors, n, expected_pct=2.0, tolerance_pct=4.0) def test_stress_extreme_injects_heavily() -> None: """Stress-extreme preset should produce >30% errors.""" - client = _make_client(preset="stress_extreme") - n = 100 - errors = sum(1 for _ in range(n) if client.post("/v1/chat/completions", json=CHAT_BODY).status_code != 200) - assert errors > 30, f"Expected >30% errors, got {errors}/{n}" + with _make_client(preset="stress_extreme") as client: + n = 100 + errors = sum(1 for _ in range(n) if client.post("/v1/chat/completions", json=CHAT_BODY).status_code != 200) + assert errors > 30, f"Expected >30% errors, got {errors}/{n}" def test_preset_plus_config_file_merge(tmp_path: Path) -> None: @@ -59,57 +59,55 @@ def test_preset_plus_config_file_merge(tmp_path: Path) -> None: config = load_config(preset="silent", config_file=overlay) app = create_app(config) - client = TestClient(app, raise_server_exceptions=False) - - statuses = [client.post("/v1/chat/completions", json=CHAT_BODY).status_code for _ in range(10)] - assert all(s == 429 for s in statuses), f"Expected all 429s, got: {statuses}" + with TestClient(app, raise_server_exceptions=False) as client: + statuses = [client.post("/v1/chat/completions", json=CHAT_BODY).status_code for _ in range(10)] + assert all(s == 429 for s in statuses), f"Expected all 429s, got: {statuses}" def test_metrics_recorded_after_requests() -> None: """After sending requests, /admin/stats should reflect them.""" - client = _make_client(preset="silent") - for _ in range(10): - client.post("/v1/chat/completions", json=CHAT_BODY) + with _make_client(preset="silent") as client: + for _ in range(10): + client.post("/v1/chat/completions", json=CHAT_BODY) - resp = client.get("/admin/stats", headers=_ADMIN_HEADERS) - assert resp.status_code == 200 - stats = resp.json() - assert isinstance(stats, dict) - assert stats.get("total_requests", 0) >= 10 + resp = client.get("/admin/stats", headers=_ADMIN_HEADERS) + assert resp.status_code == 200 + stats = resp.json() + assert isinstance(stats, dict) + assert stats.get("total_requests", 0) >= 10 def test_config_reload_endpoint() -> None: """POST /admin/config should update error injection at runtime.""" - client = _make_client(preset="silent") - - # Baseline: silent preset returns 200 - assert client.post("/v1/chat/completions", json=CHAT_BODY).status_code == 200 + with _make_client(preset="silent") as client: + # Baseline: silent preset returns 200 + assert client.post("/v1/chat/completions", json=CHAT_BODY).status_code == 200 - # Reload config to inject 100% rate-limit errors - reload_resp = client.post("/admin/config", json={"error_injection": {"rate_limit_pct": 100.0}}, headers=_ADMIN_HEADERS) - assert reload_resp.status_code == 200 - body = reload_resp.json() - assert body["status"] == "updated" + # Reload config to inject 100% rate-limit errors + reload_resp = client.post("/admin/config", json={"error_injection": {"rate_limit_pct": 100.0}}, headers=_ADMIN_HEADERS) + assert reload_resp.status_code == 200 + body = reload_resp.json() + assert body["status"] == "updated" - # Next request should be 429 - assert client.post("/v1/chat/completions", json=CHAT_BODY).status_code == 429 + # Next request should be 429 + assert client.post("/v1/chat/completions", json=CHAT_BODY).status_code == 429 def test_azure_endpoint_compatibility() -> None: """Azure-style deployment endpoint should work identically to the OpenAI path.""" - client = _make_client(preset="silent") - resp = client.post("/openai/deployments/gpt-4/chat/completions", json=CHAT_BODY) - assert resp.status_code == 200 - data = resp.json() - assert "choices" in data + with _make_client(preset="silent") as client: + resp = client.post("/openai/deployments/gpt-4/chat/completions", json=CHAT_BODY) + assert resp.status_code == 200 + data = resp.json() + assert "choices" in data def test_response_contains_choices() -> None: """Successful completion response should contain a non-empty choices list.""" - client = _make_client(preset="silent") - resp = client.post("/v1/chat/completions", json=CHAT_BODY) - assert resp.status_code == 200 - data = resp.json() - assert "choices" in data - assert len(data["choices"]) >= 1 - assert "message" in data["choices"][0] + with _make_client(preset="silent") as client: + resp = client.post("/v1/chat/completions", json=CHAT_BODY) + assert resp.status_code == 200 + data = resp.json() + assert "choices" in data + assert len(data["choices"]) >= 1 + assert "message" in data["choices"][0] diff --git a/tests/integration/test_web_pipeline.py b/tests/integration/test_web_pipeline.py index 408b0db..b6645a8 100644 --- a/tests/integration/test_web_pipeline.py +++ b/tests/integration/test_web_pipeline.py @@ -18,14 +18,13 @@ def test_silent_preset_returns_html() -> None: """Silent preset serves valid HTML with 200 status.""" config = load_config(preset="silent") app = create_app(config) - client = TestClient(app) - - resp = client.get("/") + with TestClient(app) as client: + resp = client.get("/") - assert resp.status_code == 200 - body = resp.text.lower() - assert " None: """Gentle preset injects errors at ~2% rate (rate_limit 1% + not_found 1%).""" config = load_config(preset="gentle", cli_overrides={"latency": {"base_ms": 0, "jitter_ms": 0}}) app = create_app(config) - client = TestClient(app) + with TestClient(app) as client: + total = 500 + errors = sum(1 for _ in range(total) if client.get("/").status_code != 200) - total = 500 - errors = sum(1 for _ in range(total) if client.get("/").status_code != 200) - - assert_rate_near(errors, total, expected_pct=2.0, tolerance_pct=4.0) + assert_rate_near(errors, total, expected_pct=2.0, tolerance_pct=4.0) @pytest.mark.integration @@ -59,12 +57,11 @@ def test_stress_scraping_anti_bot() -> None: ) app = create_app(config) # Disable redirect following — SSRF redirects target private IPs that TestClient can't resolve - client = TestClient(app, follow_redirects=False) - - total = 100 - errors = sum(1 for _ in range(total) if client.get("/").status_code != 200) + with TestClient(app, follow_redirects=False) as client: + total = 100 + errors = sum(1 for _ in range(total) if client.get("/").status_code != 200) - assert errors > total * 0.30, f"Expected >30% errors, got {errors}/{total}" + assert errors > total * 0.30, f"Expected >30% errors, got {errors}/{total}" @pytest.mark.integration @@ -75,11 +72,10 @@ def test_preset_plus_config_file_merge(tmp_path: Path) -> None: config = load_config(preset="silent", config_file=overlay) app = create_app(config) - client = TestClient(app) - - for _ in range(10): - resp = client.get("/") - assert resp.status_code == 429 + with TestClient(app) as client: + for _ in range(10): + resp = client.get("/") + assert resp.status_code == 429 @pytest.mark.integration @@ -87,15 +83,14 @@ def test_content_structure() -> None: """Silent preset returns HTML with standard structural elements.""" config = load_config(preset="silent") app = create_app(config) - client = TestClient(app) - - resp = client.get("/") + with TestClient(app) as client: + resp = client.get("/") - assert resp.status_code == 200 - body = resp.text.lower() - assert " None: """Requests are recorded and accessible via /admin/stats.""" config = load_config(preset="silent", cli_overrides={"server": {"admin_token": _TEST_TOKEN}}) app = create_app(config) - client = TestClient(app) - - for _ in range(10): - client.get("/") + with TestClient(app) as client: + for _ in range(10): + client.get("/") - resp = client.get("/admin/stats", headers=_ADMIN_HEADERS) + resp = client.get("/admin/stats", headers=_ADMIN_HEADERS) - assert resp.status_code == 200 - data = resp.json() - assert isinstance(data, dict) + assert resp.status_code == 200 + data = resp.json() + assert isinstance(data, dict) @pytest.mark.integration @@ -123,11 +117,10 @@ def test_redirect_deterministic() -> None: cli_overrides={"error_injection": {"ssrf_redirect_pct": 100.0}}, ) app = create_app(config) - client = TestClient(app, follow_redirects=False) - - resp = client.get("/") + with TestClient(app, follow_redirects=False) as client: + resp = client.get("/") - assert resp.status_code == 301 + assert resp.status_code == 301 @pytest.mark.integration @@ -138,10 +131,9 @@ def test_malformed_html_injection() -> None: cli_overrides={"error_injection": {"truncated_html_pct": 100.0}}, ) app = create_app(config) - client = TestClient(app) - - resp = client.get("/") + with TestClient(app) as client: + resp = client.get("/") - # Truncated HTML handler returns 200 with partial content - body = resp.text.strip() - assert resp.status_code != 200 or not body.endswith("") + # Truncated HTML handler returns 200 with partial content + body = resp.text.strip() + assert resp.status_code != 200 or not body.endswith("") diff --git a/tests/unit/engine/test_config_loader.py b/tests/unit/engine/test_config_loader.py index 3675b7d..e1ee30b 100644 --- a/tests/unit/engine/test_config_loader.py +++ b/tests/unit/engine/test_config_loader.py @@ -71,6 +71,14 @@ def test_does_not_mutate_inputs(self) -> None: assert base == base_copy assert override == override_copy + def test_result_does_not_alias_base_nested_dicts(self) -> None: + """Nested dicts in result must not be shared references to base.""" + base = {"a": {"nested": 1}} + override = {"b": 2} + result = deep_merge(base, override) + result["a"]["nested"] = 99 + assert base["a"]["nested"] == 1, "Mutating result must not affect base" + # ============================================================================= # list_presets diff --git a/tests/unit/engine/test_metrics_store.py b/tests/unit/engine/test_metrics_store.py index c64e57f..5adb0d0 100644 --- a/tests/unit/engine/test_metrics_store.py +++ b/tests/unit/engine/test_metrics_store.py @@ -471,6 +471,63 @@ def test_save_run_info(self, store: MetricsStore) -> None: # ============================================================================= +class TestPercentileAccuracy: + """Tests for percentile calculation accuracy with small sample sizes.""" + + def test_p50_nearest_rank_small_sample(self, store: MetricsStore) -> None: + """p50 uses nearest-rank formula: ceil(N*0.50)-1, not int(N*0.50). + + With 10 sorted values [1..10], p50 should be index 4 = 5.0. + The old int(10*0.50)=5 gives index 5 = 6.0 (wrong). + """ + for i in range(10): + store.record( + request_id=f"p-{i}", + timestamp_utc=f"2024-01-15T10:30:{i:02d}+00:00", + outcome="success", + status_code=200, + latency_ms=float(i + 1), # 1.0, 2.0, ..., 10.0 + ) + store.commit() + + stats = store.get_stats() + latency = stats["latency_stats"] + assert latency["p50_ms"] == 5.0, f"p50 should be 5.0 (index 4), got {latency['p50_ms']}" + assert latency["p95_ms"] == 10.0, f"p95 should be 10.0 (index 9), got {latency['p95_ms']}" + assert latency["p99_ms"] == 10.0, f"p99 should be 10.0 (index 9), got {latency['p99_ms']}" + + def test_p50_single_value(self, store: MetricsStore) -> None: + """Single value: all percentiles should return that value.""" + store.record( + request_id="solo", + timestamp_utc="2024-01-15T10:30:00+00:00", + outcome="success", + latency_ms=42.0, + ) + store.commit() + + stats = store.get_stats() + latency = stats["latency_stats"] + assert latency["p50_ms"] == 42.0 + assert latency["p95_ms"] == 42.0 + assert latency["p99_ms"] == 42.0 + + def test_p50_two_values(self, store: MetricsStore) -> None: + """Two values [1.0, 2.0]: p50=ceil(2*0.50)-1=0 -> 1.0.""" + for i in range(2): + store.record( + request_id=f"p-{i}", + timestamp_utc=f"2024-01-15T10:30:0{i}+00:00", + outcome="success", + latency_ms=float(i + 1), + ) + store.commit() + + stats = store.get_stats() + latency = stats["latency_stats"] + assert latency["p50_ms"] == 1.0 + + class TestClose: """Tests for connection cleanup.""" diff --git a/tests/unit/engine/test_validators.py b/tests/unit/engine/test_validators.py new file mode 100644 index 0000000..1f501b5 --- /dev/null +++ b/tests/unit/engine/test_validators.py @@ -0,0 +1,25 @@ +"""Tests for errorworks.engine.validators.""" + +from __future__ import annotations + +import pytest + +from errorworks.engine.validators import parse_range + + +class TestParseRangeFloatRejection: + """parse_range must reject floats with fractional parts.""" + + def test_rejects_fractional_floats(self) -> None: + with pytest.raises(ValueError, match="must be integers.*1.5"): + parse_range([1.5, 3.5]) + + def test_rejects_single_fractional_float(self) -> None: + with pytest.raises(ValueError, match="must be integers.*2.7"): + parse_range([1, 2.7]) + + def test_accepts_exact_integer_floats(self) -> None: + assert parse_range([3.0, 5.0]) == (3, 5) + + def test_accepts_plain_integers(self) -> None: + assert parse_range([1, 10]) == (1, 10) diff --git a/tests/unit/llm/test_cli.py b/tests/unit/llm/test_cli.py index ad19138..59a355c 100644 --- a/tests/unit/llm/test_cli.py +++ b/tests/unit/llm/test_cli.py @@ -74,6 +74,22 @@ def test_serve_all_error_flags(mock_run): assert result.exit_code == 0, result.output +@_patch_uvicorn_run +def test_serve_error_summary_shows_all_pct_fields(mock_run): + """Startup summary includes all non-zero _pct fields, not just a hardcoded subset.""" + result = runner.invoke( + app, + [ + "serve", + "--rate-limit-pct=10", + "--timeout-pct=50", + ], + ) + assert result.exit_code == 0, result.output + assert "rate_limit:10.0%" in result.output + assert "timeout:50.0%" in result.output + + @_patch_uvicorn_run def test_serve_burst_flags(mock_run): """serve with burst flags exits 0.""" diff --git a/tests/unit/llm/test_fixture.py b/tests/unit/llm/test_fixture.py index 69253f6..41785f3 100644 --- a/tests/unit/llm/test_fixture.py +++ b/tests/unit/llm/test_fixture.py @@ -196,6 +196,46 @@ def test_no_marker_after_marker(self, chaosllm_server): assert response.status_code == 200 +class TestChaosLLMFixtureMissingErrorFields: + """Tests for the 4 error fields missing from fixture API: forbidden, not_found, connection_failed, connection_stall.""" + + @pytest.mark.chaosllm(forbidden_pct=100.0) + def test_marker_forbidden(self, chaosllm_server): + """Marker can set forbidden_pct.""" + response = chaosllm_server.post_completion() + assert response.status_code == 403 + + @pytest.mark.chaosllm(not_found_pct=100.0) + def test_marker_not_found(self, chaosllm_server): + """Marker can set not_found_pct.""" + response = chaosllm_server.post_completion() + assert response.status_code == 404 + + def test_update_config_forbidden(self, chaosllm_server): + """update_config can set forbidden_pct at runtime.""" + chaosllm_server.update_config(forbidden_pct=100.0) + response = chaosllm_server.post_completion() + assert response.status_code == 403 + + def test_update_config_not_found(self, chaosllm_server): + """update_config can set not_found_pct at runtime.""" + chaosllm_server.update_config(not_found_pct=100.0) + response = chaosllm_server.post_completion() + assert response.status_code == 404 + + def test_update_config_connection_failed(self, chaosllm_server): + """update_config accepts connection_failed_pct without error.""" + chaosllm_server.update_config(connection_failed_pct=50.0) + current = chaosllm_server.server.get_current_config() + assert current["error_injection"]["connection_failed_pct"] == 50.0 + + def test_update_config_connection_stall(self, chaosllm_server): + """update_config accepts connection_stall_pct without error.""" + chaosllm_server.update_config(connection_stall_pct=50.0) + current = chaosllm_server.server.get_current_config() + assert current["error_injection"]["connection_stall_pct"] == 50.0 + + class TestChaosLLMErrorTypes: """Test various error injection types via the fixture.""" diff --git a/tests/unit/llm/test_latency_simulator.py b/tests/unit/llm/test_latency_simulator.py index e4345d7..53bdcac 100644 --- a/tests/unit/llm/test_latency_simulator.py +++ b/tests/unit/llm/test_latency_simulator.py @@ -285,6 +285,14 @@ def test_large_base_ms(self) -> None: delay = simulator.simulate() assert delay == 5.0 # 5000ms = 5 seconds + def test_inverted_min_max_slow_response_raises(self) -> None: + """When min_sec > max_sec, raises ValueError.""" + config = LatencyConfig() + simulator = LatencySimulator(config) + + with pytest.raises(ValueError, match="min_sec.*must be <= max_sec"): + simulator.simulate_slow_response(30, 10) + def test_equal_min_max_slow_response(self) -> None: """When min_sec equals max_sec, returns that exact value.""" config = LatencyConfig() diff --git a/tests/unit/llm/test_response_generator.py b/tests/unit/llm/test_response_generator.py index 0d8721c..33bafea 100644 --- a/tests/unit/llm/test_response_generator.py +++ b/tests/unit/llm/test_response_generator.py @@ -2,6 +2,7 @@ import json import random +import threading from pathlib import Path from typing import Any @@ -948,3 +949,39 @@ def test_lorem_vocabulary_lowercase(self) -> None: """Lorem vocabulary words are lowercase.""" for word in LOREM_VOCABULARY: assert word == word.lower(), f"'{word}' should be lowercase" + + +class TestPresetBankThreadSafety: + """Tests for thread-safe lazy initialization of PresetBank.""" + + def test_get_preset_bank_concurrent_returns_same_instance(self, tmp_path: Path) -> None: + """Multiple threads calling _get_preset_bank() all get the same PresetBank instance.""" + jsonl_file = tmp_path / "responses.jsonl" + jsonl_file.write_text('{"content": "A"}\n{"content": "B"}\n{"content": "C"}\n') + + config = ResponseConfig( + mode="preset", + preset=PresetResponseConfig( + file=str(jsonl_file), + selection="sequential", + ), + ) + generator = ResponseGenerator(config) + + results: list[PresetBank] = [] + barrier = threading.Barrier(10) + + def get_bank() -> None: + barrier.wait() # maximize contention + bank = generator._get_preset_bank() + results.append(bank) + + threads = [threading.Thread(target=get_bank) for _ in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(results) == 10 + # All threads must receive the exact same PresetBank instance + assert all(b is results[0] for b in results) diff --git a/tests/unit/web/test_cli.py b/tests/unit/web/test_cli.py index 1ac60f4..f89e2f1 100644 --- a/tests/unit/web/test_cli.py +++ b/tests/unit/web/test_cli.py @@ -75,6 +75,22 @@ def test_serve_all_error_flags(mock_run): assert result.exit_code == 0, result.output +@patch(_UVICORN_RUN) +def test_serve_error_summary_shows_all_pct_fields(mock_run): + """Startup summary includes all non-zero _pct fields, not just a hardcoded subset.""" + result = runner.invoke( + app, + [ + "serve", + "--rate-limit-pct=10", + "--timeout-pct=25", + ], + ) + assert result.exit_code == 0, result.output + assert "rate_limit:10.0%" in result.output + assert "timeout:25.0%" in result.output + + @patch(_UVICORN_RUN) def test_serve_burst_flags(mock_run): """Burst flags are accepted.""" From 233719d56b3b4c9b291a8d5ecd1a79a6cbc13d56 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 00:36:55 +1100 Subject: [PATCH 02/17] Fix 4 P3 bugs: get_requests schema check, PresetBank thread-safety, web classify_outcome guard - metrics_store: guard get_requests outcome filter against missing schema column - response_generator + content_generator: wrap PresetBank.next() random path in lock - web/metrics: add status_code is None guard to connection_error classification Co-Authored-By: Claude Opus 4.6 (1M context) --- src/errorworks/engine/metrics_store.py | 2 ++ src/errorworks/llm/response_generator.py | 10 +++---- src/errorworks/web/content_generator.py | 10 +++---- src/errorworks/web/metrics.py | 3 ++- tests/unit/engine/test_metrics_store.py | 18 +++++++++++++ tests/unit/llm/test_response_generator.py | 31 +++++++++++++++++++++ tests/unit/web/test_content_generator.py | 33 +++++++++++++++++++++++ tests/unit/web/test_metrics.py | 5 ++++ 8 files changed, 101 insertions(+), 11 deletions(-) diff --git a/src/errorworks/engine/metrics_store.py b/src/errorworks/engine/metrics_store.py index 2d50fa5..05ec403 100644 --- a/src/errorworks/engine/metrics_store.py +++ b/src/errorworks/engine/metrics_store.py @@ -566,6 +566,8 @@ def get_requests( """Get request records from the database.""" conn = self._get_connection() if outcome is not None: + if "outcome" not in self._request_col_names: + raise ValueError("Cannot filter by 'outcome': column not present in schema") cursor = conn.execute( "SELECT * FROM requests WHERE outcome = ? ORDER BY timestamp_utc DESC LIMIT ? OFFSET ?", (outcome, limit, offset), diff --git a/src/errorworks/llm/response_generator.py b/src/errorworks/llm/response_generator.py index 01063f1..cd7dc97 100644 --- a/src/errorworks/llm/response_generator.py +++ b/src/errorworks/llm/response_generator.py @@ -130,13 +130,13 @@ def __init__( def next(self) -> str: """Get the next preset response (thread-safe).""" - if self._selection == "random": - return self._rng.choice(self._responses) - else: # sequential - with self._lock: + with self._lock: + if self._selection == "random": + return self._rng.choice(self._responses) + else: # sequential response = self._responses[self._index] self._index = (self._index + 1) % len(self._responses) - return response + return response def reset(self) -> None: """Reset sequential index to beginning.""" diff --git a/src/errorworks/web/content_generator.py b/src/errorworks/web/content_generator.py index 04c69d2..0a59c1e 100644 --- a/src/errorworks/web/content_generator.py +++ b/src/errorworks/web/content_generator.py @@ -95,13 +95,13 @@ def __init__( def next(self) -> dict[str, str]: """Get the next preset page (thread-safe).""" - if self._selection == "random": - return self._rng.choice(self._pages) - else: # sequential - with self._lock: + with self._lock: + if self._selection == "random": + return self._rng.choice(self._pages) + else: # sequential page = self._pages[self._index] self._index = (self._index + 1) % len(self._pages) - return page + return page def reset(self) -> None: """Reset sequential index to beginning.""" diff --git a/src/errorworks/web/metrics.py b/src/errorworks/web/metrics.py index 3c8f07c..4a190b1 100644 --- a/src/errorworks/web/metrics.py +++ b/src/errorworks/web/metrics.py @@ -75,7 +75,8 @@ def _classify_web_outcome( forbidden=status_code == 403, not_found=status_code == 404, server_error=status_code is not None and 500 <= status_code < 600, - connection_error=error_type + connection_error=status_code is None + and error_type in ( "timeout", "connection_reset", diff --git a/tests/unit/engine/test_metrics_store.py b/tests/unit/engine/test_metrics_store.py index 5adb0d0..62ea9d9 100644 --- a/tests/unit/engine/test_metrics_store.py +++ b/tests/unit/engine/test_metrics_store.py @@ -571,6 +571,24 @@ def test_get_requests_by_outcome(self, store: MetricsStore) -> None: assert len(rows) == 1 assert rows[0]["outcome"] == "success" + def test_get_requests_outcome_filter_without_outcome_column(self) -> None: + """get_requests raises ValueError when filtering by outcome on a schema without that column.""" + schema_no_outcome = MetricsSchema( + request_columns=( + ColumnDef("request_id", "TEXT", nullable=False, primary_key=True), + ColumnDef("timestamp_utc", "TEXT", nullable=False), + ), + timeseries_columns=( + ColumnDef("bucket_utc", "TEXT", nullable=False, primary_key=True), + ColumnDef("requests_total", "INTEGER", nullable=False, default="0"), + ), + ) + config = MetricsConfig(database=":memory:") + store = MetricsStore(config, schema_no_outcome) + with pytest.raises(ValueError, match="outcome"): + store.get_requests(outcome="success") + store.close() + def test_get_timeseries_limit(self, store: MetricsStore) -> None: """get_timeseries respects limit parameter.""" for i in range(5): diff --git a/tests/unit/llm/test_response_generator.py b/tests/unit/llm/test_response_generator.py index 33bafea..8d5db6e 100644 --- a/tests/unit/llm/test_response_generator.py +++ b/tests/unit/llm/test_response_generator.py @@ -985,3 +985,34 @@ def get_bank() -> None: assert len(results) == 10 # All threads must receive the exact same PresetBank instance assert all(b is results[0] for b in results) + + +class TestPresetBankRandomThreadSafety: + """Verify PresetBank.next() in random mode is thread-safe.""" + + def test_concurrent_random_next_no_crash(self) -> None: + """Spawn many threads calling next() in random mode concurrently.""" + responses = [f"response-{i}" for i in range(10)] + bank = PresetBank(responses=responses, selection="random") + errors: list[Exception] = [] + results: list[str] = [] + barrier = threading.Barrier(20) + + def worker() -> None: + barrier.wait() + try: + for _ in range(200): + result = bank.next() + results.append(result) + except Exception as exc: + errors.append(exc) + + threads = [threading.Thread(target=worker) for _ in range(20)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Thread errors: {errors}" + assert len(results) == 20 * 200 + assert all(r in responses for r in results) diff --git a/tests/unit/web/test_content_generator.py b/tests/unit/web/test_content_generator.py index e19a71b..04f22f2 100644 --- a/tests/unit/web/test_content_generator.py +++ b/tests/unit/web/test_content_generator.py @@ -663,3 +663,36 @@ def test_returns_known_types(self) -> None: for _ in range(50): ct = generate_wrong_content_type() assert ct in known + + +class TestPresetBankRandomThreadSafety: + """Verify PresetBank.next() in random mode is thread-safe.""" + + def test_concurrent_random_next_no_crash(self) -> None: + """Spawn many threads calling next() in random mode concurrently.""" + import threading + + pages = [{"body": f"

page-{i}

", "content_type": "text/html"} for i in range(10)] + bank = PresetBank(pages=pages, selection="random") + errors: list[Exception] = [] + results: list[dict[str, str]] = [] + barrier = threading.Barrier(20) + + def worker() -> None: + barrier.wait() + try: + for _ in range(200): + result = bank.next() + results.append(result) + except Exception as exc: + errors.append(exc) + + threads = [threading.Thread(target=worker) for _ in range(20)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Thread errors: {errors}" + assert len(results) == 20 * 200 + assert all(r in pages for r in results) diff --git a/tests/unit/web/test_metrics.py b/tests/unit/web/test_metrics.py index e1f5487..d0626a7 100644 --- a/tests/unit/web/test_metrics.py +++ b/tests/unit/web/test_metrics.py @@ -117,6 +117,11 @@ def test_connection_error_incomplete_response(self) -> None: result = _classify_web_outcome("error_injected", None, "incomplete_response") assert result.connection_error is True + def test_connection_error_false_when_status_code_present(self) -> None: + """Error type in connection list but with a status code should NOT be connection_error.""" + result = _classify_web_outcome("success", 200, "timeout") + assert result.connection_error is False + def test_malformed_outcome(self) -> None: """error_malformed outcome is classified correctly.""" result = _classify_web_outcome("error_malformed", 200, None) From d6b0da1cefb494f84f4da83fb69081227552246c Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 00:54:35 +1100 Subject: [PATCH 03/17] Fix 2 P4 tasks: annotate unreachable return in _select_weighted, document BurstConfig disabled-state timing - injection_engine: add invariant comment + pragma: no cover on unreachable return None - types: document that duration_sec >= interval_sec is allowed when enabled=False Co-Authored-By: Claude Opus 4.6 (1M context) --- src/errorworks/engine/injection_engine.py | 4 +++- src/errorworks/engine/types.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/errorworks/engine/injection_engine.py b/src/errorworks/engine/injection_engine.py index 4e4d755..af59fe8 100644 --- a/src/errorworks/engine/injection_engine.py +++ b/src/errorworks/engine/injection_engine.py @@ -168,7 +168,9 @@ def _select_weighted(self, specs: list[ErrorSpec]) -> ErrorSpec | None: if roll < threshold: return spec - return None + # Unreachable: roll < total_weight (guarded above) guarantees a match + # in the loop. Defensive return for static analysis / type checkers. + return None # pragma: no cover def reset(self) -> None: """Reset the engine state (clears burst timing).""" diff --git a/src/errorworks/engine/types.py b/src/errorworks/engine/types.py index 296c202..662f771 100644 --- a/src/errorworks/engine/types.py +++ b/src/errorworks/engine/types.py @@ -138,6 +138,11 @@ class BurstConfig: enabled: Whether burst mode is active. interval_sec: Time between burst starts in seconds. duration_sec: How long each burst lasts in seconds. + + Note: + When ``enabled=False``, ``duration_sec >= interval_sec`` is permitted. + This lets users pre-configure burst timing without triggering validation + errors until burst mode is actually activated. """ enabled: bool = False From 55109a72eaf4e1f96e783930dcbdce2b03f724cf Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 01:45:07 +1100 Subject: [PATCH 04/17] Fix 11 P3 tasks: docs, defensive coding, API consistency, MCP sync, SqlType enum Documentation: - Add __all__ to top-level errorworks/__init__.py - Document should_trigger behavior for percentage > 100 - Document update_config snapshot pattern in both servers - Document _cleanup_stale_connections thread ID reuse limitation Defensive coding: - validate_error_decision: raise on unknown categories (with extra_categories escape hatch) - handle_admin_config: reject non-dict request bodies with 400 - load_config: only inject preset_name if config model has the field API consistency: - Unify LLM _template_random_words to match Web signature (min_count, max_count) - MCP describe_schema: generate from LLM_METRICS_SCHEMA instead of hardcoded - MCP find_anomalies: derive expected status codes from HTTP_ERRORS Type safety: - Replace ColumnDef.sql_type: str with SqlType StrEnum across all call sites Co-Authored-By: Claude Opus 4.6 (1M context) --- src/errorworks/__init__.py | 11 +++ src/errorworks/engine/__init__.py | 2 + src/errorworks/engine/admin.py | 5 ++ src/errorworks/engine/config_loader.py | 3 +- src/errorworks/engine/injection_engine.py | 4 +- src/errorworks/engine/metrics_store.py | 9 ++- src/errorworks/engine/types.py | 14 +++- src/errorworks/engine/validators.py | 10 +++ src/errorworks/llm/metrics.py | 54 +++++++------- src/errorworks/llm/response_generator.py | 14 +++- src/errorworks/llm/server.py | 5 ++ src/errorworks/llm_mcp/server.py | 52 +++++--------- src/errorworks/web/error_injector.py | 1 + src/errorworks/web/metrics.py | 52 +++++++------- src/errorworks/web/server.py | 5 ++ tests/unit/engine/test_admin.py | 19 +++++ tests/unit/engine/test_config_loader.py | 30 ++++++++ tests/unit/engine/test_metrics_store.py | 62 ++++++++-------- tests/unit/engine/test_types.py | 53 +++++++------- tests/unit/engine/test_validators.py | 88 ++++++++++++++++++++++- tests/unit/llm/test_response_generator.py | 25 +++++-- 21 files changed, 358 insertions(+), 160 deletions(-) diff --git a/src/errorworks/__init__.py b/src/errorworks/__init__.py index 1e0991f..a89fce3 100644 --- a/src/errorworks/__init__.py +++ b/src/errorworks/__init__.py @@ -1,3 +1,14 @@ """Composable chaos-testing servers for LLM and web scraping pipelines.""" __version__ = "0.1.0" + +__all__ = [ + "__version__", + # Engine (shared core) + "engine", + # Chaos plugins + "llm", + "web", + # Test support + "testing", +] diff --git a/src/errorworks/engine/__init__.py b/src/errorworks/engine/__init__.py index dcadb41..e662ac6 100644 --- a/src/errorworks/engine/__init__.py +++ b/src/errorworks/engine/__init__.py @@ -31,6 +31,7 @@ MetricsSchema, SelectionMode, ServerConfig, + SqlType, ) from errorworks.engine.vocabulary import ENGLISH_VOCABULARY, LOREM_VOCABULARY @@ -49,6 +50,7 @@ "MetricsStore", "SelectionMode", "ServerConfig", + "SqlType", "deep_merge", "list_presets", "load_preset", diff --git a/src/errorworks/engine/admin.py b/src/errorworks/engine/admin.py index 65172fb..3d4cec7 100644 --- a/src/errorworks/engine/admin.py +++ b/src/errorworks/engine/admin.py @@ -66,6 +66,11 @@ async def handle_admin_config(request: Request, server: ChaosServer) -> JSONResp {"error": {"type": "invalid_request_error", "message": "Request body must be valid JSON"}}, status_code=400, ) + if not isinstance(body, dict): + return JSONResponse( + {"error": {"type": "invalid_request_error", "message": "Request body must be a JSON object"}}, + status_code=400, + ) try: server.update_config(body) except (ValueError, TypeError, pydantic.ValidationError) as e: diff --git a/src/errorworks/engine/config_loader.py b/src/errorworks/engine/config_loader.py index 58a9347..497027d 100644 --- a/src/errorworks/engine/config_loader.py +++ b/src/errorworks/engine/config_loader.py @@ -148,7 +148,8 @@ def load_config[ConfigT: BaseModel]( config_dict = deep_merge(config_dict, cli_overrides) # Record preset name used for this config (if any) - config_dict["preset_name"] = preset + if "preset_name" in config_cls.model_fields: + config_dict["preset_name"] = preset # Validate and return return config_cls(**config_dict) diff --git a/src/errorworks/engine/injection_engine.py b/src/errorworks/engine/injection_engine.py index af59fe8..c108357 100644 --- a/src/errorworks/engine/injection_engine.py +++ b/src/errorworks/engine/injection_engine.py @@ -110,7 +110,9 @@ def should_trigger(self, percentage: float) -> bool: """Determine if an error should trigger based on percentage. Args: - percentage: Error percentage (0-100). + percentage: Error percentage (0-100). Values >= 100 always trigger + since ``random() * 100`` is strictly less than 100. + Values <= 0 never trigger (short-circuited). Returns: True if the error should trigger. diff --git a/src/errorworks/engine/metrics_store.py b/src/errorworks/engine/metrics_store.py index 05ec403..bd14ebd 100644 --- a/src/errorworks/engine/metrics_store.py +++ b/src/errorworks/engine/metrics_store.py @@ -188,7 +188,14 @@ def _get_connection(self) -> sqlite3.Connection: return conn def _cleanup_stale_connections(self) -> None: - """Close connections from threads that have exited.""" + """Close connections from threads that have exited. + + Note: Thread IDs can be reused by the OS. If a new thread receives the + same ID as a previously exited thread, the stale connection will not be + detected (it looks "live") and the new thread's connection will overwrite + it in ``_connections``, leaking the old one. This is an acceptable + trade-off for a testing tool where long-lived thread churn is uncommon. + """ live_thread_ids = {t.ident for t in threading.enumerate()} with self._lock: dead_ids = [tid for tid in self._connections if tid not in live_thread_ids] diff --git a/src/errorworks/engine/types.py b/src/errorworks/engine/types.py index 662f771..5662076 100644 --- a/src/errorworks/engine/types.py +++ b/src/errorworks/engine/types.py @@ -165,7 +165,17 @@ def __post_init__(self) -> None: # ============================================================================= -_VALID_SQL_TYPES = frozenset({"TEXT", "INTEGER", "REAL", "BLOB", "NUMERIC"}) +class SqlType(StrEnum): + """Valid SQLite column types for metrics schema definitions.""" + + TEXT = "TEXT" + INTEGER = "INTEGER" + REAL = "REAL" + BLOB = "BLOB" + NUMERIC = "NUMERIC" + + +_VALID_SQL_TYPES = frozenset(SqlType) _VALID_COLUMN_NAME = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") # Safe DEFAULT expressions: NULL, numeric literals, quoted strings. @@ -193,7 +203,7 @@ class ColumnDef: """ name: str - sql_type: str + sql_type: SqlType nullable: bool = True default: str | None = None primary_key: bool = False diff --git a/src/errorworks/engine/validators.py b/src/errorworks/engine/validators.py index 88859da..0722435 100644 --- a/src/errorworks/engine/validators.py +++ b/src/errorworks/engine/validators.py @@ -56,6 +56,7 @@ def validate_error_decision( malformed_category: StrEnum, valid_error_types: set[str], valid_malformed_types: set[str], + extra_categories: frozenset[StrEnum] | None = None, ) -> None: """Validate shared invariants for error decision dataclasses. @@ -77,6 +78,9 @@ def validate_error_decision( malformed_category: The MALFORMED category enum member for comparison. valid_error_types: Set of all valid error_type values. valid_malformed_types: Set of all valid malformed_type values. + extra_categories: Optional frozenset of plugin-specific category enum + members that this function should accept without category-specific + validation (the caller handles those). Raises: ValueError: If any invariant is violated. @@ -115,6 +119,12 @@ def validate_error_decision( if status_code is not None and status_code != 200: raise ValueError(f"Malformed error must have status_code 200, got {status_code}") + elif extra_categories is not None and category in extra_categories: + pass # Plugin-specific category — validated by caller + + else: + raise ValueError(f"Unknown error category: {category}") + if retry_after_sec is not None and retry_after_sec < 0: raise ValueError(f"retry_after_sec must be non-negative, got {retry_after_sec}") if delay_sec is not None and delay_sec < 0: diff --git a/src/errorworks/llm/metrics.py b/src/errorworks/llm/metrics.py index 08243f4..2ff5fd1 100644 --- a/src/errorworks/llm/metrics.py +++ b/src/errorworks/llm/metrics.py @@ -8,39 +8,39 @@ from typing import Any, NamedTuple from errorworks.engine.metrics_store import MetricsStore -from errorworks.engine.types import ColumnDef, MetricsConfig, MetricsSchema +from errorworks.engine.types import ColumnDef, MetricsConfig, MetricsSchema, SqlType # Schema definition for LLM metrics tables. LLM_METRICS_SCHEMA = MetricsSchema( request_columns=( - ColumnDef("request_id", "TEXT", nullable=False, primary_key=True), - ColumnDef("timestamp_utc", "TEXT", nullable=False), - ColumnDef("endpoint", "TEXT", nullable=False), - ColumnDef("deployment", "TEXT"), - ColumnDef("model", "TEXT"), - ColumnDef("outcome", "TEXT", nullable=False), - ColumnDef("status_code", "INTEGER"), - ColumnDef("error_type", "TEXT"), - ColumnDef("injection_type", "TEXT"), - ColumnDef("latency_ms", "REAL"), - ColumnDef("injected_delay_ms", "REAL"), - ColumnDef("message_count", "INTEGER"), - ColumnDef("prompt_tokens_approx", "INTEGER"), - ColumnDef("response_tokens", "INTEGER"), - ColumnDef("response_mode", "TEXT"), + ColumnDef("request_id", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("timestamp_utc", SqlType.TEXT, nullable=False), + ColumnDef("endpoint", SqlType.TEXT, nullable=False), + ColumnDef("deployment", SqlType.TEXT), + ColumnDef("model", SqlType.TEXT), + ColumnDef("outcome", SqlType.TEXT, nullable=False), + ColumnDef("status_code", SqlType.INTEGER), + ColumnDef("error_type", SqlType.TEXT), + ColumnDef("injection_type", SqlType.TEXT), + ColumnDef("latency_ms", SqlType.REAL), + ColumnDef("injected_delay_ms", SqlType.REAL), + ColumnDef("message_count", SqlType.INTEGER), + ColumnDef("prompt_tokens_approx", SqlType.INTEGER), + ColumnDef("response_tokens", SqlType.INTEGER), + ColumnDef("response_mode", SqlType.TEXT), ), timeseries_columns=( - ColumnDef("bucket_utc", "TEXT", nullable=False, primary_key=True), - ColumnDef("requests_total", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_success", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_rate_limited", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_capacity_error", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_server_error", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_client_error", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_connection_error", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_malformed", "INTEGER", nullable=False, default="0"), - ColumnDef("avg_latency_ms", "REAL"), - ColumnDef("p99_latency_ms", "REAL"), + ColumnDef("bucket_utc", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("requests_total", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_success", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_rate_limited", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_capacity_error", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_server_error", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_client_error", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_connection_error", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_malformed", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("avg_latency_ms", SqlType.REAL), + ColumnDef("p99_latency_ms", SqlType.REAL), ), request_indexes=( ("idx_requests_timestamp", "timestamp_utc"), diff --git a/src/errorworks/llm/response_generator.py b/src/errorworks/llm/response_generator.py index cd7dc97..eb4e9cd 100644 --- a/src/errorworks/llm/response_generator.py +++ b/src/errorworks/llm/response_generator.py @@ -279,9 +279,17 @@ def _template_random_int(self, min_val: int = 0, max_val: int = 100) -> int: """Jinja2 helper: Generate random integer in range.""" return self._rng.randint(min_val, max_val) - def _template_random_words(self, count: int = 5, vocabulary: str = "english") -> str: - """Jinja2 helper: Generate random words.""" - vocab = get_vocabulary(vocabulary) + def _template_random_words(self, min_count: int = 5, max_count: int | None = None) -> str: + """Jinja2 helper: Generate random words. + + Can be called as random_words(50) for exactly 50 words, + or random_words(50, 100) for 50-100 words. + """ + if max_count is None: + count = min_count + else: + count = self._rng.randint(min_count, max_count) + vocab = self._get_vocabulary() words = [self._rng.choice(vocab) for _ in range(count)] return " ".join(words) diff --git a/src/errorworks/llm/server.py b/src/errorworks/llm/server.py index 3cf1e1f..44acce9 100644 --- a/src/errorworks/llm/server.py +++ b/src/errorworks/llm/server.py @@ -173,6 +173,11 @@ def update_config(self, updates: dict[str, Any]) -> None: swapped atomically under _config_lock to prevent concurrent request handlers from seeing a half-updated configuration. + Note: Request handlers that have already snapshotted component references + (the config snapshot pattern) will continue using the old components for + the remainder of that request. This is intentional — it guarantees each + request sees a consistent configuration throughout its lifetime. + Args: updates: Dict with sections to update (error_injection, response, latency) """ diff --git a/src/errorworks/llm_mcp/server.py b/src/errorworks/llm_mcp/server.py index d549d37..80a113d 100644 --- a/src/errorworks/llm_mcp/server.py +++ b/src/errorworks/llm_mcp/server.py @@ -26,6 +26,9 @@ from mcp.server.stdio import stdio_server from mcp.types import CallToolResult, TextContent, Tool +from errorworks.llm.error_injector import HTTP_ERRORS +from errorworks.llm.metrics import LLM_METRICS_SCHEMA + logger = logging.getLogger(__name__) _SQLITE_OK = sqlite3.SQLITE_OK @@ -495,7 +498,7 @@ def find_anomalies(self) -> dict[str, Any]: # Check for unexpected status codes (not in the set of codes # that ChaosLLM's error injector can produce) - expected_codes = {200, 403, 404, 429, 500, 502, 503, 504, 529} + expected_codes = {200} | set(HTTP_ERRORS.values()) cursor = conn.execute( """ SELECT status_code, COUNT(*) as count @@ -809,50 +812,31 @@ def query(self, sql: str) -> list[dict[str, Any]]: return [dict(zip(columns, row, strict=True)) for row in rows] + @staticmethod + def _format_column(col: Any) -> str: + """Format a ColumnDef as a human-readable schema description.""" + parts = [f"{col.name} ({col.sql_type}"] + if col.primary_key: + parts.append(" PK") + parts.append(")") + return "".join(parts) + def describe_schema(self) -> dict[str, Any]: """Describe the metrics database schema. - NOTE: This description must be kept in sync with LLM_METRICS_SCHEMA - in errorworks/llm/metrics.py. Changes to the schema definition there - must be reflected here manually. + Generated from LLM_METRICS_SCHEMA so it stays in sync automatically + when columns are added or removed. """ + schema = LLM_METRICS_SCHEMA return { "tables": { "requests": { "description": "Individual request records", - "columns": [ - "request_id (TEXT PK)", - "timestamp_utc (TEXT)", - "endpoint (TEXT)", - "deployment (TEXT)", - "model (TEXT)", - "outcome (TEXT: success/error_injected/error_malformed)", - "status_code (INTEGER)", - "error_type (TEXT)", - "injection_type (TEXT)", - "latency_ms (REAL)", - "injected_delay_ms (REAL)", - "message_count (INTEGER)", - "prompt_tokens_approx (INTEGER)", - "response_tokens (INTEGER)", - "response_mode (TEXT)", - ], + "columns": [self._format_column(col) for col in schema.request_columns], }, "timeseries": { "description": "Time-bucketed aggregations", - "columns": [ - "bucket_utc (TEXT PK)", - "requests_total (INTEGER)", - "requests_success (INTEGER)", - "requests_rate_limited (INTEGER)", - "requests_capacity_error (INTEGER)", - "requests_server_error (INTEGER)", - "requests_client_error (INTEGER)", - "requests_connection_error (INTEGER)", - "requests_malformed (INTEGER)", - "avg_latency_ms (REAL)", - "p99_latency_ms (REAL)", - ], + "columns": [self._format_column(col) for col in schema.timeseries_columns], }, "run_info": { "description": "Run metadata", diff --git a/src/errorworks/web/error_injector.py b/src/errorworks/web/error_injector.py index 1eab750..ce3d7f0 100644 --- a/src/errorworks/web/error_injector.py +++ b/src/errorworks/web/error_injector.py @@ -128,6 +128,7 @@ def __post_init__(self) -> None: malformed_category=WebErrorCategory.MALFORMED, valid_error_types=WEB_HTTP_ERRORS.keys() | WEB_CONNECTION_ERRORS | {"malformed"} | WEB_REDIRECT_TYPES, valid_malformed_types=WEB_MALFORMED_TYPES, + extra_categories=frozenset({WebErrorCategory.REDIRECT}), ) # Web-specific validation beyond the shared checks diff --git a/src/errorworks/web/metrics.py b/src/errorworks/web/metrics.py index 4a190b1..af367dc 100644 --- a/src/errorworks/web/metrics.py +++ b/src/errorworks/web/metrics.py @@ -8,7 +8,7 @@ from typing import Any, NamedTuple from errorworks.engine.metrics_store import MetricsStore -from errorworks.engine.types import ColumnDef, MetricsConfig, MetricsSchema +from errorworks.engine.types import ColumnDef, MetricsConfig, MetricsSchema, SqlType class WebOutcomeClassification(NamedTuple): @@ -27,33 +27,33 @@ class WebOutcomeClassification(NamedTuple): # Schema definition for web metrics tables. WEB_METRICS_SCHEMA = MetricsSchema( request_columns=( - ColumnDef("request_id", "TEXT", nullable=False, primary_key=True), - ColumnDef("timestamp_utc", "TEXT", nullable=False), - ColumnDef("path", "TEXT", nullable=False), - ColumnDef("outcome", "TEXT", nullable=False), - ColumnDef("status_code", "INTEGER"), - ColumnDef("error_type", "TEXT"), - ColumnDef("injection_type", "TEXT"), - ColumnDef("latency_ms", "REAL"), - ColumnDef("injected_delay_ms", "REAL"), - ColumnDef("content_type_served", "TEXT"), - ColumnDef("encoding_served", "TEXT"), - ColumnDef("redirect_target", "TEXT"), - ColumnDef("redirect_hops", "INTEGER"), + ColumnDef("request_id", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("timestamp_utc", SqlType.TEXT, nullable=False), + ColumnDef("path", SqlType.TEXT, nullable=False), + ColumnDef("outcome", SqlType.TEXT, nullable=False), + ColumnDef("status_code", SqlType.INTEGER), + ColumnDef("error_type", SqlType.TEXT), + ColumnDef("injection_type", SqlType.TEXT), + ColumnDef("latency_ms", SqlType.REAL), + ColumnDef("injected_delay_ms", SqlType.REAL), + ColumnDef("content_type_served", SqlType.TEXT), + ColumnDef("encoding_served", SqlType.TEXT), + ColumnDef("redirect_target", SqlType.TEXT), + ColumnDef("redirect_hops", SqlType.INTEGER), ), timeseries_columns=( - ColumnDef("bucket_utc", "TEXT", nullable=False, primary_key=True), - ColumnDef("requests_total", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_success", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_rate_limited", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_forbidden", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_not_found", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_server_error", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_connection_error", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_malformed", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_redirect", "INTEGER", nullable=False, default="0"), - ColumnDef("avg_latency_ms", "REAL"), - ColumnDef("p99_latency_ms", "REAL"), + ColumnDef("bucket_utc", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("requests_total", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_success", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_rate_limited", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_forbidden", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_not_found", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_server_error", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_connection_error", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_malformed", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_redirect", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("avg_latency_ms", SqlType.REAL), + ColumnDef("p99_latency_ms", SqlType.REAL), ), request_indexes=( ("idx_requests_timestamp", "timestamp_utc"), diff --git a/src/errorworks/web/server.py b/src/errorworks/web/server.py index 23ac281..27497ba 100644 --- a/src/errorworks/web/server.py +++ b/src/errorworks/web/server.py @@ -146,6 +146,11 @@ def update_config(self, updates: dict[str, Any]) -> None: Components are built outside the lock (validation may be expensive), then swapped atomically under _config_lock to prevent concurrent request handlers from seeing a half-updated configuration. + + Note: Request handlers that have already snapshotted component references + (the config snapshot pattern) will continue using the old components for + the remainder of that request. This is intentional — it guarantees each + request sees a consistent configuration throughout its lifetime. """ # Build new components outside the lock new_error: WebErrorInjector | None = None diff --git a/tests/unit/engine/test_admin.py b/tests/unit/engine/test_admin.py index b0e7040..22a36e6 100644 --- a/tests/unit/engine/test_admin.py +++ b/tests/unit/engine/test_admin.py @@ -193,6 +193,25 @@ def test_post_validation_error_returns_422(self) -> None: assert resp.status_code == 422 assert "invalid mode" in resp.json()["error"]["message"] + def test_post_list_body_returns_400(self) -> None: + """POST with a JSON array instead of object returns 400.""" + server = _make_mock_server() + client = _make_app(server) + resp = client.post("/admin/config", json=[1, 2, 3], headers=self._headers()) + assert resp.status_code == 400 + assert resp.json()["error"]["type"] == "invalid_request_error" + assert "JSON object" in resp.json()["error"]["message"] + server.update_config.assert_not_called() + + def test_post_string_body_returns_400(self) -> None: + """POST with a JSON string instead of object returns 400.""" + server = _make_mock_server() + client = _make_app(server) + resp = client.post("/admin/config", json="hello", headers=self._headers()) + assert resp.status_code == 400 + assert resp.json()["error"]["type"] == "invalid_request_error" + server.update_config.assert_not_called() + # ============================================================================= # handle_admin_stats diff --git a/tests/unit/engine/test_config_loader.py b/tests/unit/engine/test_config_loader.py index e1ee30b..efb27e8 100644 --- a/tests/unit/engine/test_config_loader.py +++ b/tests/unit/engine/test_config_loader.py @@ -165,6 +165,36 @@ def test_path_traversal_names_rejected(self, tmp_path: Path, name: str) -> None: # ============================================================================= +class TestLoadConfigPresetNameField: + """Tests for load_config conditional preset_name injection.""" + + def test_skips_preset_name_when_model_lacks_field(self, tmp_path: Path) -> None: + """load_config does not inject preset_name if config_cls lacks that field.""" + from pydantic import BaseModel + + class SimpleConfig(BaseModel): + mode: str = "default" + + presets_dir = tmp_path / "presets" + presets_dir.mkdir() + (presets_dir / "basic.yaml").write_text(yaml.dump({"mode": "custom"})) + + # Should not raise even though SimpleConfig has no preset_name field + config = load_config(SimpleConfig, presets_dir, preset="basic") + assert config.mode == "custom" + + def test_injects_preset_name_when_model_has_field(self, tmp_path: Path) -> None: + """load_config still injects preset_name if config_cls has that field.""" + from errorworks.llm.config import ChaosLLMConfig + + presets_dir = tmp_path / "presets" + presets_dir.mkdir() + (presets_dir / "test.yaml").write_text(yaml.dump({})) + + config = load_config(ChaosLLMConfig, presets_dir, preset="test") + assert config.preset_name == "test" + + class TestLoadConfigFileValidation: """Tests for load_config config file type validation.""" diff --git a/tests/unit/engine/test_metrics_store.py b/tests/unit/engine/test_metrics_store.py index 62ea9d9..2e24e6e 100644 --- a/tests/unit/engine/test_metrics_store.py +++ b/tests/unit/engine/test_metrics_store.py @@ -12,24 +12,24 @@ import pytest from errorworks.engine.metrics_store import MetricsStore, _column_ddl, _generate_ddl, _get_bucket_utc -from errorworks.engine.types import ColumnDef, MetricsConfig, MetricsSchema +from errorworks.engine.types import ColumnDef, MetricsConfig, MetricsSchema, SqlType # Minimal test schema for MetricsStore unit tests. _TEST_SCHEMA = MetricsSchema( request_columns=( - ColumnDef("request_id", "TEXT", nullable=False, primary_key=True), - ColumnDef("timestamp_utc", "TEXT", nullable=False), - ColumnDef("outcome", "TEXT", nullable=False), - ColumnDef("status_code", "INTEGER"), - ColumnDef("latency_ms", "REAL"), + ColumnDef("request_id", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("timestamp_utc", SqlType.TEXT, nullable=False), + ColumnDef("outcome", SqlType.TEXT, nullable=False), + ColumnDef("status_code", SqlType.INTEGER), + ColumnDef("latency_ms", SqlType.REAL), ), timeseries_columns=( - ColumnDef("bucket_utc", "TEXT", nullable=False, primary_key=True), - ColumnDef("requests_total", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_success", "INTEGER", nullable=False, default="0"), - ColumnDef("requests_error", "INTEGER", nullable=False, default="0"), - ColumnDef("avg_latency_ms", "REAL"), - ColumnDef("p99_latency_ms", "REAL"), + ColumnDef("bucket_utc", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("requests_total", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_success", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("requests_error", SqlType.INTEGER, nullable=False, default="0"), + ColumnDef("avg_latency_ms", SqlType.REAL), + ColumnDef("p99_latency_ms", SqlType.REAL), ), request_indexes=( ("idx_req_ts", "timestamp_utc"), @@ -48,53 +48,53 @@ class TestColumnDefValidation: def test_valid_numeric_default(self) -> None: """Numeric defaults are accepted.""" - col = ColumnDef("count", "INTEGER", default="0") + col = ColumnDef("count", SqlType.INTEGER, default="0") assert col.default == "0" def test_valid_null_default(self) -> None: """NULL default is accepted.""" - col = ColumnDef("value", "TEXT", default="NULL") + col = ColumnDef("value", SqlType.TEXT, default="NULL") assert col.default == "NULL" def test_valid_quoted_string_default(self) -> None: """Single-quoted string default is accepted.""" - col = ColumnDef("status", "TEXT", default="'active'") + col = ColumnDef("status", SqlType.TEXT, default="'active'") assert col.default == "'active'" def test_valid_negative_numeric_default(self) -> None: """Negative numeric default is accepted.""" - col = ColumnDef("offset_val", "INTEGER", default="-1") + col = ColumnDef("offset_val", SqlType.INTEGER, default="-1") assert col.default == "-1" def test_valid_float_default(self) -> None: """Float default is accepted.""" - col = ColumnDef("ratio", "REAL", default="0.5") + col = ColumnDef("ratio", SqlType.REAL, default="0.5") assert col.default == "0.5" def test_no_default_is_fine(self) -> None: """None default is accepted (no DEFAULT clause).""" - col = ColumnDef("name", "TEXT", default=None) + col = ColumnDef("name", SqlType.TEXT, default=None) assert col.default is None def test_sql_injection_in_default_raises(self) -> None: """SQL injection attempt in default is rejected.""" with pytest.raises(ValueError, match="default must be"): - ColumnDef("x", "TEXT", default="0; DROP TABLE requests") + ColumnDef("x", SqlType.TEXT, default="0; DROP TABLE requests") def test_subquery_in_default_raises(self) -> None: """Subquery in default is rejected.""" with pytest.raises(ValueError, match="default must be"): - ColumnDef("x", "TEXT", default="(SELECT 1)") + ColumnDef("x", SqlType.TEXT, default="(SELECT 1)") def test_function_call_in_default_raises(self) -> None: """Function call in default is rejected.""" with pytest.raises(ValueError, match="default must be"): - ColumnDef("x", "TEXT", default="CURRENT_TIMESTAMP") + ColumnDef("x", SqlType.TEXT, default="CURRENT_TIMESTAMP") def test_unquoted_string_in_default_raises(self) -> None: """Unquoted string in default is rejected.""" with pytest.raises(ValueError, match="default must be"): - ColumnDef("x", "TEXT", default="active") + ColumnDef("x", SqlType.TEXT, default="active") # ============================================================================= @@ -140,12 +140,12 @@ def test_no_indexes_when_empty(self) -> None: """Schema with no indexes generates no CREATE INDEX.""" schema = MetricsSchema( request_columns=( - ColumnDef("id", "TEXT", nullable=False, primary_key=True), - ColumnDef("timestamp_utc", "TEXT", nullable=False), + ColumnDef("id", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("timestamp_utc", SqlType.TEXT, nullable=False), ), timeseries_columns=( - ColumnDef("bucket_utc", "TEXT", nullable=False, primary_key=True), - ColumnDef("requests_total", "INTEGER", nullable=False, default="0"), + ColumnDef("bucket_utc", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("requests_total", SqlType.INTEGER, nullable=False, default="0"), ), ) ddl = _generate_ddl(schema) @@ -153,7 +153,7 @@ def test_no_indexes_when_empty(self) -> None: def test_primary_key_text_column_emits_not_null(self) -> None: """TEXT PRIMARY KEY columns must emit NOT NULL for SQLite correctness.""" - col = ColumnDef(name="id", sql_type="TEXT", nullable=False, primary_key=True) + col = ColumnDef(name="id", sql_type=SqlType.TEXT, nullable=False, primary_key=True) ddl = _column_ddl(col) assert "PRIMARY KEY" in ddl assert "NOT NULL" in ddl @@ -575,12 +575,12 @@ def test_get_requests_outcome_filter_without_outcome_column(self) -> None: """get_requests raises ValueError when filtering by outcome on a schema without that column.""" schema_no_outcome = MetricsSchema( request_columns=( - ColumnDef("request_id", "TEXT", nullable=False, primary_key=True), - ColumnDef("timestamp_utc", "TEXT", nullable=False), + ColumnDef("request_id", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("timestamp_utc", SqlType.TEXT, nullable=False), ), timeseries_columns=( - ColumnDef("bucket_utc", "TEXT", nullable=False, primary_key=True), - ColumnDef("requests_total", "INTEGER", nullable=False, default="0"), + ColumnDef("bucket_utc", SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef("requests_total", SqlType.INTEGER, nullable=False, default="0"), ), ) config = MetricsConfig(database=":memory:") diff --git a/tests/unit/engine/test_types.py b/tests/unit/engine/test_types.py index 0f92e68..ed1ad26 100644 --- a/tests/unit/engine/test_types.py +++ b/tests/unit/engine/test_types.py @@ -18,6 +18,7 @@ ErrorSpec, MetricsSchema, ServerConfig, + SqlType, ) # ============================================================================= @@ -126,24 +127,24 @@ class TestColumnDefValidation: """Tests for ColumnDef.__post_init__ validation.""" def test_valid_column(self) -> None: - col = ColumnDef(name="request_id", sql_type="TEXT", nullable=False, primary_key=True) + col = ColumnDef(name="request_id", sql_type=SqlType.TEXT, nullable=False, primary_key=True) assert col.name == "request_id" def test_valid_column_with_default(self) -> None: - col = ColumnDef(name="count", sql_type="INTEGER", default="0") + col = ColumnDef(name="count", sql_type=SqlType.INTEGER, default="0") assert col.default == "0" def test_valid_column_with_string_default(self) -> None: - col = ColumnDef(name="status", sql_type="TEXT", default="'pending'") + col = ColumnDef(name="status", sql_type=SqlType.TEXT, default="'pending'") assert col.default == "'pending'" def test_valid_column_with_null_default(self) -> None: - col = ColumnDef(name="extra", sql_type="TEXT", default="NULL") + col = ColumnDef(name="extra", sql_type=SqlType.TEXT, default="NULL") assert col.default == "NULL" def test_empty_name_raises(self) -> None: with pytest.raises(ValueError, match="name must not be empty"): - ColumnDef(name="", sql_type="TEXT") + ColumnDef(name="", sql_type=SqlType.TEXT) @pytest.mark.parametrize( "name", @@ -157,12 +158,12 @@ def test_empty_name_raises(self) -> None: ) def test_invalid_column_name_raises(self, name: str) -> None: with pytest.raises(ValueError, match="must be a valid SQL identifier"): - ColumnDef(name=name, sql_type="TEXT") + ColumnDef(name=name, sql_type=SqlType.TEXT) def test_valid_column_names(self) -> None: """Underscores and mixed case are valid SQL identifiers.""" for name in ["_private", "Col123", "UPPER", "lower_case", "a"]: - col = ColumnDef(name=name, sql_type="TEXT") + col = ColumnDef(name=name, sql_type=SqlType.TEXT) assert col.name == name @pytest.mark.parametrize("sql_type", ["VARCHAR", "STRING", "INT", "bool", "FLOAT", ""]) @@ -170,17 +171,17 @@ def test_invalid_sql_type_raises(self, sql_type: str) -> None: with pytest.raises(ValueError, match="sql_type must be one of"): ColumnDef(name="col", sql_type=sql_type) - @pytest.mark.parametrize("sql_type", ["TEXT", "INTEGER", "REAL", "BLOB", "NUMERIC"]) - def test_valid_sql_types(self, sql_type: str) -> None: + @pytest.mark.parametrize("sql_type", [SqlType.TEXT, SqlType.INTEGER, SqlType.REAL, SqlType.BLOB, SqlType.NUMERIC]) + def test_valid_sql_types(self, sql_type: SqlType) -> None: col = ColumnDef(name="col", sql_type=sql_type) assert col.sql_type == sql_type def test_primary_key_nullable_raises(self) -> None: with pytest.raises(ValueError, match="primary_key columns cannot be nullable"): - ColumnDef(name="id", sql_type="INTEGER", nullable=True, primary_key=True) + ColumnDef(name="id", sql_type=SqlType.INTEGER, nullable=True, primary_key=True) def test_primary_key_not_nullable_is_valid(self) -> None: - col = ColumnDef(name="id", sql_type="INTEGER", nullable=False, primary_key=True) + col = ColumnDef(name="id", sql_type=SqlType.INTEGER, nullable=False, primary_key=True) assert col.primary_key is True @pytest.mark.parametrize( @@ -194,7 +195,7 @@ def test_primary_key_not_nullable_is_valid(self) -> None: ) def test_invalid_default_raises(self, default: str) -> None: with pytest.raises(ValueError, match="default must be NULL, a numeric literal, or a single-quoted string"): - ColumnDef(name="col", sql_type="TEXT", default=default) + ColumnDef(name="col", sql_type=SqlType.TEXT, default=default) @pytest.mark.parametrize( "default", @@ -209,7 +210,7 @@ def test_invalid_default_raises(self, default: str) -> None: def test_control_characters_in_quoted_default_raises(self, default: str) -> None: """Control characters in quoted defaults produce malformed DDL and must be rejected.""" with pytest.raises(ValueError, match="default must be NULL, a numeric literal, or a single-quoted string"): - ColumnDef(name="col", sql_type="TEXT", default=default) + ColumnDef(name="col", sql_type=SqlType.TEXT, default=default) # ============================================================================= @@ -219,14 +220,14 @@ def test_control_characters_in_quoted_default_raises(self, default: str) -> None def _minimal_request_columns() -> tuple[ColumnDef, ...]: """Return the minimum required request columns.""" - return (ColumnDef(name="timestamp_utc", sql_type="TEXT"),) + return (ColumnDef(name="timestamp_utc", sql_type=SqlType.TEXT),) def _minimal_timeseries_columns() -> tuple[ColumnDef, ...]: """Return the minimum required timeseries columns.""" return ( - ColumnDef(name="bucket_utc", sql_type="TEXT", nullable=False, primary_key=True), - ColumnDef(name="requests_total", sql_type="INTEGER", default="0"), + ColumnDef(name="bucket_utc", sql_type=SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef(name="requests_total", sql_type=SqlType.INTEGER, default="0"), ) @@ -259,8 +260,8 @@ def test_duplicate_request_column_names_raises(self) -> None: with pytest.raises(ValueError, match="Duplicate request column names"): MetricsSchema( request_columns=( - ColumnDef(name="timestamp_utc", sql_type="TEXT"), - ColumnDef(name="timestamp_utc", sql_type="INTEGER"), + ColumnDef(name="timestamp_utc", sql_type=SqlType.TEXT), + ColumnDef(name="timestamp_utc", sql_type=SqlType.INTEGER), ), timeseries_columns=_minimal_timeseries_columns(), ) @@ -270,9 +271,9 @@ def test_duplicate_timeseries_column_names_raises(self) -> None: MetricsSchema( request_columns=_minimal_request_columns(), timeseries_columns=( - ColumnDef(name="bucket_utc", sql_type="TEXT", nullable=False, primary_key=True), - ColumnDef(name="requests_total", sql_type="INTEGER", default="0"), - ColumnDef(name="requests_total", sql_type="REAL"), + ColumnDef(name="bucket_utc", sql_type=SqlType.TEXT, nullable=False, primary_key=True), + ColumnDef(name="requests_total", sql_type=SqlType.INTEGER, default="0"), + ColumnDef(name="requests_total", sql_type=SqlType.REAL), ), ) @@ -304,20 +305,20 @@ def test_missing_bucket_utc_in_timeseries_raises(self) -> None: with pytest.raises(ValueError, match=r"timeseries_columns must include.*bucket_utc"): MetricsSchema( request_columns=_minimal_request_columns(), - timeseries_columns=(ColumnDef(name="requests_total", sql_type="INTEGER", default="0"),), + timeseries_columns=(ColumnDef(name="requests_total", sql_type=SqlType.INTEGER, default="0"),), ) def test_missing_requests_total_in_timeseries_raises(self) -> None: with pytest.raises(ValueError, match=r"timeseries_columns must include.*requests_total"): MetricsSchema( request_columns=_minimal_request_columns(), - timeseries_columns=(ColumnDef(name="bucket_utc", sql_type="TEXT", nullable=False, primary_key=True),), + timeseries_columns=(ColumnDef(name="bucket_utc", sql_type=SqlType.TEXT, nullable=False, primary_key=True),), ) def test_missing_timestamp_utc_in_request_columns_raises(self) -> None: with pytest.raises(ValueError, match="request_columns must include 'timestamp_utc'"): MetricsSchema( - request_columns=(ColumnDef(name="other_col", sql_type="TEXT"),), + request_columns=(ColumnDef(name="other_col", sql_type=SqlType.TEXT),), timeseries_columns=_minimal_timeseries_columns(), ) @@ -326,8 +327,8 @@ def test_bucket_utc_not_primary_key_raises(self) -> None: MetricsSchema( request_columns=_minimal_request_columns(), timeseries_columns=( - ColumnDef(name="bucket_utc", sql_type="TEXT"), # nullable=True, primary_key=False - ColumnDef(name="requests_total", sql_type="INTEGER", default="0"), + ColumnDef(name="bucket_utc", sql_type=SqlType.TEXT), # nullable=True, primary_key=False + ColumnDef(name="requests_total", sql_type=SqlType.INTEGER, default="0"), ), ) diff --git a/tests/unit/engine/test_validators.py b/tests/unit/engine/test_validators.py index 1f501b5..3c6e656 100644 --- a/tests/unit/engine/test_validators.py +++ b/tests/unit/engine/test_validators.py @@ -2,9 +2,19 @@ from __future__ import annotations +from enum import StrEnum + import pytest -from errorworks.engine.validators import parse_range +from errorworks.engine.validators import parse_range, validate_error_decision + + +class _TestCategory(StrEnum): + HTTP = "http" + CONNECTION = "connection" + MALFORMED = "malformed" + REDIRECT = "redirect" + UNKNOWN = "unknown" class TestParseRangeFloatRejection: @@ -23,3 +33,79 @@ def test_accepts_exact_integer_floats(self) -> None: def test_accepts_plain_integers(self) -> None: assert parse_range([1, 10]) == (1, 10) + + +class TestValidateErrorDecisionUnknownCategory: + """validate_error_decision must reject unknown error categories.""" + + def test_unknown_category_raises_value_error(self) -> None: + """An unrecognised category raises ValueError instead of silently passing.""" + with pytest.raises(ValueError, match="Unknown error category"): + validate_error_decision( + error_type="some_error", + category=_TestCategory.UNKNOWN, + status_code=None, + retry_after_sec=None, + delay_sec=None, + start_delay_sec=None, + malformed_type=None, + http_category=_TestCategory.HTTP, + connection_category=_TestCategory.CONNECTION, + malformed_category=_TestCategory.MALFORMED, + valid_error_types={"some_error"}, + valid_malformed_types=set(), + ) + + def test_known_category_does_not_raise(self) -> None: + """A known HTTP category passes validation without error.""" + validate_error_decision( + error_type="rate_limit", + category=_TestCategory.HTTP, + status_code=429, + retry_after_sec=60, + delay_sec=None, + start_delay_sec=None, + malformed_type=None, + http_category=_TestCategory.HTTP, + connection_category=_TestCategory.CONNECTION, + malformed_category=_TestCategory.MALFORMED, + valid_error_types={"rate_limit"}, + valid_malformed_types=set(), + ) + + def test_extra_category_accepted(self) -> None: + """A plugin-specific category listed in extra_categories passes.""" + validate_error_decision( + error_type="redirect_loop", + category=_TestCategory.REDIRECT, + status_code=None, + retry_after_sec=None, + delay_sec=None, + start_delay_sec=None, + malformed_type=None, + http_category=_TestCategory.HTTP, + connection_category=_TestCategory.CONNECTION, + malformed_category=_TestCategory.MALFORMED, + valid_error_types={"redirect_loop"}, + valid_malformed_types=set(), + extra_categories=frozenset({_TestCategory.REDIRECT}), + ) + + def test_extra_category_not_listed_still_raises(self) -> None: + """A category not in base or extra_categories is still rejected.""" + with pytest.raises(ValueError, match="Unknown error category"): + validate_error_decision( + error_type="redirect_loop", + category=_TestCategory.REDIRECT, + status_code=None, + retry_after_sec=None, + delay_sec=None, + start_delay_sec=None, + malformed_type=None, + http_category=_TestCategory.HTTP, + connection_category=_TestCategory.CONNECTION, + malformed_category=_TestCategory.MALFORMED, + valid_error_types={"redirect_loop"}, + valid_malformed_types=set(), + # extra_categories not provided — REDIRECT is unknown + ) diff --git a/tests/unit/llm/test_response_generator.py b/tests/unit/llm/test_response_generator.py index 8d5db6e..7d4b062 100644 --- a/tests/unit/llm/test_response_generator.py +++ b/tests/unit/llm/test_response_generator.py @@ -462,7 +462,7 @@ def test_random_words_helper(self) -> None: """Template random_words helper generates words.""" config = ResponseConfig( mode="template", - template=TemplateResponseConfig(body="{{ random_words(5, 'english') }}"), + template=TemplateResponseConfig(body="{{ random_words(5) }}"), ) generator = ResponseGenerator(config) @@ -472,17 +472,28 @@ def test_random_words_helper(self) -> None: words = response.content.split() assert len(words) == 5 - def test_random_words_unknown_vocabulary_raises(self) -> None: - """Template random_words with unknown vocabulary raises ValueError.""" + def test_random_words_range(self) -> None: + """Template random_words(min, max) generates words in range.""" config = ResponseConfig( mode="template", - template=TemplateResponseConfig(body="{{ random_words(3, 'klingon') }}"), + template=TemplateResponseConfig(body="{{ random_words(10, 20) }}"), ) generator = ResponseGenerator(config) - request = {"model": "test", "messages": []} - with pytest.raises(ValueError, match="klingon"): - generator.generate(request) + request = {"model": "gpt-4", "messages": []} + response = generator.generate(request) + + words = response.content.split() + assert 10 <= len(words) <= 20 + + def test_random_words_unknown_vocabulary_raises(self) -> None: + """Config with unknown vocabulary is rejected by Pydantic validation.""" + with pytest.raises(Exception, match="klingon|literal_error"): + ResponseConfig( + mode="template", + template=TemplateResponseConfig(body="{{ random_words(3) }}"), + random=RandomResponseConfig(vocabulary="klingon"), + ) def test_timestamp_helper(self) -> None: """Template timestamp helper returns current time.""" From 9a1c12960d9d049b37c21000370837e56cbc95e3 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 07:52:29 +1100 Subject: [PATCH 05/17] Add spec: professionalize errorworks package presentation Design document covering README expansion, PyPI metadata, MkDocs-Material docs site on GitHub Pages, community files, GitHub templates, and pre-commit config. Establishes DTA branding hooks for future use. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...26-03-16-professionalize-package-design.md | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-16-professionalize-package-design.md diff --git a/docs/superpowers/specs/2026-03-16-professionalize-package-design.md b/docs/superpowers/specs/2026-03-16-professionalize-package-design.md new file mode 100644 index 0000000..b0cc7c0 --- /dev/null +++ b/docs/superpowers/specs/2026-03-16-professionalize-package-design.md @@ -0,0 +1,194 @@ +# Professionalize Errorworks Package + +**Date:** 2026-03-16 +**Status:** Draft +**Author:** Claude (with John Morrissey) + +## Context + +Errorworks is a general-purpose composable chaos-testing service framework at v0.1.0 on PyPI. The engineering foundation is strong (583 tests, strict mypy, comprehensive CI/CD, proper src/ layout) but the public-facing presentation — README, docs, metadata, community files — reads as an early-stage hobby project. The goal is to make it look like a professionally maintained tool. + +Errorworks is maintained by the Digital Transformation Agency (DTA). Its primary use case is as a dependency in a reference implementation for semantic data tracing, but it's a general-purpose tool that should stand on its own. DTA branding material doesn't exist yet but should be easy to slot in later. + +The project will grow to include additional server types beyond ChaosLLM and ChaosWeb (e.g. email), so documentation and structure must be extensible. + +## Deliverables + +### 1. README.md (~150 lines) + +Expand from current 3-line stub to a full professional README. + +**Structure:** +- Title + one-line tagline + 1-2 sentence value proposition +- Badge row: CI status, PyPI version, Python versions, License +- **What is errorworks?** — 2-3 paragraphs explaining the problem it solves, who it's for +- **Features** — bullet list grouped by capability: + - Error injection (HTTP errors, connection failures, malformed responses) + - Latency simulation (base + jitter, clamped) + - Response generation modes (random, template, echo, preset) + - Metrics collection (SQLite-backed, timeseries aggregation) + - OpenAI-compatible API server (ChaosLLM) + - Web server for scraping resilience (ChaosWeb) + - Pytest fixtures for in-process testing +- **Quick Start** — `pip install errorworks`, 3-4 line CLI example +- **Usage** — brief examples for: CLI server, pytest fixture, presets; each linking to docs +- **Documentation** — link to MkDocs site +- **Architecture** — brief composition-based design explanation +- Footer: *"An open-source project by the Digital Transformation Agency."* + MIT license + +**Badges** (shields.io / GitHub Actions): +- `![CI](https://github.com/johnm-dta/errorworks/actions/workflows/ci.yml/badge.svg)` +- `![PyPI](https://img.shields.io/pypi/v/errorworks)` +- `![Python](https://img.shields.io/pypi/pyversions/errorworks)` +- `![License](https://img.shields.io/pypi/l/errorworks)` + +### 2. PyPI Metadata (pyproject.toml) + +Update `[project.urls]` from single Repository link to: + +```toml +[project.urls] +Homepage = "https://github.com/johnm-dta/errorworks" +Documentation = "https://johnm-dta.github.io/errorworks" +Changelog = "https://github.com/johnm-dta/errorworks/blob/main/CHANGELOG.md" +"Bug Tracker" = "https://github.com/johnm-dta/errorworks/issues" +Repository = "https://github.com/johnm-dta/errorworks" +``` + +### 3. MkDocs-Material Documentation Site + +Deployed to GitHub Pages at `https://johnm-dta.github.io/errorworks`. + +**Directory structure:** +``` +docs/ +├── index.md # Landing page (value prop, links deeper) +├── getting-started/ +│ ├── installation.md # pip/uv install, prerequisites, verification +│ └── quickstart.md # First server, first request, first chaos scenario +├── guide/ +│ ├── chaosllm.md # ChaosLLM — config, error categories, response modes +│ ├── chaosweb.md # ChaosWeb — same pattern +│ ├── presets.md # Available presets, creating custom ones +│ ├── configuration.md # YAML config, CLI flags, precedence rules +│ ├── metrics.md # MetricsStore, timeseries, querying stats +│ └── testing-fixtures.md # Pytest markers, fixtures, in-process testing +├── reference/ +│ ├── cli.md # chaosengine / chaosllm / chaosweb CLI reference +│ ├── api.md # HTTP API endpoints +│ └── configuration.md # Full config model reference (all fields, defaults, types) +├── architecture.md # Composition pattern, engine components, config snapshot +└── changelog.md # Include or symlink of CHANGELOG.md +``` + +**Design decisions:** +- Each server type (chaosllm, chaosweb, future email etc.) follows the same guide page template, making it easy to add new ones +- `guide/` is the extensibility point — new server types = new pages, no restructuring +- `reference/` separates "how to use" from "what every field means" +- MkDocs-Material theme with search plugin +- DTA attribution in footer; `logo` field commented out in mkdocs.yml, ready for DTA branding +- No `mike` versioning yet — structured so it can be added later without restructuring + +**mkdocs.yml key settings:** +- `site_name: errorworks` +- `site_description: Composable chaos-testing services for LLM and web scraping pipelines` +- `site_url: https://johnm-dta.github.io/errorworks` +- `repo_url: https://github.com/johnm-dta/errorworks` +- `theme: material` with appropriate palette +- `copyright: "An open-source project by the Digital Transformation Agency"` +- Navigation structure matching directory layout above + +### 4. GitHub Actions Docs Workflow + +New `.github/workflows/docs.yml`: +- Triggers on push to `main` (paths: `docs/**`, `mkdocs.yml`) +- Builds MkDocs site +- Deploys to `gh-pages` branch using `peaceiris/actions-gh-pages` or equivalent +- Requires `mkdocs-material` as build dependency + +### 5. Community Files + +**CONTRIBUTING.md** (~30 lines): +- Prerequisites: Python 3.12+, uv +- Setup: `uv sync --all-extras` +- Running tests: `uv run pytest` +- Linting: `uv run ruff check src tests` + `uv run ruff format src tests` +- Type checking: `uv run mypy src` +- PR expectations: tests pass, ruff clean, mypy clean, changelog entry for user-facing changes +- Attribution: "This project is maintained by the Digital Transformation Agency" + +**SECURITY.md** (~15 lines): +- Responsible disclosure instructions (GitHub security advisories preferred) +- Scope clarification: errorworks intentionally generates error responses, malformed data, and simulated faults — these are features, not vulnerabilities. Security issues are things like: arbitrary code execution outside sandbox, unintended data exposure, dependency vulnerabilities. + +**CODE_OF_CONDUCT.md**: +- Adopt Contributor Covenant v2.1 verbatim +- Contact: maintainer email or GitHub discussions + +### 6. GitHub Templates + +**.github/ISSUE_TEMPLATE/bug_report.md:** +- Frontmatter: name, about, labels +- Sections: Describe the bug, Steps to reproduce, Expected behavior, Actual behavior, Environment (OS, Python version, errorworks version) + +**.github/ISSUE_TEMPLATE/feature_request.md:** +- Frontmatter: name, about, labels +- Sections: Use case, Proposed solution, Alternatives considered + +**.github/PULL_REQUEST_TEMPLATE.md:** +- Checklist: tests pass, ruff clean, mypy clean, changelog entry, docs updated (if applicable) +- Description section +- Related issues section + +### 7. Pre-commit Config + +`.pre-commit-config.yaml`: + +```yaml +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.10 # pin to current version used in CI + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.15.0 # pin to current version + hooks: + - id: mypy + additional_dependencies: [pydantic] + args: [src] +``` + +Matches CI enforcement exactly. + +### 8. DTA Branding Hooks + +- README footer: `An open-source project by the [Digital Transformation Agency](https://www.dta.gov.au/).` +- MkDocs `copyright` field: same text +- MkDocs `theme.logo`: commented out placeholder, ready for PNG drop-in +- MkDocs `theme.favicon`: commented out placeholder +- No logo files committed — just configuration wired for future use + +## Out of Scope (for now) + +- Dynamic versioning from git tags (Approach 2 cherry-pick — deferred) +- Codecov integration +- CODEOWNERS file +- Dependabot config +- Full C-level docs (tutorials, integration guides, auto-generated API docs) +- DTA branding assets (logo, colour palette) + +## Dependencies + +- `mkdocs-material` added as a dev/docs dependency +- GitHub Pages enabled on the repository (Settings > Pages > Source: gh-pages branch) +- No other external service accounts needed + +## Success Criteria + +- Someone visiting the GitHub repo immediately understands what errorworks does and how to use it +- The PyPI page has functional sidebar links (Docs, Changelog, Issues, Homepage) +- The docs site is live and navigable at `johnm-dta.github.io/errorworks` +- `pre-commit install` works and catches lint/format/type issues locally +- Adding a new server type (e.g. email) requires adding one guide page and one nav entry — no restructuring From f29e4f8f9223a87db4f6c05f5ab4df6f883903c0 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 09:05:06 +1100 Subject: [PATCH 06/17] Fix spec review issues: mypy hook, versions, docs build details - Switch mypy pre-commit from mirrors-mypy to local hook using uv run (avoids missing dependency issue) - Remove pinned ruff/mypy versions (implementer pins to actual versions) - Rename reference/configuration.md to config-schema.md (avoid name clash) - Specify changelog copy step instead of ambiguous symlink/include - Add permissions note for docs workflow - Create separate docs optional-dependency group - Add sdist exclusion and MkDocs nav notes - Mark spec as Approved Co-Authored-By: Claude Opus 4.6 (1M context) --- ...26-03-16-professionalize-package-design.md | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/specs/2026-03-16-professionalize-package-design.md b/docs/superpowers/specs/2026-03-16-professionalize-package-design.md index b0cc7c0..d5e334a 100644 --- a/docs/superpowers/specs/2026-03-16-professionalize-package-design.md +++ b/docs/superpowers/specs/2026-03-16-professionalize-package-design.md @@ -1,7 +1,7 @@ # Professionalize Errorworks Package **Date:** 2026-03-16 -**Status:** Draft +**Status:** Approved **Author:** Claude (with John Morrissey) ## Context @@ -76,9 +76,9 @@ docs/ ├── reference/ │ ├── cli.md # chaosengine / chaosllm / chaosweb CLI reference │ ├── api.md # HTTP API endpoints -│ └── configuration.md # Full config model reference (all fields, defaults, types) +│ └── config-schema.md # Full config model reference (all fields, defaults, types) ├── architecture.md # Composition pattern, engine components, config snapshot -└── changelog.md # Include or symlink of CHANGELOG.md +└── changelog.md # Copy of CHANGELOG.md (pre-build copy step in docs workflow) ``` **Design decisions:** @@ -102,8 +102,10 @@ docs/ New `.github/workflows/docs.yml`: - Triggers on push to `main` (paths: `docs/**`, `mkdocs.yml`) +- Pre-build step: copy `CHANGELOG.md` to `docs/changelog.md` - Builds MkDocs site - Deploys to `gh-pages` branch using `peaceiris/actions-gh-pages` or equivalent +- Requires `permissions: contents: write` for gh-pages push - Requires `mkdocs-material` as build dependency ### 5. Community Files @@ -147,20 +149,22 @@ New `.github/workflows/docs.yml`: ```yaml repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.10 # pin to current version used in CI + rev: hooks: - id: ruff args: [--fix] - id: ruff-format - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.15.0 # pin to current version + - repo: local hooks: - id: mypy - additional_dependencies: [pydantic] - args: [src] + name: mypy + entry: uv run mypy src + language: system + types: [python] + pass_filenames: false ``` -Matches CI enforcement exactly. +Uses a local hook for mypy so it runs against the project's full venv (all dependencies available), matching CI exactly. Ruff version should be pinned to whatever version is currently in use (check `uv run ruff --version`). ### 8. DTA Branding Hooks @@ -181,10 +185,16 @@ Matches CI enforcement exactly. ## Dependencies -- `mkdocs-material` added as a dev/docs dependency +- `mkdocs-material` added as a new `docs` optional-dependency group in pyproject.toml: `[project.optional-dependencies] docs = ["mkdocs-material>=9,<10"]`. The existing `dev` group remains for test/lint tooling. `uv sync --all-extras` continues to install everything. - GitHub Pages enabled on the repository (Settings > Pages > Source: gh-pages branch) - No other external service accounts needed +## Build Notes + +- The `docs/superpowers/` and `docs/plans/` directories (internal specs) should be excluded from MkDocs nav via `mkdocs.yml` `not_in_nav` or by only listing explicit nav entries (no auto-discovery). +- `docs/` source files are included in the sdist by default. Add `docs/` to the sdist exclusion list in pyproject.toml since documentation source is not needed in the distributed package. +- The existing `CHANGELOG.md` is used as-is — it already follows Keep a Changelog format. + ## Success Criteria - Someone visiting the GitHub repo immediately understands what errorworks does and how to use it From 994d8271707402601b7f558c52c6dcf90b7f5d0a Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 18:20:06 +1100 Subject: [PATCH 07/17] Add implementation plan: professionalize errorworks package 8 tasks across 4 chunks: PyPI metadata, community files, GitHub templates, pre-commit config, README expansion, MkDocs-Material site, and docs workflow. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-03-16-professionalize-package.md | 730 ++++++++++++++++++ 1 file changed, 730 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-16-professionalize-package.md diff --git a/docs/superpowers/plans/2026-03-16-professionalize-package.md b/docs/superpowers/plans/2026-03-16-professionalize-package.md new file mode 100644 index 0000000..5996e49 --- /dev/null +++ b/docs/superpowers/plans/2026-03-16-professionalize-package.md @@ -0,0 +1,730 @@ +# Professionalize Errorworks Package — Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Transform errorworks from a well-engineered but poorly-presented 0.1.0 package into one that looks professionally maintained — README, docs site, PyPI metadata, community files, GitHub templates, pre-commit config. + +**Architecture:** No code changes. Eight independent deliverables that each produce a committable unit. The MkDocs site depends on having docs content written first, and the docs workflow depends on mkdocs.yml existing. + +**Tech Stack:** MkDocs-Material, GitHub Actions, GitHub Pages, shields.io badges, pre-commit + +**Spec:** `docs/superpowers/specs/2026-03-16-professionalize-package-design.md` + +--- + +## Chunk 1: Package Metadata, Community Files, and GitHub Templates + +These are small, independent files that establish the project's professional baseline. + +### Task 1: Update PyPI metadata in pyproject.toml + +**Files:** +- Modify: `pyproject.toml:66-67` (project.urls section) +- Modify: `pyproject.toml:47-58` (optional-dependencies — add docs group) +- Modify: `pyproject.toml:76-83` (sdist exclusions — add docs/) + +- [ ] **Step 1: Add project URLs** + +Replace the existing `[project.urls]` section: + +```toml +[project.urls] +Homepage = "https://github.com/johnm-dta/errorworks" +Documentation = "https://johnm-dta.github.io/errorworks" +Changelog = "https://github.com/johnm-dta/errorworks/blob/main/CHANGELOG.md" +"Bug Tracker" = "https://github.com/johnm-dta/errorworks/issues" +Repository = "https://github.com/johnm-dta/errorworks" +``` + +- [ ] **Step 2: Add docs dependency group** + +Add after the existing `dev` group in `[project.optional-dependencies]`: + +```toml +docs = [ + "mkdocs-material>=9,<10", +] +``` + +- [ ] **Step 3: Add docs/ to sdist exclusions** + +Add `"docs/"` to the `[tool.hatch.build.targets.sdist] exclude` list (after the existing `"docs/plans/"` entry). Remove the now-redundant `"docs/plans/"` and `"docs/arch-analysis-*"` entries since `"docs/"` covers them. + +- [ ] **Step 4: Run uv sync to verify** + +Run: `uv sync --all-extras` +Expected: Installs mkdocs-material and its dependencies without errors. + +- [ ] **Step 5: Commit** + +```bash +git add pyproject.toml +git commit -m "chore: add PyPI project URLs, docs dependency group, sdist exclusions" +``` + +--- + +### Task 2: Add community files — CONTRIBUTING.md, SECURITY.md, CODE_OF_CONDUCT.md + +**Files:** +- Create: `CONTRIBUTING.md` +- Create: `SECURITY.md` +- Create: `CODE_OF_CONDUCT.md` + +- [ ] **Step 1: Create CONTRIBUTING.md** + +```markdown +# Contributing to errorworks + +Thank you for your interest in contributing to errorworks. + +## Prerequisites + +- Python 3.12+ +- [uv](https://docs.astral.sh/uv/) package manager + +## Setup + +```bash +git clone https://github.com/johnm-dta/errorworks.git +cd errorworks +uv sync --all-extras +``` + +## Development workflow + +```bash +# Run tests +uv run pytest + +# Run tests with coverage +uv run pytest --cov + +# Lint and format +uv run ruff check src tests +uv run ruff format src tests + +# Type check +uv run mypy src +``` + +## Pull request expectations + +- All tests pass (`uv run pytest`) +- No lint issues (`uv run ruff check src tests`) +- No format issues (`uv run ruff format --check src tests`) +- No type errors (`uv run mypy src`) +- Add a changelog entry for user-facing changes + +## Project + +This project is maintained by the [Digital Transformation Agency](https://www.dta.gov.au/). +``` + +- [ ] **Step 2: Create SECURITY.md** + +```markdown +# Security Policy + +## Scope + +Errorworks intentionally generates error responses, malformed data, and simulated faults — these are features, not vulnerabilities. + +Security issues include: +- Arbitrary code execution outside the Jinja2 sandbox +- Unintended data exposure from the metrics store +- Dependency vulnerabilities with exploitable impact + +## Reporting a vulnerability + +Please report security vulnerabilities through [GitHub Security Advisories](https://github.com/johnm-dta/errorworks/security/advisories/new). + +Do not open a public issue for security vulnerabilities. +``` + +- [ ] **Step 3: Create CODE_OF_CONDUCT.md** + +Fetch the full markdown text from `https://www.contributor-covenant.org/version/2/1/code_of_conduct/code_of_conduct.md` and use it verbatim. Set the `[INSERT CONTACT METHOD]` placeholder to: `john.morrissey@dta.gov.au`. + +- [ ] **Step 4: Commit** + +```bash +git add CONTRIBUTING.md SECURITY.md CODE_OF_CONDUCT.md +git commit -m "docs: add CONTRIBUTING, SECURITY, and CODE_OF_CONDUCT" +``` + +--- + +### Task 3: Add GitHub issue and PR templates + +**Files:** +- Create: `.github/ISSUE_TEMPLATE/bug_report.md` +- Create: `.github/ISSUE_TEMPLATE/feature_request.md` +- Create: `.github/PULL_REQUEST_TEMPLATE.md` + +- [ ] **Step 1: Create bug report template** + +Create `.github/ISSUE_TEMPLATE/bug_report.md`: + +```markdown +--- +name: Bug report +about: Report a bug in errorworks +labels: bug +--- + +## Description + +A clear description of the bug. + +## Steps to reproduce + +1. +2. +3. + +## Expected behavior + +What you expected to happen. + +## Actual behavior + +What actually happened. Include error messages or tracebacks if applicable. + +## Environment + +- OS: +- Python version: +- errorworks version: +- Installation method (pip/uv): +``` + +- [ ] **Step 2: Create feature request template** + +Create `.github/ISSUE_TEMPLATE/feature_request.md`: + +```markdown +--- +name: Feature request +about: Suggest a new feature or enhancement +labels: enhancement +--- + +## Use case + +Describe the problem or need this feature would address. + +## Proposed solution + +How you think this could work. + +## Alternatives considered + +Any other approaches you've thought about. +``` + +- [ ] **Step 3: Create PR template** + +Create `.github/PULL_REQUEST_TEMPLATE.md`: + +```markdown +## Description + +What does this PR do? + +## Related issues + +Closes # + +## Checklist + +- [ ] Tests pass (`uv run pytest`) +- [ ] Lint clean (`uv run ruff check src tests`) +- [ ] Format clean (`uv run ruff format --check src tests`) +- [ ] Types clean (`uv run mypy src`) +- [ ] Changelog entry added (if user-facing change) +- [ ] Docs updated (if applicable) +``` + +- [ ] **Step 4: Commit** + +```bash +git add .github/ISSUE_TEMPLATE/ .github/PULL_REQUEST_TEMPLATE.md +git commit -m "docs: add GitHub issue and PR templates" +``` + +--- + +### Task 4: Add pre-commit configuration + +**Files:** +- Create: `.pre-commit-config.yaml` + +- [ ] **Step 1: Check current ruff version** + +Run: `uv run ruff --version` +Note the version number — you'll pin to this in the config. + +- [ ] **Step 2: Create .pre-commit-config.yaml** + +```yaml +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + - repo: local + hooks: + - id: mypy + name: mypy + entry: uv run mypy src + language: system + types: [python] + pass_filenames: false +``` + +- [ ] **Step 3: Verify pre-commit works** + +Run: `uv run pre-commit run --all-files` +Expected: All hooks pass (ruff check, ruff format, mypy). + +- [ ] **Step 4: Commit** + +```bash +git add .pre-commit-config.yaml +git commit -m "chore: add pre-commit config for ruff and mypy" +``` + +--- + +## Chunk 2: README + +The single highest-impact deliverable — what everyone sees first on GitHub and PyPI. + +### Task 5: Write the full README + +**Files:** +- Modify: `README.md` (replace the existing 3-line stub) + +**Important context for the implementer:** +- errorworks is a general-purpose chaos testing framework, maintained by the Digital Transformation Agency (DTA) +- Two server types exist today: ChaosLLM (OpenAI-compatible) and ChaosWeb (scraping resilience). More will come (e.g. email). +- CLI entry points: `chaosllm serve`, `chaosweb serve`, `chaosengine llm serve` / `chaosengine web serve` +- Presets: silent, gentle, realistic, chaos/stress variants +- The completions endpoint is `POST /v1/chat/completions` with standard OpenAI request format +- The web endpoint is `GET /{any-path}` returning HTML +- Admin endpoints (`/admin/stats`, `/admin/config`, `/admin/export`, `/admin/reset`) require `Authorization: Bearer {admin_token}` +- Pytest fixtures use marker-based config: `@pytest.mark.chaosllm(preset="realistic")` +- Python 3.12+, install via pip or uv + +- [ ] **Step 1: Write README.md** + +Replace the entire file. The README should follow this structure (refer to the spec for the full outline): + +**Header:** +```markdown +# errorworks + +Composable chaos-testing services for LLM and web scraping pipelines. + +[![CI](https://github.com/johnm-dta/errorworks/actions/workflows/ci.yml/badge.svg)](https://github.com/johnm-dta/errorworks/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/errorworks)](https://pypi.org/project/errorworks/) +[![Python](https://img.shields.io/pypi/pyversions/errorworks)](https://pypi.org/project/errorworks/) +[![License](https://img.shields.io/pypi/l/errorworks)](https://github.com/johnm-dta/errorworks/blob/main/LICENSE) +``` + +**What is errorworks?** (2-3 paragraphs): +- Explain the problem: testing how your code handles API failures, malformed responses, rate limits, and network issues is hard. You need a server that reproducibly generates these faults. +- Explain the solution: errorworks provides fake servers that inject configurable faults into HTTP responses. Use them to verify your LLM client retries on 429s, your scraper handles malformed HTML, your pipeline degrades gracefully under load. +- Mention: composable, configurable via CLI/YAML/presets, in-process pytest fixtures for CI, SQLite-backed metrics. + +**Features** — grouped bullet list: +- Error injection categories (HTTP errors, connection failures, malformed responses) +- Latency simulation +- Response generation modes (random, template, echo, preset) +- Built-in presets (silent → stress) +- Metrics & observability (SQLite, timeseries, admin API) +- Testing support (pytest fixtures, markers, in-process — no sockets) + +**Quick Start:** +```bash +pip install errorworks + +# Start a fake OpenAI server with realistic fault injection +chaosllm serve --preset=realistic + +# In another terminal, make a request +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +**Usage** — three subsections with brief examples: +1. CLI server (`chaosllm serve`, `chaosweb serve`, preset flags) +2. Pytest fixture (marker example, `post_completion()` / `fetch_page()` helpers) +3. Configuration (YAML file example, precedence: CLI > file > preset > defaults) + +**Documentation** — link to `https://johnm-dta.github.io/errorworks` + +**Architecture** — 3-4 sentences about composition-based design, engine components, the config snapshot pattern. Link to docs for full detail. + +**Footer:** +```markdown +--- + +An open-source project by the [Digital Transformation Agency](https://www.dta.gov.au/). + +Licensed under [MIT](LICENSE). +``` + +Target: ~120-160 lines. Don't pad it — every line should earn its place. + +- [ ] **Step 2: Verify rendering** + +Run: `uv run python -m markdown README.md > /dev/null 2>&1 || echo "check markdown syntax"` +Or just visually inspect the structure makes sense. + +- [ ] **Step 3: Commit** + +```bash +git add README.md +git commit -m "docs: expand README with badges, quickstart, features, and architecture" +``` + +--- + +## Chunk 3: MkDocs Site and Docs Workflow + +### Task 6: Create mkdocs.yml and docs site structure + +**Files:** +- Create: `mkdocs.yml` +- Create: `docs/index.md` +- Create: `docs/getting-started/installation.md` +- Create: `docs/getting-started/quickstart.md` +- Create: `docs/guide/chaosllm.md` +- Create: `docs/guide/chaosweb.md` +- Create: `docs/guide/presets.md` +- Create: `docs/guide/configuration.md` +- Create: `docs/guide/metrics.md` +- Create: `docs/guide/testing-fixtures.md` +- Create: `docs/reference/cli.md` +- Create: `docs/reference/api.md` +- Create: `docs/reference/config-schema.md` +- Create: `docs/architecture.md` + +**Important context for the implementer:** +- The `docs/` directory already contains `superpowers/`, `plans/`, `arch-analysis-*`, and `file_breakdown.md` — these are internal files. They must NOT appear in the MkDocs nav. Use explicit nav entries in mkdocs.yml (no auto-discovery). +- `docs/changelog.md` is NOT committed — it's copied from `CHANGELOG.md` at build time by the docs workflow (Task 7). Add it to `.gitignore`. +- All content should be written from the perspective of a developer who has never seen errorworks before. Don't assume they've read the README. + +- [ ] **Step 1: Create mkdocs.yml** + +```yaml +site_name: errorworks +site_description: Composable chaos-testing services for LLM and web scraping pipelines +site_url: https://johnm-dta.github.io/errorworks +repo_url: https://github.com/johnm-dta/errorworks +repo_name: johnm-dta/errorworks + +copyright: An open-source project by the Digital Transformation Agency. + +theme: + name: material + palette: + - scheme: default + primary: deep purple + accent: amber + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: deep purple + accent: amber + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.sections + - navigation.expand + - navigation.top + - content.code.copy + - search.suggest + - search.highlight + # logo: assets/logo.png # Uncomment when DTA branding is available + # favicon: assets/favicon.png # Uncomment when DTA branding is available + +nav: + - Home: index.md + - Getting Started: + - Installation: getting-started/installation.md + - Quick Start: getting-started/quickstart.md + - Guide: + - ChaosLLM: guide/chaosllm.md + - ChaosWeb: guide/chaosweb.md + - Presets: guide/presets.md + - Configuration: guide/configuration.md + - Metrics: guide/metrics.md + - Testing Fixtures: guide/testing-fixtures.md + - Reference: + - CLI: reference/cli.md + - HTTP API: reference/api.md + - Configuration Schema: reference/config-schema.md + - Architecture: architecture.md + - Changelog: changelog.md + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.tabbed: + alternate_style: true + - toc: + permalink: true +``` + +- [ ] **Step 2: Add docs/changelog.md to .gitignore** + +Append to `.gitignore`: +``` +docs/changelog.md +``` + +- [ ] **Step 3: Create docs/index.md** + +Landing page. Should: +- Restate the value proposition (don't just link to README) +- Have a "Get started in 60 seconds" code block (install + start server + make request) +- Link to Getting Started, Guide, and Reference sections +- Mention both ChaosLLM and ChaosWeb with one-line descriptions + +- [ ] **Step 4: Create docs/getting-started/installation.md** + +Cover: +- Prerequisites (Python 3.12+) +- Install from PyPI: `pip install errorworks` +- Install with uv: `uv add errorworks` +- Verify: `chaosllm --help` +- Development install: `git clone ... && uv sync --all-extras` + +- [ ] **Step 5: Create docs/getting-started/quickstart.md** + +Walk through a complete scenario: +1. Start ChaosLLM with `realistic` preset +2. Make a curl request to `/v1/chat/completions` +3. Observe: sometimes you get a 200, sometimes a 429 or 503 +4. Check metrics via `/admin/stats` (with admin token) +5. Try ChaosWeb: `chaosweb serve --preset=realistic`, fetch a page +6. Use the pytest fixture in a test file (complete working example) + +- [ ] **Step 6: Create docs/guide/chaosllm.md** + +Cover: +- What ChaosLLM is (fake OpenAI-compatible server) +- Supported endpoints (`/v1/chat/completions`, `/health`, admin endpoints) +- Error injection categories: HTTP errors (429, 529, 503, 502, 504, 500), connection failures (timeout, reset, stall), malformed responses (invalid JSON, truncated, missing fields, wrong content-type) +- Response modes (random, template, echo, preset) with examples +- Streaming vs non-streaming +- Burst patterns +- Available presets with what each simulates + +- [ ] **Step 7: Create docs/guide/chaosweb.md** + +Same structure as chaosllm.md, covering: +- What ChaosWeb is (fake web server for scraping resilience) +- Supported endpoints (`/{any-path}`, `/health`, `/redirect`, admin endpoints) +- Error categories: all LLM categories plus SSRF redirects, content malformations (encoding mismatch, truncated HTML, charset confusion) +- Content modes +- Anti-scraping simulation features +- Available presets + +- [ ] **Step 8: Create docs/guide/presets.md** + +Cover: +- What presets are (pre-built configuration profiles) +- Table of all presets for each server type with key settings +- How to use: `--preset=realistic` +- How to create custom presets (YAML file structure) +- Precedence: CLI flags > config file > preset > defaults + +- [ ] **Step 9: Create docs/guide/configuration.md** + +Cover: +- Configuration methods: CLI flags, YAML file (`--config`), presets +- Precedence rules with example +- YAML file structure (complete example) +- Runtime config updates via `POST /admin/config` with deep merge +- The immutable config pattern (frozen Pydantic models, atomic swap under lock) + +- [ ] **Step 10: Create docs/guide/metrics.md** + +Cover: +- What's recorded (per-request: endpoint, outcome, status, latency, model/path) +- SQLite storage (WAL mode, thread-safe, thread-local connections) +- Timeseries aggregation (UPSERT bucketing) +- Querying via admin API (`/admin/stats`, `/admin/export`) +- Database persistence (`--database` flag) +- Reset (`/admin/reset`) + +- [ ] **Step 11: Create docs/guide/testing-fixtures.md** + +Cover: +- In-process testing (Starlette TestClient, no real sockets) +- Marker-based configuration: `@pytest.mark.chaosllm(preset="realistic", rate_limit_pct=25.0)` +- Available marker kwargs (map to CLI flags) +- Fixture helpers: `post_completion()`, `fetch_page()`, `update_config()`, `get_stats()`, `wait_for_requests()` +- Complete working test example for both ChaosLLM and ChaosWeb +- How to register the fixtures (plugin or conftest import) + +- [ ] **Step 12: Create docs/reference/cli.md** + +Full CLI reference for: +- `chaosengine` (with `llm` and `web` subcommands) +- `chaosllm serve` — all flags with defaults and descriptions +- `chaosweb serve` — all flags with defaults and descriptions +- `chaosllm-mcp` — brief description + +Format as tables: Flag | Default | Description + +- [ ] **Step 13: Create docs/reference/api.md** + +Full HTTP API reference for both servers: +- `POST /v1/chat/completions` (ChaosLLM) — request/response format, streaming +- `GET /{path}` (ChaosWeb) — response format +- `GET /health` — both servers +- `GET /admin/stats` — response schema +- `GET /admin/config` — response schema +- `POST /admin/config` — request/response schema, deep merge behavior +- `GET /admin/export` — response schema +- `POST /admin/reset` — response schema +- Authentication: `Authorization: Bearer {admin_token}` for admin endpoints + +- [ ] **Step 14: Create docs/reference/config-schema.md** + +Document all Pydantic config model fields: +- `ServerConfig` (host, port, workers, admin_token, database) +- `ErrorInjectionConfig` (all percentage fields, selection_mode) +- `BurstConfig` (enabled, interval_sec, duration_sec) +- `LatencyConfig` (base_ms, jitter_ms) +- `ResponseConfig` / content mode configs +- ChaosLLM-specific and ChaosWeb-specific fields + +Format as tables: Field | Type | Default | Description + +Source these from the actual Pydantic models in: +- `src/errorworks/engine/config.py` +- `src/errorworks/llm/config.py` +- `src/errorworks/web/config.py` + +- [ ] **Step 15: Create docs/architecture.md** + +Cover: +- Composition over inheritance (engine utilities composed, not inherited) +- Package structure diagram (engine, llm, web, llm_mcp, testing) +- Key engine components: InjectionEngine, MetricsStore, LatencySimulator, ConfigLoader +- Config snapshot pattern (why and how) +- Immutable config update flow +- Thread safety model +- How to add a new server type (extensibility path) + +- [ ] **Step 16: Add site/ to .gitignore** + +Append to `.gitignore`: +``` +site/ +``` + +- [ ] **Step 17: Verify docs build locally** + +Run: `cp CHANGELOG.md docs/changelog.md && uv run mkdocs build --strict && rm docs/changelog.md` +Expected: Builds without warnings or errors. Output in `site/` directory. The `docs/changelog.md` copy is temporary (it's gitignored — the docs workflow handles this at build time). + +- [ ] **Step 18: Commit** + +```bash +git add mkdocs.yml docs/ .gitignore +git commit -m "docs: add MkDocs-Material site with getting started, guides, reference, and architecture" +``` + +--- + +### Task 7: Add GitHub Actions docs workflow + +**Files:** +- Create: `.github/workflows/docs.yml` + +- [ ] **Step 1: Create the docs workflow** + +```yaml +name: Docs + +on: + push: + branches: [main] + paths: + - "docs/**" + - "mkdocs.yml" + - "CHANGELOG.md" + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uv sync --extra docs + - run: cp CHANGELOG.md docs/changelog.md + - run: uv run mkdocs gh-deploy --force +``` + +Note: `mkdocs gh-deploy` handles pushing to the `gh-pages` branch directly — no need for a separate GitHub Pages action. + +- [ ] **Step 2: Commit** + +```bash +git add .github/workflows/docs.yml +git commit -m "ci: add docs workflow for GitHub Pages deployment" +``` + +--- + +## Chunk 4: Final Verification + +### Task 8: End-to-end verification + +- [ ] **Step 1: Run full lint/typecheck/test suite** + +Run: `uv run ruff check src tests && uv run ruff format --check src tests && uv run mypy src && uv run pytest` +Expected: All pass. No existing functionality broken. + +- [ ] **Step 2: Verify pre-commit** + +Run: `uv run pre-commit run --all-files` +Expected: All hooks pass. + +- [ ] **Step 3: Verify docs build** + +Run: `cp CHANGELOG.md docs/changelog.md && uv run mkdocs build --strict` +Expected: Builds cleanly. + +- [ ] **Step 4: Verify package build** + +Run: `uv build` +Expected: Builds wheel and sdist. The `docs/` directory should NOT be in the sdist. + +- [ ] **Step 5: Spot-check sdist contents** + +Run: `tar tzf dist/errorworks-*.tar.gz | grep docs/ | head -5` +Expected: No docs/ files listed (excluded by sdist config). + +- [ ] **Step 6: Clean up build artifacts** + +Run: `rm -rf site/ dist/ docs/changelog.md` From fad564f6fbfa2b76d47176bd29fb5d10b10b4f53 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 18:42:04 +1100 Subject: [PATCH 08/17] chore: add .worktrees/ to .gitignore Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1840eb0..cbc1fec 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,9 @@ Thumbs.db .env.* .hypothesis/ +# Worktrees +.worktrees/ + # Filigree issue tracker .filigree/ .claude/settings.json From cee76ce57b9fa4a5963b10f8d93246bc562663d9 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 19:00:51 +1100 Subject: [PATCH 09/17] chore: add PyPI project URLs, docs dependency group, sdist exclusions Co-Authored-By: Claude Opus 4.6 (1M context) --- pyproject.toml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dfbf128..3fd75cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,9 @@ dev = [ "pre-commit>=4.0,<5", "types-PyYAML>=6.0,<7", ] +docs = [ + "mkdocs-material>=9,<10", +] [project.scripts] chaosengine = "errorworks.engine.cli:main" @@ -64,6 +67,10 @@ chaosllm-mcp = "errorworks.llm.cli:mcp_main_entry" chaosweb = "errorworks.web.cli:main" [project.urls] +Homepage = "https://github.com/johnm-dta/errorworks" +Documentation = "https://johnm-dta.github.io/errorworks" +Changelog = "https://github.com/johnm-dta/errorworks/blob/main/CHANGELOG.md" +"Bug Tracker" = "https://github.com/johnm-dta/errorworks/issues" Repository = "https://github.com/johnm-dta/errorworks" [build-system] @@ -77,8 +84,7 @@ packages = ["src/errorworks"] exclude = [ ".claude/", "CLAUDE.md", - "docs/plans/", - "docs/arch-analysis-*", + "docs/", "uv.lock", ] From e0c75af21e54ddd756f2c8161e0722dd5deba30c Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 19:11:24 +1100 Subject: [PATCH 10/17] docs: add CONTRIBUTING, SECURITY, and CODE_OF_CONDUCT Co-Authored-By: Claude Opus 4.6 (1M context) --- CODE_OF_CONDUCT.md | 83 ++++++++++++++++++++++++++++++++++++++++++++++ CONTRIBUTING.md | 45 +++++++++++++++++++++++++ SECURITY.md | 16 +++++++++ 3 files changed, 144 insertions(+) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3e46bae --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,83 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at john.morrissey@dta.gov.au. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..538cf28 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,45 @@ +# Contributing to errorworks + +Thank you for your interest in contributing to errorworks. + +## Prerequisites + +- Python 3.12+ +- [uv](https://docs.astral.sh/uv/) package manager + +## Setup + +```bash +git clone https://github.com/johnm-dta/errorworks.git +cd errorworks +uv sync --all-extras +``` + +## Development workflow + +```bash +# Run tests +uv run pytest + +# Run tests with coverage +uv run pytest --cov + +# Lint and format +uv run ruff check src tests +uv run ruff format src tests + +# Type check +uv run mypy src +``` + +## Pull request expectations + +- All tests pass (`uv run pytest`) +- No lint issues (`uv run ruff check src tests`) +- No format issues (`uv run ruff format --check src tests`) +- No type errors (`uv run mypy src`) +- Add a changelog entry for user-facing changes + +## Project + +This project is maintained by the [Digital Transformation Agency](https://www.dta.gov.au/). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..6f798a2 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security Policy + +## Scope + +Errorworks intentionally generates error responses, malformed data, and simulated faults — these are features, not vulnerabilities. + +Security issues include: +- Arbitrary code execution outside the Jinja2 sandbox +- Unintended data exposure from the metrics store +- Dependency vulnerabilities with exploitable impact + +## Reporting a vulnerability + +Please report security vulnerabilities through [GitHub Security Advisories](https://github.com/johnm-dta/errorworks/security/advisories/new). + +Do not open a public issue for security vulnerabilities. From f4215829ff4d0ec74cabe3bb2c04458cb3a989fa Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 19:12:27 +1100 Subject: [PATCH 11/17] docs: add GitHub issue and PR templates Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/ISSUE_TEMPLATE/bug_report.md | 30 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 17 +++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 16 ++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..33c1461 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Report a bug in errorworks +labels: bug +--- + +## Description + +A clear description of the bug. + +## Steps to reproduce + +1. +2. +3. + +## Expected behavior + +What you expected to happen. + +## Actual behavior + +What actually happened. Include error messages or tracebacks if applicable. + +## Environment + +- OS: +- Python version: +- errorworks version: +- Installation method (pip/uv): diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..7d5e52c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest a new feature or enhancement +labels: enhancement +--- + +## Use case + +Describe the problem or need this feature would address. + +## Proposed solution + +How you think this could work. + +## Alternatives considered + +Any other approaches you've thought about. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..18c89e6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Description + +What does this PR do? + +## Related issues + +Closes # + +## Checklist + +- [ ] Tests pass (`uv run pytest`) +- [ ] Lint clean (`uv run ruff check src tests`) +- [ ] Format clean (`uv run ruff format --check src tests`) +- [ ] Types clean (`uv run mypy src`) +- [ ] Changelog entry added (if user-facing change) +- [ ] Docs updated (if applicable) From ea75e7f2260b9c6605d47fbb1276e391198260e6 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 19:15:27 +1100 Subject: [PATCH 12/17] chore: add pre-commit config for ruff and mypy Also fix existing lint issues caught by the new hooks: sort __all__, use raw strings for regex match patterns, remove unused variable. Co-Authored-By: Claude Opus 4.6 (1M context) --- .pre-commit-config.yaml | 15 +++++++++++++++ src/errorworks/__init__.py | 5 +---- tests/unit/engine/test_validators.py | 4 ++-- tests/unit/llm/test_cli.py | 2 -- tests/unit/llm/test_latency_simulator.py | 2 +- tests/unit/llm/test_response_generator.py | 2 +- 6 files changed, 20 insertions(+), 10 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a763174 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.5 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + - repo: local + hooks: + - id: mypy + name: mypy + entry: uv run mypy src + language: system + types: [python] + pass_filenames: false diff --git a/src/errorworks/__init__.py b/src/errorworks/__init__.py index a89fce3..fdb896e 100644 --- a/src/errorworks/__init__.py +++ b/src/errorworks/__init__.py @@ -4,11 +4,8 @@ __all__ = [ "__version__", - # Engine (shared core) "engine", - # Chaos plugins "llm", - "web", - # Test support "testing", + "web", ] diff --git a/tests/unit/engine/test_validators.py b/tests/unit/engine/test_validators.py index 3c6e656..28b4492 100644 --- a/tests/unit/engine/test_validators.py +++ b/tests/unit/engine/test_validators.py @@ -21,11 +21,11 @@ class TestParseRangeFloatRejection: """parse_range must reject floats with fractional parts.""" def test_rejects_fractional_floats(self) -> None: - with pytest.raises(ValueError, match="must be integers.*1.5"): + with pytest.raises(ValueError, match=r"must be integers.*1.5"): parse_range([1.5, 3.5]) def test_rejects_single_fractional_float(self) -> None: - with pytest.raises(ValueError, match="must be integers.*2.7"): + with pytest.raises(ValueError, match=r"must be integers.*2.7"): parse_range([1, 2.7]) def test_accepts_exact_integer_floats(self) -> None: diff --git a/tests/unit/llm/test_cli.py b/tests/unit/llm/test_cli.py index 59a355c..58eb63e 100644 --- a/tests/unit/llm/test_cli.py +++ b/tests/unit/llm/test_cli.py @@ -234,8 +234,6 @@ def test_preset_values_not_overridden_by_cli_defaults(mock_run): result = runner.invoke(app, ["serve", "--preset=gentle"]) assert result.exit_code == 0, result.output - uvicorn_app = mock_run.call_args.args[0] - server = uvicorn_app.state.server # gentle preset sets workers=4 — CLI should NOT override to 1 assert mock_run.call_args.kwargs["workers"] == 4 diff --git a/tests/unit/llm/test_latency_simulator.py b/tests/unit/llm/test_latency_simulator.py index 53bdcac..20b8f46 100644 --- a/tests/unit/llm/test_latency_simulator.py +++ b/tests/unit/llm/test_latency_simulator.py @@ -290,7 +290,7 @@ def test_inverted_min_max_slow_response_raises(self) -> None: config = LatencyConfig() simulator = LatencySimulator(config) - with pytest.raises(ValueError, match="min_sec.*must be <= max_sec"): + with pytest.raises(ValueError, match=r"min_sec.*must be <= max_sec"): simulator.simulate_slow_response(30, 10) def test_equal_min_max_slow_response(self) -> None: diff --git a/tests/unit/llm/test_response_generator.py b/tests/unit/llm/test_response_generator.py index 7d4b062..bfebc3d 100644 --- a/tests/unit/llm/test_response_generator.py +++ b/tests/unit/llm/test_response_generator.py @@ -488,7 +488,7 @@ def test_random_words_range(self) -> None: def test_random_words_unknown_vocabulary_raises(self) -> None: """Config with unknown vocabulary is rejected by Pydantic validation.""" - with pytest.raises(Exception, match="klingon|literal_error"): + with pytest.raises(Exception, match=r"klingon|literal_error"): ResponseConfig( mode="template", template=TemplateResponseConfig(body="{{ random_words(3) }}"), From 2297c6634f4236d42a74978d5306ccf195949dbf Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 19:17:12 +1100 Subject: [PATCH 13/17] docs: expand README with badges, quickstart, features, and architecture Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 138 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 725efd6..0654f91 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,140 @@ # errorworks -Composable chaos-testing services for various pipelines. +Composable chaos-testing services for LLM and web scraping pipelines. + +[![CI](https://github.com/johnm-dta/errorworks/actions/workflows/ci.yml/badge.svg)](https://github.com/johnm-dta/errorworks/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/errorworks)](https://pypi.org/project/errorworks/) +[![Python](https://img.shields.io/pypi/pyversions/errorworks)](https://pypi.org/project/errorworks/) +[![License](https://img.shields.io/pypi/l/errorworks)](https://github.com/johnm-dta/errorworks/blob/main/LICENSE) + +## What is errorworks? + +Testing how your code handles API failures, malformed responses, rate limits, and +network issues is hard. Unit-testing a retry loop against a mock is easy; knowing +whether your pipeline actually degrades gracefully under realistic fault patterns +requires a server that behaves badly on purpose. + +errorworks provides fake servers that inject configurable faults into your test +traffic. Point your LLM client at a ChaosLLM server to verify it retries on 429s +and surfaces clean errors on malformed JSON. Point your scraper at a ChaosWeb +server to confirm it handles truncated HTML, encoding mismatches, and SSRF +redirects. Fault rates, error distributions, and latency profiles are all +configurable via CLI flags, YAML files, or built-in presets. + +Everything runs in-process during CI via pytest fixtures (no sockets, no +containers), records metrics to a thread-safe SQLite store, and supports live +reconfiguration through admin endpoints. + +## Features + +**Error injection** +- HTTP errors: 429, 529, 503, 502, 504, 500 +- Connection failures: timeout, reset, stall +- Malformed responses: invalid JSON, truncated bodies, missing fields, wrong content-type +- Web-specific: SSRF redirects (private IPs, cloud metadata), encoding mismatches, truncated HTML, charset confusion + +**Latency simulation** +- Configurable base delay with jitter +- Per-request latency injection, independent of error selection + +**Response generation** +- Four content modes: `random` (vocabulary-based), `template` (Jinja2 sandbox), `echo` (reflect input), `preset` (JSONL bank) +- ChaosLLM returns OpenAI-compatible chat completion responses +- ChaosWeb returns HTML pages + +**Presets** +- LLM: `silent`, `gentle`, `realistic`, `chaos`, `stress_aimd` +- Web: `silent`, `gentle`, `realistic`, `chaos`, `stress_scraping`, `stress_extreme` + +**Metrics and admin** +- SQLite-backed metrics with timeseries aggregation +- Admin endpoints for stats, config, export, and reset (bearer-token auth) + +**Testing support** +- In-process pytest fixtures with marker-based configuration +- No sockets or containers required in CI + +## Quick start + +```bash +pip install errorworks + +# Start a fake OpenAI server with realistic fault injection +chaosllm serve --preset=realistic + +# In another terminal +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +## Usage + +### CLI servers + +```bash +# LLM server +chaosllm serve --preset=realistic --port=8000 + +# Web server +chaosweb serve --preset=chaos --port=9000 + +# Unified CLI +chaosengine llm serve --preset=gentle +chaosengine web serve --preset=stress_scraping +``` + +### Pytest fixtures + +```python +import pytest + +@pytest.mark.chaosllm(preset="realistic", rate_limit_pct=25.0) +def test_retry_on_rate_limit(chaosllm): + response = chaosllm.post_completion( + model="gpt-4", + messages=[{"role": "user", "content": "test"}], + ) + assert response.status_code in (200, 429) +``` + +### Configuration + +Presets provide sensible defaults. Override individual settings with a YAML config +file or CLI flags. Precedence: CLI flags > config file > preset > defaults. + +```yaml +# config.yaml +error_rate_pct: 30.0 +rate_limit_pct: 10.0 +latency: + base_ms: 50 + jitter_ms: 20 +response: + mode: random + vocabulary: english +``` + +```bash +chaosllm serve --preset=gentle --config=config.yaml --port=8080 +``` + +## Documentation + +Full documentation is available at [johnm-dta.github.io/errorworks](https://johnm-dta.github.io/errorworks). + +## Architecture + +errorworks uses a composition-based design: each server type (ChaosLLM, ChaosWeb) +composes shared engine components rather than inheriting from base classes. The +core engine provides an `InjectionEngine` for fault selection, a `MetricsStore` +for recording, a `LatencySimulator` for delays, and a `ConfigLoader` for +YAML/preset merging. All configuration models are frozen Pydantic instances; +runtime updates create new model instances and atomically swap references under +lock, ensuring thread-safe request handling without mid-request inconsistency. + +--- + +An open-source project by the [Digital Transformation Agency](https://www.dta.gov.au/). + +Licensed under [MIT](LICENSE). From 7fb52b331ef20a38c4cff827ea41d4ce71384cf0 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 19:47:24 +1100 Subject: [PATCH 14/17] docs: add MkDocs-Material site with getting started, guides, reference, and architecture 13 documentation pages covering installation, quickstart, ChaosLLM/ChaosWeb guides, presets, configuration, metrics, testing fixtures, CLI reference, HTTP API reference, config schema reference, and architecture overview. Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 4 + docs/architecture.md | 196 ++++++++++++ docs/getting-started/installation.md | 51 ++++ docs/getting-started/quickstart.md | 112 +++++++ docs/guide/chaosllm.md | 249 +++++++++++++++ docs/guide/chaosweb.md | 243 +++++++++++++++ docs/guide/configuration.md | 313 +++++++++++++++++++ docs/guide/metrics.md | 256 ++++++++++++++++ docs/guide/presets.md | 126 ++++++++ docs/guide/testing-fixtures.md | 321 +++++++++++++++++++ docs/index.md | 73 +++++ docs/reference/api.md | 440 +++++++++++++++++++++++++++ docs/reference/cli.md | 243 +++++++++++++++ docs/reference/config-schema.md | 322 ++++++++++++++++++++ mkdocs.yml | 63 ++++ 15 files changed, 3012 insertions(+) create mode 100644 docs/architecture.md create mode 100644 docs/getting-started/installation.md create mode 100644 docs/getting-started/quickstart.md create mode 100644 docs/guide/chaosllm.md create mode 100644 docs/guide/chaosweb.md create mode 100644 docs/guide/configuration.md create mode 100644 docs/guide/metrics.md create mode 100644 docs/guide/presets.md create mode 100644 docs/guide/testing-fixtures.md create mode 100644 docs/index.md create mode 100644 docs/reference/api.md create mode 100644 docs/reference/cli.md create mode 100644 docs/reference/config-schema.md create mode 100644 mkdocs.yml diff --git a/.gitignore b/.gitignore index cbc1fec..5da735c 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,7 @@ Thumbs.db # Filigree issue tracker .filigree/ .claude/settings.json + +# MkDocs +docs/changelog.md +site/ diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..a3b1ce4 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,196 @@ +# Architecture Overview + +Errorworks is a composable chaos-testing service framework. Each server type (LLM, Web) is built from shared engine components rather than inheriting from a base class. This document explains the design rationale, key components, and extension points. + +## Composition Over Inheritance + +The central design principle is that **HTTP concerns stay out of domain logic**. Each chaos plugin (ChaosLLM, ChaosWeb) creates instances of shared engine utilities and delegates specific responsibilities to them: + +- **InjectionEngine** handles burst state and error selection algorithms +- **MetricsStore** handles SQLite persistence and timeseries aggregation +- **LatencySimulator** handles delay calculation +- **ConfigLoader** handles YAML loading and config precedence + +The server classes (`ChaosLLMServer`, `ChaosWebServer`) own the HTTP routing, request parsing, and response formatting. They compose engine components but never extend them. This means a new server type (e.g., email, gRPC) can reuse the same engine components without inheriting HTTP-specific behavior it does not need. + +## Package Structure + +``` +src/errorworks/ +├── engine/ # Shared core utilities +│ ├── types.py # ServerConfig, MetricsConfig, LatencyConfig, +│ │ # BurstConfig, ErrorSpec, SelectionMode, +│ │ # MetricsSchema, ColumnDef +│ ├── injection_engine.py # Burst state machine + selection algorithms +│ ├── metrics_store.py # Thread-safe SQLite with schema-driven DDL +│ ├── latency.py # Latency simulation (base +/- jitter) +│ ├── config_loader.py # YAML preset loading + deep merge +│ ├── admin.py # Shared admin endpoint handlers +│ ├── validators.py # Shared Pydantic validators (range parsing) +│ └── cli.py # Unified chaosengine CLI +│ +├── llm/ # ChaosLLM: Fake OpenAI-compatible server +│ ├── config.py # ChaosLLMConfig, ErrorInjectionConfig, ResponseConfig +│ ├── server.py # ChaosLLMServer (Starlette ASGI app) +│ ├── error_injector.py # LLM-specific error decision logic +│ ├── response_generator.py# OpenAI-format response generation +│ ├── metrics.py # LLM-specific MetricsRecorder wrapper +│ ├── cli.py # chaosllm CLI +│ └── presets/ # YAML preset files +│ +├── web/ # ChaosWeb: Fake web server for scraping tests +│ ├── config.py # ChaosWebConfig, WebErrorInjectionConfig, WebContentConfig +│ ├── server.py # ChaosWebServer (Starlette ASGI app) +│ ├── error_injector.py # Web-specific error decision logic +│ ├── content_generator.py # HTML content generation + corruption functions +│ ├── metrics.py # Web-specific MetricsRecorder wrapper +│ ├── cli.py # chaosweb CLI +│ └── presets/ # YAML preset files +│ +├── llm_mcp/ # MCP server for ChaosLLM metrics analysis +│ └── server.py # Claude-optimized metrics tools via MCP protocol +│ +└── testing/ # Pytest fixture support + └── ... # In-process test fixtures using Starlette TestClient +``` + +## Key Engine Components + +### InjectionEngine + +**File:** `engine/injection_engine.py` + +The InjectionEngine is the decision-making core for error injection. It handles two concerns: + +1. **Burst state machine** -- Periodic burst windows where error rates are elevated. Bursts occur every `interval_sec` seconds and last for `duration_sec` seconds. The state is computed from elapsed time using modular arithmetic (`elapsed % interval < duration`), making it stateless beyond the start timestamp. + +2. **Error selection** -- Two algorithms: + - **Priority mode:** Specs are evaluated in order. The first one that triggers (based on a random roll against its weight) wins. This gives deterministic precedence to high-priority errors. + - **Weighted mode:** A single error is selected proportionally from all active specs. Success probability is implicitly `max(0, 100 - total_weight)`. + +The engine is deliberately domain-agnostic. It works with `ErrorSpec(tag, weight)` objects where `tag` is an opaque string. The calling plugin builds the spec list (with domain-specific tags like `"rate_limit"` or `"ssrf_redirect"`) and interprets the selected tag to produce a response. + +**Thread safety:** The burst start time is protected by a lock. The RNG is not thread-safe, but this is handled by the config snapshot pattern (each request snapshots the engine reference, so concurrent requests use different engine instances after a config update). + +**Testability:** Both `time_func` and `rng` are injectable. Tests pass `time.monotonic` replacements and seeded `random.Random` instances for deterministic behavior. + +### MetricsStore + +**File:** `engine/metrics_store.py` + +Thread-safe SQLite storage with several notable design choices: + +- **Thread-local connections:** Each thread gets its own `sqlite3.Connection` via `threading.local()`. This avoids SQLite's thread-safety limitations while allowing concurrent access from uvicorn workers. + +- **WAL mode for file databases:** File-backed databases use Write-Ahead Logging (`PRAGMA journal_mode=WAL`) with `PRAGMA synchronous=NORMAL` for better concurrent read/write performance. In-memory databases use `PRAGMA journal_mode=MEMORY` with `PRAGMA synchronous=OFF` for maximum speed. + +- **Schema-driven DDL:** Table structures are defined declaratively via `MetricsSchema` dataclasses containing `ColumnDef` tuples. The store generates `CREATE TABLE IF NOT EXISTS` statements from the schema at initialization. This means each plugin defines its own schema (LLM requests have `model` and `deployment` columns; Web requests have `path` and `redirect_hops` columns) without modifying the store. + +- **Timeseries UPSERT:** The `update_timeseries()` method uses SQLite's `INSERT ... ON CONFLICT(bucket_utc) DO UPDATE SET` to atomically increment counters per time bucket. Latency statistics (avg, p99) are computed via SQL aggregation rather than loading all values into Python. + +- **Stale connection cleanup:** When a new connection is created, connections from dead threads are detected and closed. Thread ID reuse is an acknowledged edge case that is acceptable for a testing tool. + +### LatencySimulator + +**File:** `engine/latency.py` + +Adds artificial delays to simulate real service latency. The formula is: + +``` +delay_seconds = max(0, (base_ms + uniform(-jitter_ms, +jitter_ms))) / 1000 +``` + +The result is always non-negative (clamped to 0). The simulator also provides `simulate_slow_response(min_sec, max_sec)` for slow response error injection where delays are specified as second-level ranges. + +Like the InjectionEngine, the RNG is injectable for deterministic testing. + +### ConfigLoader + +**File:** `engine/config_loader.py` + +Handles configuration loading with a four-layer precedence model: + +1. **CLI flags** (highest) -- Only explicitly provided values; `None` values are excluded so they do not override lower layers. +2. **Config file** -- YAML file specified by `--config`. +3. **Preset** -- Named YAML file from the plugin's `presets/` directory. +4. **Built-in defaults** (lowest) -- Pydantic field defaults. + +The `deep_merge(base, override)` function recursively merges dicts so that nested updates (e.g., changing only `burst.enabled` within `error_injection`) preserve sibling fields rather than resetting them to defaults. The function returns a new dict and never mutates its inputs. + +**Preset safety:** Preset names are validated against `^[a-zA-Z0-9][a-zA-Z0-9_-]*$` to prevent path traversal attacks. + +## Config Snapshot Pattern + +Request handlers in both `ChaosLLMServer` and `ChaosWebServer` snapshot component references at the start of each request: + +```python +with self._config_lock: + error_injector = self._error_injector + response_generator = self._response_generator + latency_simulator = self._latency_simulator +``` + +This snapshot is taken under `_config_lock` and produces local references that the handler uses for the remainder of the request. If a concurrent `update_config()` call swaps in new components while the request is in progress, the request continues using the old components, guaranteeing a consistent configuration view throughout its lifetime. + +This pattern is critical because the alternative -- reading `self._error_injector` at error check time and `self._response_generator` later at response time -- could produce a half-updated view where the error rates come from the new config but the response settings come from the old one. + +## Immutable Config Update Flow + +All Pydantic config models use `frozen=True` and `extra="forbid"`. This means fields cannot be mutated after construction and unknown fields cause validation errors. + +Runtime configuration updates through `POST /admin/config` follow this sequence: + +1. **Receive** the partial update dict from the HTTP request body. +2. **Deep-merge** the update with the current config (preserving unspecified nested fields). +3. **Construct** new Pydantic model instances from the merged dict (validation happens here). +4. **Create** new component instances (e.g., new `ErrorInjector`, new `ResponseGenerator`) from the new config. This happens outside the lock because construction and validation may be expensive. +5. **Swap** the new components atomically under `_config_lock`. + +If validation fails at step 3, no changes are applied and a 422 error is returned. If construction succeeds, the swap in step 5 is an atomic pointer replacement -- there is no intermediate state where some components are updated and others are not. + +## Thread Safety Model + +Errorworks is designed for multi-worker uvicorn deployments. The thread safety strategy has several layers: + +- **`_config_lock`** (per-server instance): Protects reads and writes of component references (`_error_injector`, `_response_generator`, etc.). The lock is held briefly for pointer reads (snapshot) and pointer swaps (update), never for request processing. + +- **InjectionEngine lock**: Protects the burst start timestamp. Held only for the time calculation. + +- **MetricsStore thread-local connections**: Each thread gets its own SQLite connection, avoiding cross-thread connection sharing entirely. + +- **Immutable config models**: Frozen Pydantic models cannot be accidentally mutated by concurrent readers. + +- **Best-effort metrics recording**: Metrics writes that fail (SQLite errors) are logged but never propagated to the caller. A metrics side-effect must not replace an intended chaos response with an unintended real 500 error. + +## Adding a New Server Type + +To add a new chaos server type (e.g., email, gRPC, GraphQL), follow this pattern: + +1. **Create a new package** under `src/errorworks/` (e.g., `src/errorworks/email/`). + +2. **Define config models** in `config.py`: + - Create an error injection config with domain-specific `_pct` fields + - Create a content/response config appropriate to the protocol + - Create a top-level config composing `ServerConfig`, `MetricsConfig`, `LatencyConfig`, and your domain configs + - Wire up `load_config()` using the shared `config_loader.load_config()` generic function + +3. **Define a metrics schema** using `MetricsSchema` and `ColumnDef` with domain-specific columns for the requests and timeseries tables. + +4. **Create an error injector** that: + - Composes an `InjectionEngine` instance + - Builds `ErrorSpec` lists from your config (with burst-aware adjustments) + - Calls `engine.select(specs)` and maps the selected tag to a domain-specific decision dataclass + +5. **Create a server class** that: + - Composes all components (error injector, content generator, latency simulator, metrics recorder) + - Implements the `ChaosServer` protocol from `engine/admin.py` (`get_admin_token`, `get_current_config`, `update_config`, `reset`, `export_metrics`, `get_stats`) + - Uses the config snapshot pattern in request handlers + - Uses the immutable config update flow in `update_config()` + +6. **Register routes** including `/health`, `/admin/*` (delegating to `engine.admin` handlers), and your domain-specific endpoints. + +7. **Add a CLI** using Typer, with a `serve` command and a `presets` command. Register it as a console script in `pyproject.toml` and add it as a subcommand to `chaosengine`. + +8. **Add presets** as YAML files in a `presets/` directory within your package. + +The shared engine layer handles all the infrastructure: burst timing, selection algorithms, SQLite management, config loading, admin authentication, and deep merge. Your plugin only needs to define what errors look like in your domain and how to render responses. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 0000000..5e017e9 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,51 @@ +# Installation + +## Prerequisites + +- **Python 3.12** or later + +## Install from PyPI + +=== "pip" + + ```bash + pip install errorworks + ``` + +=== "uv" + + ```bash + uv add errorworks + ``` + +## Verify installation + +After installing, confirm the CLI is available: + +```bash +chaosllm --help +``` + +You should see usage information for the ChaosLLM server. You can also check +ChaosWeb: + +```bash +chaosweb --help +``` + +## Development install + +To work on errorworks itself or run the test suite: + +```bash +git clone https://github.com/johnm-dta/errorworks.git +cd errorworks +uv sync --all-extras +``` + +This installs all dependencies including test and development extras. Run the +test suite to verify everything is working: + +```bash +uv run pytest +``` diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 0000000..8245b9c --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,112 @@ +# Quick Start + +This walkthrough takes you from zero to chaos-testing in a few minutes. You will +start fake servers, observe fault injection in action, check metrics, and write +your first test using the pytest fixture. + +## 1. Start ChaosLLM + +Launch a fake OpenAI-compatible server with the `realistic` preset, which +configures a mix of successful responses, rate limits, and server errors: + +```bash +chaosllm serve --preset=realistic +``` + +The server starts on `http://localhost:8000` by default. + +## 2. Make a request + +In another terminal, send a standard chat completion request: + +```bash +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer test-key" \ + -d '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "What is chaos testing?"}] + }' +``` + +## 3. Observe the chaos + +Run the curl command several times. You will see a mix of: + +- **200 OK** -- a generated chat completion response +- **429 Too Many Requests** -- simulated rate limiting +- **503 Service Unavailable** -- simulated server overload +- Occasional malformed responses (truncated JSON, wrong content types) + +This is exactly the kind of unreliability your client code needs to handle. + +## 4. Check metrics + +ChaosLLM records every request. Query the admin stats endpoint to see what +happened: + +```bash +curl http://localhost:8000/admin/stats \ + -H "Authorization: Bearer " +``` + +!!! note + The admin token is randomly generated at startup and printed to the server + log. You can also set it explicitly via the `server.admin_token` config + field or the `--admin-token` CLI flag. + +The response includes counts of each status code returned, latency percentiles, +and error category breakdowns. + +## 5. Try ChaosWeb + +ChaosWeb works the same way, but serves HTML pages for scraping resilience tests: + +```bash +chaosweb serve --preset=realistic +``` + +The web server starts on `http://localhost:8200` by default. Fetch a page: + +```bash +curl http://localhost:8200/articles/test +``` + +You will see a mix of valid HTML, encoding mismatches, truncated content, and +other failure modes that commonly break web scrapers. + +## 6. Use the pytest fixture + +The real power of errorworks is in automated testing. The built-in pytest fixtures +let you spin up an in-process fake server with no real network socket: + +```python +import pytest + + +@pytest.mark.chaosllm(preset="realistic", rate_limit_pct=25.0) +def test_retry_on_rate_limit(chaosllm): + response = chaosllm.post_completion( + model="gpt-4", + messages=[{"role": "user", "content": "test"}], + ) + assert response.status_code in (200, 429) +``` + +The `@pytest.mark.chaosllm` marker configures the server for that test. The +`chaosllm` fixture provides helper methods like `post_completion()`, +`get_stats()`, and `update_config()`. + +To run this test, make sure errorworks is installed with test extras and invoke +pytest: + +```bash +uv run pytest -m chaosllm +``` + +## Next steps + +- Learn about all the fault types ChaosLLM can inject: [ChaosLLM Guide](../guide/chaosllm.md) +- Explore ChaosWeb's scraping-specific faults: [ChaosWeb Guide](../guide/chaosweb.md) +- See available presets and how to customize them: [Presets](../guide/presets.md) +- Dive into the testing fixture API: [Testing Fixtures](../guide/testing-fixtures.md) diff --git a/docs/guide/chaosllm.md b/docs/guide/chaosllm.md new file mode 100644 index 0000000..537a023 --- /dev/null +++ b/docs/guide/chaosllm.md @@ -0,0 +1,249 @@ +# ChaosLLM Guide + +ChaosLLM is a fake OpenAI-compatible chat completions server that injects configurable faults into LLM API responses. Point your LLM client at ChaosLLM instead of the real API, and it will return a mix of successful responses and realistic failures -- rate limits, timeouts, malformed JSON, and more. + +Use ChaosLLM to verify that your LLM pipeline handles every failure mode before it hits production. + +## Quick Start + +```bash +# Start with a realistic error profile +uv run chaosllm serve --preset=realistic + +# Your client talks to localhost:8000 instead of api.openai.com +export OPENAI_BASE_URL=http://127.0.0.1:8000/v1 +``` + +## Endpoints + +### Chat Completions + +| Endpoint | Method | Description | +|---|---|---| +| `/v1/chat/completions` | POST | OpenAI-compatible chat completions | +| `/openai/deployments/{deployment}/chat/completions` | POST | Azure OpenAI-compatible chat completions | + +Both endpoints accept the standard OpenAI request body: + +```json +{ + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello"} + ] +} +``` + +### Health and Admin + +| Endpoint | Method | Auth | Description | +|---|---|---|---| +| `/health` | GET | None | Server health check (includes `run_id`, `started_utc`, `in_burst`) | +| `/admin/config` | GET | Bearer token | View current configuration | +| `/admin/config` | POST | Bearer token | Update configuration at runtime | +| `/admin/stats` | GET | Bearer token | Request statistics summary | +| `/admin/export` | GET | Bearer token | Export raw metrics data | +| `/admin/reset` | POST | Bearer token | Reset metrics and start new run | + +Admin endpoints require an `Authorization: Bearer ` header. The token is auto-generated at startup and printed to the console, or you can set it in your config file. + +## Error Injection + +ChaosLLM injects three categories of errors, each controlled by percentage fields (0-100) in the configuration. + +### HTTP Errors + +These return proper HTTP error responses with OpenAI-formatted error bodies: + +| Error Type | Status Code | Config Field | Description | +|---|---|---|---| +| Rate Limit | 429 | `rate_limit_pct` | Includes `Retry-After` header | +| Capacity | 529 | `capacity_529_pct` | Azure-specific "model overloaded" | +| Service Unavailable | 503 | `service_unavailable_pct` | Temporary outage | +| Bad Gateway | 502 | `bad_gateway_pct` | Upstream failure | +| Gateway Timeout | 504 | `gateway_timeout_pct` | Upstream timeout | +| Internal Error | 500 | `internal_error_pct` | Server-side failure | +| Forbidden | 403 | `forbidden_pct` | Permission denied | +| Not Found | 404 | `not_found_pct` | Resource missing | + +Rate limit and capacity errors include a `Retry-After` header with a random value in the configured range (default `[1, 5]` seconds). + +### Connection Failures + +These simulate network-level problems that your HTTP client must handle: + +| Error Type | Config Field | Behavior | +|---|---|---| +| Timeout | `timeout_pct` | Hangs for `timeout_sec` range, then returns 504 or drops | +| Connection Reset | `connection_reset_pct` | Immediately drops the TCP connection | +| Connection Failed | `connection_failed_pct` | Short delay (`connection_failed_lead_sec`), then drops | +| Connection Stall | `connection_stall_pct` | Optional start delay, then stalls for `connection_stall_sec`, then drops | +| Slow Response | `slow_response_pct` | Delays `slow_response_sec` then returns a successful response | + +### Malformed Responses + +These return HTTP 200 with corrupted content -- the hardest failures to detect: + +| Error Type | Config Field | What Goes Wrong | +|---|---|---| +| Invalid JSON | `invalid_json_pct` | Response body is not parseable JSON | +| Truncated | `truncated_pct` | JSON cuts off mid-stream | +| Empty Body | `empty_body_pct` | 200 OK with zero-length body | +| Missing Fields | `missing_fields_pct` | Valid JSON but missing `choices`, `message`, etc. | +| Wrong Content-Type | `wrong_content_type_pct` | Returns `text/html` instead of `application/json` | + +### Selection Mode + +The `selection_mode` field controls how errors are chosen when multiple types are configured: + +- **`priority`** (default): Errors are evaluated in a fixed order. The first one whose random check passes wins. This gives predictable behavior -- higher-priority errors (connection failures) fire before lower-priority ones (malformed responses). +- **`weighted`**: All configured error percentages are treated as proportional weights. A single random roll selects the error type. This gives a more uniform distribution. + +## Response Modes + +When a request is not selected for error injection, ChaosLLM generates a successful response using one of four modes: + +### Random (default) + +Generates responses with random words from a configurable vocabulary: + +```yaml +response: + mode: random + random: + min_words: 20 + max_words: 100 + vocabulary: english # or "lorem" for Lorem Ipsum +``` + +### Template + +Renders responses through a Jinja2 `SandboxedEnvironment` with built-in helpers: + +```yaml +response: + mode: template + template: + body: '{"result": "processed at {{ timestamp() }}"}' +``` + +Available template helpers include `random_choice`, `random_float`, `timestamp`, and others. + +### Echo + +Reflects the user's input back in the response. Useful for verifying your client sends the right data: + +```yaml +response: + mode: echo +``` + +### Preset + +Loads canned responses from a JSONL file: + +```yaml +response: + mode: preset + preset: + file: ./responses.jsonl + selection: random # or "sequential" +``` + +### Per-Request Overrides + +When `allow_header_overrides` is `true` (the default), clients can override the response mode per-request: + +```bash +curl -H "X-Fake-Response-Mode: echo" \ + -H "X-Fake-Template: {\"echo\": \"{{ messages[-1].content }}\"}" \ + http://localhost:8000/v1/chat/completions \ + -d '{"model": "gpt-4", "messages": [{"role": "user", "content": "test"}]}' +``` + +## Burst Patterns + +Bursts simulate periodic provider stress -- a wave of rate limits and capacity errors that comes and goes: + +```yaml +error_injection: + burst: + enabled: true + interval_sec: 60 # A burst starts every 60 seconds + duration_sec: 5 # Each burst lasts 5 seconds + rate_limit_pct: 50 # During burst: 50% rate limits + capacity_pct: 30 # During burst: 30% capacity errors +``` + +Outside burst windows, normal error percentages apply. During a burst, the burst percentages temporarily override the baseline rates for rate limits and capacity errors. + +!!! tip + The `/health` endpoint includes an `in_burst` field so you can observe burst timing from your test harness. + +## Available Presets + +ChaosLLM ships with six presets. Use them with `--preset=`: + +| Preset | Error Rate | Latency | Burst | Best For | +|---|---|---|---|---| +| `silent` | 0% | 10ms +/- 5ms | Off | Baseline measurements, throughput testing | +| `gentle` | ~2% | 50ms +/- 20ms | Off | Basic functionality testing, debugging | +| `realistic` | ~10% | 100ms +/- 50ms | 60s/5s | Production-like conditions | +| `stress_aimd` | ~23% | 30ms +/- 15ms | 30s/5s | AIMD throttle and backoff testing | +| `stress_extreme` | ~45% | 10ms +/- 5ms | 15s/5s | Survival under harsh conditions | +| `chaos` | ~25% | 100ms +/- 100ms | 20s/8s | Error handling coverage, every failure type | + +### Preset Details + +**`silent`** -- Zero errors. Every request succeeds. Use this to establish baseline throughput before adding chaos. + +**`gentle`** -- Minimal error injection (1% rate limit, 0.5% capacity, 0.5% service unavailable). No connection failures or malformed responses. Good for verifying your pipeline works at all. + +**`realistic`** -- Mimics typical Azure OpenAI behavior. Moderate rate limiting (5%), occasional capacity errors (2%), rare connection issues, and very rare malformed responses. Bursts every 60 seconds elevate rate limits to 50%. + +**`stress_aimd`** -- Specifically targets AIMD throttle testing. High rate limits (15%) and capacity errors (5%) with frequent 30-second burst cycles. Connection failures are disabled to focus on HTTP-level retry behavior. + +**`stress_extreme`** -- Maximum HTTP stress. 20% rate limits, 10% capacity errors, 5% internal errors. Aggressive 15-second burst cycles with 90% rate limiting during bursts. Includes malformed responses (3% invalid JSON, 2% truncated). + +**`chaos`** -- Every error type is active. HTTP errors, connection failures, malformed responses, and aggressive bursts. Not suitable for performance measurements due to the wide variance, but excellent for error handling coverage. + +## Usage Examples + +### CLI + +```bash +# Start with a preset +uv run chaosllm serve --preset=realistic + +# Start with a custom config file +uv run chaosllm serve --config=my-config.yaml + +# Preset + overrides +uv run chaosllm serve --preset=gentle --rate-limit-pct=10.0 + +# Via the unified CLI +uv run chaosengine llm serve --preset=realistic +``` + +### Python + +```python +from errorworks.llm.config import ChaosLLMConfig, load_config +from errorworks.llm.server import ChaosLLMServer, create_app + +# Quick start +config = load_config(preset="realistic") +app = create_app(config) + +# With full control +server = ChaosLLMServer(config) +server.update_config({"error_injection": {"rate_limit_pct": 25.0}}) +stats = server.get_stats() +``` + +## Related Pages + +- [Presets](presets.md) -- Full preset comparison and customization +- [Configuration](configuration.md) -- YAML config file structure and precedence rules +- [Metrics](metrics.md) -- Querying request statistics +- [Testing Fixtures](testing-fixtures.md) -- In-process testing with pytest diff --git a/docs/guide/chaosweb.md b/docs/guide/chaosweb.md new file mode 100644 index 0000000..71ad204 --- /dev/null +++ b/docs/guide/chaosweb.md @@ -0,0 +1,243 @@ +# ChaosWeb Guide + +ChaosWeb is a fake web server that injects configurable faults into HTTP responses for testing web scraping pipeline resilience. It serves HTML pages on any URL path and randomly injects errors -- anti-scraping blocks, broken encoding, SSRF redirects, and more. + +Point your scraper at ChaosWeb to verify it handles every real-world failure mode before scraping production sites. + +## Quick Start + +```bash +# Start with a realistic error profile +uv run chaosweb serve --preset=realistic + +# Your scraper fetches from localhost:8200 instead of the real site +curl http://127.0.0.1:8200/articles/some-page +``` + +## Endpoints + +### Content Serving + +| Endpoint | Method | Description | +|---|---|---| +| `/{any-path}` | GET | Catch-all route -- serves HTML with error injection | +| `/redirect` | GET | Redirect loop handler (hop counter management) | + +Any GET request to any path returns either a successful HTML page or an injected error. The path is available to templates and echo mode for generating path-specific content. + +### Health and Admin + +| Endpoint | Method | Auth | Description | +|---|---|---|---| +| `/health` | GET | None | Server health check (includes `run_id`, `started_utc`, `in_burst`) | +| `/admin/config` | GET | Bearer token | View current configuration | +| `/admin/config` | POST | Bearer token | Update configuration at runtime | +| `/admin/stats` | GET | Bearer token | Request statistics summary | +| `/admin/export` | GET | Bearer token | Export raw metrics data | +| `/admin/reset` | POST | Bearer token | Reset metrics and start new run | + +Admin endpoints require an `Authorization: Bearer ` header. + +## Error Injection + +ChaosWeb injects five categories of errors, each controlled by percentage fields (0-100). + +### HTTP Errors + +Standard HTTP error responses with HTML error pages: + +| Error Type | Status Code | Config Field | Description | +|---|---|---|---| +| Rate Limit | 429 | `rate_limit_pct` | Anti-scraping throttle, includes `Retry-After` | +| Forbidden | 403 | `forbidden_pct` | Bot detection block | +| Not Found | 404 | `not_found_pct` | Deleted or missing page | +| Gone | 410 | `gone_pct` | Permanently removed resource | +| Payment Required | 402 | `payment_required_pct` | Paywall / quota exceeded | +| Unavailable for Legal | 451 | `unavailable_for_legal_pct` | Geo-blocking | +| Service Unavailable | 503 | `service_unavailable_pct` | Maintenance page | +| Bad Gateway | 502 | `bad_gateway_pct` | Upstream failure | +| Gateway Timeout | 504 | `gateway_timeout_pct` | Upstream timeout | +| Internal Error | 500 | `internal_error_pct` | Server-side failure | + +### Connection Failures + +Network-level problems your scraper must handle: + +| Error Type | Config Field | Behavior | +|---|---|---| +| Timeout | `timeout_pct` | Hangs for `timeout_sec` range, then returns 504 | +| Connection Reset | `connection_reset_pct` | Sends headers then drops the connection | +| Connection Stall | `connection_stall_pct` | Delays, stalls, then disconnects | +| Slow Response | `slow_response_pct` | Delays `slow_response_sec` then returns successful HTML | +| Incomplete Response | `incomplete_response_pct` | Sends partial HTML body then disconnects | + +!!! warning + Incomplete responses are particularly tricky -- your scraper receives a 200 status code and partial HTML, then the connection drops. Always validate that your parsed HTML is structurally complete. + +### Content Malformations + +HTTP 200 responses with corrupted content -- the subtlest failures: + +| Error Type | Config Field | What Goes Wrong | +|---|---|---| +| Wrong Content-Type | `wrong_content_type_pct` | Declares `application/pdf` or similar instead of `text/html` | +| Encoding Mismatch | `encoding_mismatch_pct` | Header says UTF-8, body is ISO-8859-1 | +| Truncated HTML | `truncated_html_pct` | HTML cut off mid-tag | +| Invalid Encoding | `invalid_encoding_pct` | Non-decodable bytes in the declared encoding | +| Charset Confusion | `charset_confusion_pct` | HTTP header says one charset, `` tag says another | +| Malformed Meta | `malformed_meta_pct` | Invalid `` directives | + +### Redirect Injection + +Tests your scraper's redirect handling: + +| Error Type | Config Field | Behavior | +|---|---|---| +| Redirect Loop | `redirect_loop_pct` | Chain of 301 redirects up to `max_redirect_loop_hops` (default 10) | +| SSRF Redirect | `ssrf_redirect_pct` | 301 redirect to private IPs (169.254.169.254, 10.x.x.x, etc.) | + +!!! warning + SSRF redirect testing verifies that your scraper blocks redirects to private/internal addresses. Real scrapers should never follow redirects to cloud metadata endpoints like `http://169.254.169.254/`. + +### Burst Patterns + +Bursts simulate coordinated anti-scraping escalation -- a site suddenly blocks most requests, then backs off: + +```yaml +error_injection: + burst: + enabled: true + interval_sec: 60 # Burst every 60 seconds + duration_sec: 8 # Lasts 8 seconds + rate_limit_pct: 40 # During burst: 40% rate limits + forbidden_pct: 30 # During burst: 30% forbidden +``` + +### Selection Mode + +- **`priority`** (default): Errors evaluated in category order (connection > redirect > HTTP > malformed). First match wins. +- **`weighted`**: All percentages treated as proportional weights for uniform distribution. + +## Content Modes + +When a request is not selected for error injection, ChaosWeb generates HTML content using one of four modes: + +### Random (default) + +Generates syntactically valid HTML pages with random content: + +```yaml +content: + mode: random + random: + min_words: 100 + max_words: 500 + vocabulary: english # or "lorem" +``` + +### Template + +Renders HTML through a Jinja2 `SandboxedEnvironment`: + +```yaml +content: + mode: template + template: + body: > + {{ path }} +

{{ path }}

+

{{ random_words(100, 300) }}

+``` + +Template helpers include `random_words`, `random_choice`, `random_float`, `timestamp`, and more. The `path` variable contains the requested URL path. + +### Echo + +Reflects request information as HTML. Content is HTML-escaped to prevent XSS when rendering in a browser: + +```yaml +content: + mode: echo +``` + +### Preset + +Loads HTML page snapshots from a JSONL file: + +```yaml +content: + mode: preset + preset: + file: ./pages.jsonl + selection: random # or "sequential" +``` + +### Per-Request Overrides + +When `allow_header_overrides` is `true` (the default), use the `X-Fake-Content-Mode` header: + +```bash +curl -H "X-Fake-Content-Mode: echo" http://localhost:8200/articles/test +``` + +## Available Presets + +ChaosWeb ships with five presets. Use them with `--preset=`: + +| Preset | Error Rate | Latency | Burst | Best For | +|---|---|---|---|---| +| `silent` | 0% | 200ms +/- 100ms | Off | Baseline throughput measurement | +| `gentle` | ~2% | 100ms +/- 50ms | Off | Basic scraping functionality testing | +| `realistic` | ~19% | 300ms +/- 150ms | 60s/8s | Production-like scraping conditions | +| `stress_scraping` | ~57% | 500ms +/- 200ms | 60s/10s | Heavy anti-scraping resilience testing | +| `stress_extreme` | ~98% | 800ms +/- 400ms | 30s/8s | Breaking-point stress testing | + +### Preset Details + +**`silent`** -- Zero errors. Every request returns HTML. Use this to establish baseline scraping throughput. + +**`gentle`** -- Minimal error injection: 1% rate limits and 1% not-found errors. No connection failures, malformations, or bursts. Verifies your scraper handles basic error paths. + +**`realistic`** -- Mimics typical web scraping conditions. Moderate rate limiting (5%), bot detection (3% forbidden), occasional slow responses (5%), and rare encoding issues. Bursts every 60 seconds simulate coordinated anti-scraping responses. + +**`stress_scraping`** -- Heavy anti-scraping simulation. 15% rate limits, 10% forbidden, connection failures (5% timeout, 3% reset), content malformations (3% wrong content-type, 2% encoding mismatch), and SSRF redirect testing (1%). Aggressive burst escalation with 80% rate limiting. + +**`stress_extreme`** -- Every error type is active at high rates. 25% rate limits, 15% forbidden, 10% timeout, 5% connection reset, heavy content malformations, redirect loops (3%), and SSRF redirects (2%). Very aggressive 30-second burst cycles. Use for finding failure modes and verifying graceful degradation. + +## Usage Examples + +### CLI + +```bash +# Start with a preset +uv run chaosweb serve --preset=realistic + +# Start with a custom config file +uv run chaosweb serve --config=my-config.yaml + +# Via the unified CLI +uv run chaosengine web serve --preset=realistic +``` + +### Python + +```python +from errorworks.web.config import ChaosWebConfig, load_config +from errorworks.web.server import ChaosWebServer, create_app + +# Quick start +config = load_config(preset="realistic") +app = create_app(config) + +# With full control +server = ChaosWebServer(config) +server.update_config({"error_injection": {"rate_limit_pct": 25.0}}) +stats = server.get_stats() +``` + +## Related Pages + +- [Presets](presets.md) -- Full preset comparison and customization +- [Configuration](configuration.md) -- YAML config file structure and precedence rules +- [Metrics](metrics.md) -- Querying request statistics +- [Testing Fixtures](testing-fixtures.md) -- In-process testing with pytest diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md new file mode 100644 index 0000000..4deb20f --- /dev/null +++ b/docs/guide/configuration.md @@ -0,0 +1,313 @@ +# Configuration Guide + +Errorworks servers are configured through three layered methods: presets, YAML config files, and CLI flags. Each layer overrides the one below it, giving you fine-grained control without rewriting entire configuration files. + +## Configuration Precedence + +From highest to lowest priority: + +``` +CLI flags > Config file > Preset > Built-in defaults +``` + +Each layer is deep-merged into the one below. You only need to specify the fields you want to change -- everything else inherits from the lower layer. + +### Example: How Layers Combine + +Given this preset (`realistic`): + +```yaml +error_injection: + rate_limit_pct: 5.0 + capacity_529_pct: 2.0 + burst: + enabled: true + interval_sec: 60 + duration_sec: 5 +latency: + base_ms: 100 + jitter_ms: 50 +``` + +And this config file (`my-config.yaml`): + +```yaml +error_injection: + rate_limit_pct: 15.0 + burst: + interval_sec: 30 +``` + +And this CLI flag: + +```bash +uv run chaosllm serve --preset=realistic --config=my-config.yaml --rate-limit-pct=25.0 +``` + +The final configuration is: + +```yaml +error_injection: + rate_limit_pct: 25.0 # CLI flag wins + capacity_529_pct: 2.0 # From preset (config file didn't touch it) + burst: + enabled: true # From preset (preserved by deep merge) + interval_sec: 30 # Config file overrode preset + duration_sec: 5 # From preset (preserved by deep merge) +latency: + base_ms: 100 # From preset + jitter_ms: 50 # From preset +``` + +!!! note + Deep merge is recursive. When the config file sets `burst.interval_sec`, it does not reset `burst.enabled` or `burst.duration_sec` to their defaults. Only the fields you explicitly set are changed. + +## YAML Config File + +Pass a YAML file with `--config=path/to/config.yaml`. The file structure mirrors the configuration models. + +### ChaosLLM Full Example + +```yaml +server: + host: "127.0.0.1" + port: 8000 + workers: 4 + admin_token: "my-secret-token" # Auto-generated if omitted + +metrics: + database: "file:chaosllm-metrics?mode=memory&cache=shared" + timeseries_bucket_sec: 1 + +response: + mode: random # random | template | echo | preset + allow_header_overrides: true + max_template_length: 10000 + random: + min_words: 20 + max_words: 100 + vocabulary: english # english | lorem + template: + body: '{"result": "ok"}' + preset: + file: ./responses.jsonl + selection: random # random | sequential + +latency: + base_ms: 100 + jitter_ms: 50 + +error_injection: + # Selection strategy + selection_mode: priority # priority | weighted + + # HTTP errors (0-100 percentage) + rate_limit_pct: 5.0 + capacity_529_pct: 2.0 + service_unavailable_pct: 0.5 + bad_gateway_pct: 0.1 + gateway_timeout_pct: 0.2 + internal_error_pct: 0.2 + forbidden_pct: 0.0 + not_found_pct: 0.0 + + # Retry-After header range for rate limit errors + retry_after_sec: [1, 5] + + # Connection failures + timeout_pct: 0.2 + timeout_sec: [30, 60] + connection_reset_pct: 0.1 + connection_failed_pct: 0.0 + connection_failed_lead_sec: [2, 5] + connection_stall_pct: 0.0 + connection_stall_start_sec: [0, 2] + connection_stall_sec: [30, 60] + slow_response_pct: 1.0 + slow_response_sec: [5, 15] + + # Malformed responses + invalid_json_pct: 0.1 + truncated_pct: 0.1 + empty_body_pct: 0.0 + missing_fields_pct: 0.1 + wrong_content_type_pct: 0.0 + + # Burst patterns + burst: + enabled: true + interval_sec: 60 + duration_sec: 5 + rate_limit_pct: 50 + capacity_pct: 30 +``` + +### ChaosWeb Full Example + +```yaml +server: + host: "127.0.0.1" + port: 8200 + workers: 4 + admin_token: "my-secret-token" + +metrics: + database: "file:chaosweb-metrics?mode=memory&cache=shared" + timeseries_bucket_sec: 1 + +content: + mode: random # random | template | echo | preset + allow_header_overrides: true + max_template_length: 10000 + default_content_type: "text/html; charset=utf-8" + random: + min_words: 100 + max_words: 500 + vocabulary: english + template: + body: "

{{ path }}

{{ random_words(100, 300) }}

" + preset: + file: ./pages.jsonl + selection: random + +latency: + base_ms: 300 + jitter_ms: 150 + +error_injection: + selection_mode: priority + + # HTTP errors + rate_limit_pct: 5.0 + forbidden_pct: 3.0 + not_found_pct: 2.0 + gone_pct: 0.0 + payment_required_pct: 0.0 + unavailable_for_legal_pct: 0.0 + service_unavailable_pct: 1.0 + bad_gateway_pct: 0.2 + gateway_timeout_pct: 0.3 + internal_error_pct: 0.5 + retry_after_sec: [5, 30] + + # Connection failures + timeout_pct: 0.5 + timeout_sec: [10, 30] + connection_reset_pct: 0.2 + connection_stall_pct: 0.0 + connection_stall_start_sec: [0, 2] + connection_stall_sec: [30, 60] + slow_response_pct: 5.0 + slow_response_sec: [3, 10] + incomplete_response_pct: 0.0 + incomplete_response_bytes: [100, 1000] + + # Content malformations + wrong_content_type_pct: 1.0 + encoding_mismatch_pct: 1.0 + truncated_html_pct: 0.5 + invalid_encoding_pct: 0.0 + charset_confusion_pct: 0.0 + malformed_meta_pct: 0.0 + + # Redirects + redirect_loop_pct: 0.0 + max_redirect_loop_hops: 10 + ssrf_redirect_pct: 0.0 + + # Burst patterns + burst: + enabled: true + interval_sec: 60 + duration_sec: 8 + rate_limit_pct: 40 + forbidden_pct: 30 +``` + +## Configuration Sections + +### Server + +| Field | Type | Default (LLM/Web) | Description | +|---|---|---|---| +| `host` | string | `127.0.0.1` | Bind address | +| `port` | int | `8000` / `8200` | Listen port | +| `workers` | int | `4` | Uvicorn worker count | +| `admin_token` | string | auto-generated | Bearer token for `/admin/*` endpoints | + +!!! warning + Binding to `0.0.0.0` or `::` is blocked by default. ChaosLLM and ChaosWeb are testing tools and should not be exposed to the network. Set `allow_external_bind: true` at the top level to override this safety check. + +### Metrics + +| Field | Type | Default | Description | +|---|---|---|---| +| `database` | string | in-memory (shared) | SQLite database path or URI | +| `timeseries_bucket_sec` | int | `1` | Time-series aggregation bucket size in seconds | + +Use `--database=/path/to/metrics.db` for persistent file-backed storage. See the [Metrics Guide](metrics.md) for details. + +### Latency + +| Field | Type | Default | Description | +|---|---|---|---| +| `base_ms` | int | `50` | Base latency added to every response (milliseconds) | +| `jitter_ms` | int | `30` | Random jitter range (+/- milliseconds) | + +The actual delay per request is `(base_ms + random(-jitter_ms, +jitter_ms)) / 1000` seconds, clamped to zero. + +## Runtime Config Updates + +You can update the configuration while the server is running via `POST /admin/config`: + +```bash +# Increase rate limiting at runtime +curl -X POST http://localhost:8000/admin/config \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "error_injection": { + "rate_limit_pct": 25.0 + } + }' +``` + +```bash +# View current config +curl http://localhost:8000/admin/config \ + -H "Authorization: Bearer $ADMIN_TOKEN" +``` + +Runtime updates use the same deep-merge behavior as config file layering. You only send the fields you want to change -- everything else is preserved. + +### How Runtime Updates Work + +The server uses immutable Pydantic models (`frozen=True`) for all configuration. When you POST an update: + +1. The current config is serialized to a dict +2. Your update is deep-merged into it +3. New immutable model instances are created and validated +4. The new components (error injector, response generator, latency simulator) are built +5. References are atomically swapped under a lock + +Requests that are already in-flight continue using the old configuration. This guarantees each request sees a consistent configuration throughout its lifetime. + +### Validation + +If your update contains invalid values, the server returns 422 with a validation error and the configuration is unchanged: + +```json +{ + "error": { + "type": "validation_error", + "message": "1 validation error for ErrorInjectionConfig\nrate_limit_pct\n Input should be less than or equal to 100 [type=less_than_equal, ...]" + } +} +``` + +## Related Pages + +- [Presets](presets.md) -- Pre-built configuration profiles +- [ChaosLLM](chaosllm.md) -- LLM-specific error types and endpoints +- [ChaosWeb](chaosweb.md) -- Web-specific error types and endpoints +- [Metrics](metrics.md) -- Metrics storage configuration diff --git a/docs/guide/metrics.md b/docs/guide/metrics.md new file mode 100644 index 0000000..62f4a89 --- /dev/null +++ b/docs/guide/metrics.md @@ -0,0 +1,256 @@ +# Metrics Guide + +Every request that passes through a ChaosLLM or ChaosWeb server is recorded in a SQLite database. You can query these metrics to understand error rates, latency distributions, and how your pipeline responds to injected faults. + +## What Gets Recorded + +Each request produces a row in the `requests` table with fields including: + +| Field | Description | +|---|---| +| `request_id` | Unique ID for this request | +| `timestamp_utc` | ISO-8601 timestamp | +| `endpoint` / `path` | The requested URL path | +| `outcome` | `success`, `error_injected`, `error_malformed`, `error_redirect`, etc. | +| `status_code` | HTTP status code returned (NULL for connection-level errors) | +| `error_type` | Specific error injected (e.g., `rate_limit`, `timeout`, `malformed_truncated`) | +| `injection_type` | Category of injection applied | +| `latency_ms` | Total request duration in milliseconds | +| `injected_delay_ms` | Artificial delay added (latency simulation + slow response) | + +ChaosLLM additionally records: + +| Field | Description | +|---|---| +| `model` | Requested model name (as sent by client, not fabricated) | +| `deployment` | Azure deployment name (if using Azure endpoint) | +| `message_count` | Number of messages in the chat request | +| `prompt_tokens_approx` | Approximate prompt token count | +| `response_tokens` | Response token count | +| `response_mode` | Content generation mode used (`random`, `template`, `echo`, `preset`) | + +ChaosWeb additionally records: + +| Field | Description | +|---|---| +| `content_type_served` | Content-Type header returned | +| `encoding_served` | Actual encoding used (for encoding mismatch errors) | +| `redirect_target` | SSRF redirect destination URL | +| `redirect_hops` | Number of hops in redirect chain | + +## Querying via Admin API + +All admin endpoints require authentication with `Authorization: Bearer `. + +### GET /admin/stats -- Summary Statistics + +Returns aggregated statistics for the current run: + +```bash +curl http://localhost:8000/admin/stats \ + -H "Authorization: Bearer $ADMIN_TOKEN" +``` + +Example response: + +```json +{ + "run_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "started_utc": "2025-03-15T14:30:00.123456+00:00", + "total_requests": 1500, + "requests_by_outcome": { + "success": 1275, + "error_injected": 195, + "error_malformed": 30 + }, + "error_rate": 15.0, + "requests_by_status_code": { + "200": 1305, + "429": 120, + "529": 45, + "503": 15, + "500": 15 + }, + "latency_stats": { + "avg_ms": 125.4, + "p50_ms": 108.2, + "p95_ms": 215.6, + "p99_ms": 485.3, + "max_ms": 15234.1 + } +} +``` + +The `latency_stats` object provides percentile-based latency distribution: + +| Field | Description | +|---|---| +| `avg_ms` | Mean latency across all requests | +| `p50_ms` | Median latency (50th percentile) | +| `p95_ms` | 95th percentile latency | +| `p99_ms` | 99th percentile latency | +| `max_ms` | Maximum observed latency | + +### GET /admin/export -- Raw Data Export + +Returns all raw request records and time-series data for external analysis: + +```bash +curl http://localhost:8000/admin/export \ + -H "Authorization: Bearer $ADMIN_TOKEN" +``` + +Example response: + +```json +{ + "run_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "started_utc": "2025-03-15T14:30:00.123456+00:00", + "requests": [ + { + "request_id": "req-001", + "timestamp_utc": "2025-03-15T14:30:01.000000+00:00", + "endpoint": "/v1/chat/completions", + "outcome": "success", + "status_code": 200, + "latency_ms": 112.5, + "model": "gpt-4", + "message_count": 3, + "response_mode": "random" + }, + { + "request_id": "req-002", + "timestamp_utc": "2025-03-15T14:30:01.500000+00:00", + "endpoint": "/v1/chat/completions", + "outcome": "error_injected", + "status_code": 429, + "error_type": "rate_limit", + "latency_ms": 2.1, + "model": "gpt-4", + "message_count": 1 + } + ], + "timeseries": [ + { + "bucket_utc": "2025-03-15T14:30:01+00:00", + "requests_total": 42, + "requests_success": 36, + "requests_rate_limited": 4, + "requests_error": 2, + "avg_latency_ms": 118.7, + "p99_latency_ms": 312.4 + } + ], + "config": { + "server": {"host": "127.0.0.1", "port": 8000, "workers": 4}, + "metrics": {"database": "file:chaosllm-metrics?mode=memory&cache=shared", "timeseries_bucket_sec": 1}, + "error_injection": {"rate_limit_pct": 5.0, "...": "..."}, + "response": {"mode": "random", "...": "..."}, + "latency": {"base_ms": 100, "jitter_ms": 50} + } +} +``` + +The export includes the full server configuration used for this run, making it self-documenting for later analysis. + +### POST /admin/reset -- Reset Metrics + +Clears all request and timeseries data and starts a new run: + +```bash +curl -X POST http://localhost:8000/admin/reset \ + -H "Authorization: Bearer $ADMIN_TOKEN" +``` + +Response: + +```json +{ + "status": "reset", + "new_run_id": "new-uuid-here" +} +``` + +!!! tip + Reset between test scenarios so metrics from one test do not contaminate the next. Each reset generates a new `run_id`. + +## Time-Series Aggregation + +Metrics are aggregated into time-series buckets using SQLite UPSERT. The bucket size is configurable via `timeseries_bucket_sec` (default: 1 second). + +Each bucket tracks: + +- `requests_total` -- Total requests in this time window +- Per-outcome counters (e.g., `requests_success`, `requests_rate_limited`) +- `avg_latency_ms` -- Average latency for the bucket +- `p99_latency_ms` -- Approximate 99th percentile latency + +Time-series data is included in the `/admin/export` response and is useful for observing how error rates and latency change over time, especially around burst windows. + +## Storage Options + +### In-Memory (Default) + +By default, metrics are stored in a shared in-memory SQLite database: + +```yaml +metrics: + database: "file:chaosllm-metrics?mode=memory&cache=shared" +``` + +This is fast and requires no cleanup, but data is lost when the server stops. The `cache=shared` URI parameter allows multiple threads to access the same in-memory database. + +### File-Backed + +For persistent storage, specify a file path: + +```bash +uv run chaosllm serve --preset=realistic --database=/tmp/metrics.db +``` + +Or in YAML: + +```yaml +metrics: + database: /tmp/metrics.db +``` + +File-backed databases use WAL (Write-Ahead Logging) mode and `synchronous=NORMAL` for good write performance without sacrificing durability. The directory is created automatically if it does not exist. + +!!! note + In-memory databases use `journal_mode=MEMORY` and `synchronous=OFF` for maximum speed, since durability is not a concern. + +## Thread Safety + +The MetricsStore uses thread-local SQLite connections. Each worker thread gets its own connection, avoiding contention. Connections are tracked and cleaned up when threads exit. + +Metrics recording is best-effort: if a SQLite write fails, the error is logged but the chaos response is still returned to the client. A metrics side-effect should never replace an intended chaos response with an unintended real 500. + +## Python API + +When using the server programmatically, you have direct access to metrics: + +```python +from errorworks.llm.server import ChaosLLMServer +from errorworks.llm.config import load_config + +config = load_config(preset="realistic") +server = ChaosLLMServer(config) + +# After running some requests... +stats = server.get_stats() +print(f"Total: {stats['total_requests']}, Error rate: {stats['error_rate']:.1f}%") + +# Export everything +data = server.export_metrics() + +# Reset for next test +new_run_id = server.reset() +``` + +## Related Pages + +- [Configuration](configuration.md) -- Metrics storage configuration options +- [ChaosLLM](chaosllm.md) -- LLM-specific metrics fields +- [ChaosWeb](chaosweb.md) -- Web-specific metrics fields +- [Testing Fixtures](testing-fixtures.md) -- Accessing metrics in pytest diff --git a/docs/guide/presets.md b/docs/guide/presets.md new file mode 100644 index 0000000..1c9e23f --- /dev/null +++ b/docs/guide/presets.md @@ -0,0 +1,126 @@ +# Presets Guide + +Presets are pre-built configuration profiles that set error rates, latency, burst patterns, and response generation for common testing scenarios. Instead of manually configuring dozens of fields, pick a preset that matches your testing goal. + +## Using Presets + +```bash +# CLI +uv run chaosllm serve --preset=realistic +uv run chaosweb serve --preset=stress_scraping + +# Python +from errorworks.llm.config import load_config +config = load_config(preset="realistic") +``` + +## ChaosLLM Presets + +### Comparison Table + +| Preset | Rate Limit | Capacity 529 | Server Errors | Connection Failures | Malformed | Burst | Latency | +|---|---|---|---|---|---|---|---| +| `silent` | 0% | 0% | 0% | 0% | 0% | Off | 10 +/- 5ms | +| `gentle` | 1% | 0.5% | 0.5% svc unavail | 0% | 0% | Off | 50 +/- 20ms | +| `realistic` | 5% | 2% | 0.5% svc, 0.2% internal, 0.1% bad gw, 0.2% gw timeout | 0.2% timeout, 0.1% reset, 1% slow | 0.1% invalid JSON, 0.1% truncated, 0.1% missing fields | 60s/5s (50% rl, 30% cap) | 100 +/- 50ms | +| `stress_aimd` | 15% | 5% | 2% svc, 0.5% internal, 0.2% bad gw, 0.3% gw timeout | 0.5% slow | 0% | 30s/5s (80% rl, 50% cap) | 30 +/- 15ms | +| `stress_extreme` | 20% | 10% | 3% svc, 5% internal, 1% bad gw, 1% gw timeout | 0% | 3% invalid JSON, 2% truncated | 15s/5s (90% rl, 70% cap) | 10 +/- 5ms | +| `chaos` | 6.25% | 3.13% | 1.88% svc, 1.88% internal, 1.25% bad gw, 1.25% gw timeout | 1.25% timeout, 0.94% reset, 1.88% slow | 1.25% invalid JSON, 0.94% truncated, 0.63% empty, 0.94% missing, 0.31% wrong ct | 20s/8s (90% rl, 70% cap) | 100 +/- 100ms | + +### When to Use Each + +| Preset | Use When | +|---|---| +| `silent` | Measuring maximum pipeline throughput without noise | +| `gentle` | Verifying basic pipeline operation and debugging | +| `realistic` | Testing against production-like Azure OpenAI conditions | +| `stress_aimd` | Tuning AIMD throttle parameters and backoff behavior | +| `stress_extreme` | Verifying pipeline survives under harsh HTTP error rates | +| `chaos` | Achieving error handling coverage across every failure type | + +## ChaosWeb Presets + +### Comparison Table + +| Preset | Rate Limit | Forbidden | Not Found | Connection Failures | Content Malform. | Redirects | Burst | Latency | +|---|---|---|---|---|---|---|---|---| +| `silent` | 0% | 0% | 0% | 0% | 0% | 0% | Off | 200 +/- 100ms | +| `gentle` | 1% | 0% | 1% | 0% | 0% | 0% | Off | 100 +/- 50ms | +| `realistic` | 5% | 3% | 2% | 0.5% timeout, 0.2% reset, 5% slow | 1% wrong ct, 1% encoding, 0.5% truncated | 0% | 60s/8s (40% rl, 30% forbid) | 300 +/- 150ms | +| `stress_scraping` | 15% | 10% | 3% | 5% timeout, 3% reset, 8% slow, 2% incomplete | 3% wrong ct, 2% encoding, 2% truncated, 1% charset | 1% SSRF | 60s/10s (80% rl, 50% forbid) | 500 +/- 200ms | +| `stress_extreme` | 25% | 15% | 5% | 10% timeout, 5% reset, 3% stall, 5% slow, 5% incomplete | 5% wrong ct, 3% encoding, 3% truncated, 2% invalid enc, 2% charset, 1% meta | 3% loop, 2% SSRF | 30s/8s (90% rl, 70% forbid) | 800 +/- 400ms | + +### When to Use Each + +| Preset | Use When | +|---|---| +| `silent` | Measuring scraping throughput without error injection overhead | +| `gentle` | Verifying basic scraping functionality works before adding stress | +| `realistic` | Testing against typical web scraping production conditions | +| `stress_scraping` | Verifying retry logic, backoff, and error routing under pressure | +| `stress_extreme` | Finding failure modes and verifying graceful degradation | + +## Combining Presets with Overrides + +Presets provide the base configuration, but you can override any setting on top. Configuration precedence (highest to lowest): + +1. **CLI flags** -- Override individual settings +2. **Config file** -- `--config=my-config.yaml` +3. **Preset** -- `--preset=realistic` +4. **Defaults** -- Built-in Pydantic defaults + +### Example: Preset with CLI Override + +```bash +# Start with realistic, but increase rate limiting +uv run chaosllm serve --preset=realistic --rate-limit-pct=20.0 +``` + +### Example: Preset with Config File Override + +```yaml +# my-overrides.yaml +error_injection: + rate_limit_pct: 20.0 + burst: + interval_sec: 30 # More frequent bursts +``` + +```bash +uv run chaosllm serve --preset=realistic --config=my-overrides.yaml +``` + +The config file's `rate_limit_pct: 20.0` overrides the preset's `5.0`, and the burst `interval_sec: 30` overrides the preset's `60`. All other preset values are preserved. + +### Example: Python with Overrides + +```python +from errorworks.llm.config import load_config + +config = load_config( + preset="realistic", + cli_overrides={ + "error_injection": {"rate_limit_pct": 20.0}, + "latency": {"base_ms": 200}, + }, +) +``` + +!!! tip + Deep merge means you only need to specify the fields you want to change. Nested objects are merged recursively -- you do not need to repeat the entire `burst` section just to change `interval_sec`. + +## Listing Available Presets + +```python +from errorworks.llm.config import list_presets as list_llm_presets +from errorworks.web.config import list_presets as list_web_presets + +print(list_llm_presets()) # ['chaos', 'gentle', 'realistic', 'silent', 'stress_aimd', 'stress_extreme'] +print(list_web_presets()) # ['gentle', 'realistic', 'silent', 'stress_extreme', 'stress_scraping'] +``` + +## Related Pages + +- [ChaosLLM](chaosllm.md) -- Full ChaosLLM endpoint and error injection reference +- [ChaosWeb](chaosweb.md) -- Full ChaosWeb endpoint and error injection reference +- [Configuration](configuration.md) -- YAML config file structure and precedence details diff --git a/docs/guide/testing-fixtures.md b/docs/guide/testing-fixtures.md new file mode 100644 index 0000000..93af88f --- /dev/null +++ b/docs/guide/testing-fixtures.md @@ -0,0 +1,321 @@ +# Testing Fixtures Guide + +Errorworks provides pytest fixtures that run ChaosLLM and ChaosWeb servers in-process using Starlette's `TestClient`. No real network sockets are opened -- requests go directly through the ASGI stack, making tests fast, isolated, and safe to run in parallel. + +## Setup + +Import the fixtures in your `conftest.py`: + +```python +# tests/conftest.py +from tests.fixtures.chaosllm import chaosllm_server # noqa: F401 +from tests.fixtures.chaosweb import chaosweb_server # noqa: F401 +``` + +Register the custom markers to avoid pytest warnings: + +```python +# tests/conftest.py (or pyproject.toml) +def pytest_configure(config): + config.addinivalue_line("markers", "chaosllm: ChaosLLM server configuration") + config.addinivalue_line("markers", "chaosweb: ChaosWeb server configuration") +``` + +## ChaosLLM Fixture + +### Basic Usage + +```python +def test_successful_completion(chaosllm_server): + """Test that completions work with no error injection.""" + response = chaosllm_server.post_completion( + messages=[{"role": "user", "content": "Hello"}], + model="gpt-4", + ) + assert response.status_code == 200 + data = response.json() + assert "choices" in data + assert data["choices"][0]["message"]["content"] +``` + +### Marker-Based Configuration + +Use `@pytest.mark.chaosllm(...)` to configure the server for a specific test: + +```python +import pytest + +@pytest.mark.chaosllm(preset="realistic", rate_limit_pct=25.0) +def test_rate_limit_handling(chaosllm_server): + """Test that the pipeline handles rate limits.""" + errors = 0 + for _ in range(100): + response = chaosllm_server.post_completion() + if response.status_code == 429: + errors += 1 + # With 25% rate limiting, expect roughly 25 errors out of 100 + assert errors > 10 +``` + +### Available Marker Kwargs + +The `chaosllm` marker accepts these keyword arguments: + +| Kwarg | Type | Description | +|---|---|---| +| `preset` | str | Base preset name (`silent`, `gentle`, `realistic`, etc.) | +| `rate_limit_pct` | float | 429 Rate Limit percentage | +| `capacity_529_pct` | float | 529 Capacity error percentage | +| `service_unavailable_pct` | float | 503 Service Unavailable percentage | +| `bad_gateway_pct` | float | 502 Bad Gateway percentage | +| `gateway_timeout_pct` | float | 504 Gateway Timeout percentage | +| `internal_error_pct` | float | 500 Internal Server Error percentage | +| `timeout_pct` | float | Timeout (hang) percentage | +| `connection_reset_pct` | float | Connection reset percentage | +| `connection_failed_pct` | float | Connection failed percentage | +| `connection_stall_pct` | float | Connection stall percentage | +| `slow_response_pct` | float | Slow response percentage | +| `invalid_json_pct` | float | Invalid JSON response percentage | +| `truncated_pct` | float | Truncated response percentage | +| `empty_body_pct` | float | Empty body response percentage | +| `missing_fields_pct` | float | Missing fields response percentage | +| `wrong_content_type_pct` | float | Wrong Content-Type percentage | +| `forbidden_pct` | float | 403 Forbidden percentage | +| `not_found_pct` | float | 404 Not Found percentage | +| `selection_mode` | str | `priority` or `weighted` | +| `base_ms` | int | Base latency in milliseconds | +| `jitter_ms` | int | Latency jitter in milliseconds | +| `mode` | str | Response mode (`random`, `template`, `echo`, `preset`) | + +!!! note + The fixture sets `base_ms=0` and `jitter_ms=0` by default so tests run without artificial delays. If you need latency simulation, set these explicitly in the marker. + +### Fixture Helpers + +The `ChaosLLMFixture` object provides: + +| Method/Property | Description | +|---|---| +| `post_completion(messages=..., model=..., **kwargs)` | POST to `/v1/chat/completions` | +| `post_azure_completion(deployment, messages=..., **kwargs)` | POST to Azure endpoint | +| `get_stats()` | Get metrics summary (same as `/admin/stats`) | +| `export_metrics()` | Export raw metrics data | +| `update_config(rate_limit_pct=..., ...)` | Update config at runtime | +| `reset()` | Reset metrics and start new run | +| `wait_for_requests(count, timeout=10.0)` | Block until N requests recorded | +| `run_id` | Current run ID | +| `url` | Base URL (`http://testserver`) | +| `admin_headers` | Dict with auth headers for admin endpoints | + +### Azure Endpoint Testing + +```python +def test_azure_deployment(chaosllm_server): + """Test Azure OpenAI endpoint compatibility.""" + response = chaosllm_server.post_azure_completion( + deployment="my-gpt4-deployment", + messages=[{"role": "user", "content": "Hello"}], + api_version="2024-02-01", + ) + assert response.status_code == 200 +``` + +## ChaosWeb Fixture + +### Basic Usage + +```python +def test_page_fetch(chaosweb_server): + """Test that pages load successfully.""" + response = chaosweb_server.fetch_page("/articles/test") + assert response.status_code == 200 + assert "html" in response.text.lower() +``` + +### Marker-Based Configuration + +```python +import pytest + +@pytest.mark.chaosweb(preset="stress_scraping", rate_limit_pct=25.0) +def test_scraper_resilience(chaosweb_server): + """Test scraper handles rate limiting under stress.""" + success = 0 + for _ in range(50): + response = chaosweb_server.fetch_page("/articles/test") + if response.status_code == 200: + success += 1 + assert success > 0 # At least some succeed +``` + +### Available Marker Kwargs + +The `chaosweb` marker accepts these keyword arguments: + +| Kwarg | Type | Description | +|---|---|---| +| `preset` | str | Base preset name | +| `rate_limit_pct` | float | 429 Rate Limit percentage | +| `forbidden_pct` | float | 403 Forbidden percentage | +| `not_found_pct` | float | 404 Not Found percentage | +| `gone_pct` | float | 410 Gone percentage | +| `payment_required_pct` | float | 402 Payment Required percentage | +| `unavailable_for_legal_pct` | float | 451 Unavailable for Legal Reasons percentage | +| `service_unavailable_pct` | float | 503 Service Unavailable percentage | +| `bad_gateway_pct` | float | 502 Bad Gateway percentage | +| `gateway_timeout_pct` | float | 504 Gateway Timeout percentage | +| `internal_error_pct` | float | 500 Internal Server Error percentage | +| `timeout_pct` | float | Timeout percentage | +| `connection_reset_pct` | float | Connection reset percentage | +| `connection_stall_pct` | float | Connection stall percentage | +| `slow_response_pct` | float | Slow response percentage | +| `incomplete_response_pct` | float | Incomplete response percentage | +| `wrong_content_type_pct` | float | Wrong Content-Type percentage | +| `encoding_mismatch_pct` | float | Encoding mismatch percentage | +| `truncated_html_pct` | float | Truncated HTML percentage | +| `invalid_encoding_pct` | float | Invalid encoding percentage | +| `charset_confusion_pct` | float | Charset confusion percentage | +| `malformed_meta_pct` | float | Malformed meta tag percentage | +| `redirect_loop_pct` | float | Redirect loop percentage | +| `ssrf_redirect_pct` | float | SSRF redirect percentage | +| `selection_mode` | str | `priority` or `weighted` | +| `base_ms` | int | Base latency in milliseconds | +| `jitter_ms` | int | Latency jitter in milliseconds | +| `content_mode` | str | Content mode (`random`, `template`, `echo`, `preset`) | + +### Fixture Helpers + +The `ChaosWebFixture` object provides: + +| Method/Property | Description | +|---|---| +| `fetch_page(path="/", headers=..., follow_redirects=False)` | GET a page | +| `get_stats()` | Get metrics summary | +| `export_metrics()` | Export raw metrics data | +| `update_config(rate_limit_pct=..., ...)` | Update config at runtime | +| `reset()` | Reset metrics and start new run | +| `wait_for_requests(count, timeout=10.0)` | Block until N requests recorded | +| `run_id` | Current run ID | +| `base_url` | Base URL (`http://testserver`) | +| `admin_headers` | Dict with auth headers for admin endpoints | + +## Complete Working Examples + +### ChaosLLM: Testing Error Recovery + +```python +import pytest + +@pytest.mark.chaosllm(rate_limit_pct=100.0) +def test_all_requests_rate_limited(chaosllm_server): + """Verify behavior when every request is rate limited.""" + response = chaosllm_server.post_completion() + assert response.status_code == 429 + assert "Retry-After" in response.headers + data = response.json() + assert data["error"]["type"] == "rate_limit_error" + + +@pytest.mark.chaosllm(invalid_json_pct=100.0) +def test_malformed_json_handling(chaosllm_server): + """Verify the client can detect invalid JSON responses.""" + response = chaosllm_server.post_completion() + assert response.status_code == 200 # Malformed responses return 200 + # The body is not valid JSON + try: + response.json() + assert False, "Expected JSON decode error" + except Exception: + pass # Expected + + +def test_runtime_config_update(chaosllm_server): + """Verify runtime config updates take effect.""" + # Start with no errors + response = chaosllm_server.post_completion() + assert response.status_code == 200 + + # Enable 100% rate limiting + chaosllm_server.update_config(rate_limit_pct=100.0) + + response = chaosllm_server.post_completion() + assert response.status_code == 429 + + # Check metrics + stats = chaosllm_server.get_stats() + assert stats["total_requests"] == 2 + + +def test_metrics_tracking(chaosllm_server): + """Verify metrics are recorded for each request.""" + for _ in range(10): + chaosllm_server.post_completion() + + chaosllm_server.wait_for_requests(10) + stats = chaosllm_server.get_stats() + assert stats["total_requests"] == 10 + assert "latency_stats" in stats +``` + +### ChaosWeb: Testing Scraper Resilience + +```python +import pytest + +@pytest.mark.chaosweb(forbidden_pct=100.0) +def test_all_requests_blocked(chaosweb_server): + """Verify behavior when bot detection blocks everything.""" + response = chaosweb_server.fetch_page("/articles/test") + assert response.status_code == 403 + + +@pytest.mark.chaosweb(encoding_mismatch_pct=100.0) +def test_encoding_mismatch_detection(chaosweb_server): + """Verify the scraper detects encoding mismatches.""" + response = chaosweb_server.fetch_page("/articles/test") + assert response.status_code == 200 + # Header says UTF-8 but body is ISO-8859-1 + assert "utf-8" in response.headers.get("content-type", "").lower() + + +@pytest.mark.chaosweb(redirect_loop_pct=100.0) +def test_redirect_loop_handling(chaosweb_server): + """Verify the scraper detects redirect loops.""" + response = chaosweb_server.fetch_page("/articles/test") + assert response.status_code == 301 + assert "Location" in response.headers + # Without follow_redirects, you get the first redirect + assert "/redirect?" in response.headers["Location"] + + +@pytest.mark.chaosweb(preset="realistic") +def test_realistic_scraping(chaosweb_server): + """Test with realistic error distribution.""" + results = {"success": 0, "error": 0} + for _ in range(100): + response = chaosweb_server.fetch_page("/articles/test") + if response.status_code == 200: + results["success"] += 1 + else: + results["error"] += 1 + # Realistic preset: ~80% success rate + assert results["success"] > 50 +``` + +## How It Works + +The fixtures use Starlette's `TestClient`, which wraps the ASGI application and routes HTTP calls through the stack without opening a network socket. This means: + +- **No port conflicts** -- multiple tests can run in parallel +- **No startup delay** -- the server is ready immediately +- **Full fidelity** -- the same request handling code runs as in production +- **Isolated state** -- each test gets a fresh server instance via `tmp_path` + +The `_build_config_from_marker()` function translates marker kwargs into a `ChaosLLMConfig` or `ChaosWebConfig` object. It applies the same precedence rules as the CLI: marker kwargs override the preset, and the fixture always forces latency to zero and sets a deterministic admin token for test convenience. + +## Related Pages + +- [ChaosLLM](chaosllm.md) -- Error types and response modes +- [ChaosWeb](chaosweb.md) -- Error types and content modes +- [Metrics](metrics.md) -- Understanding metrics data in tests +- [Configuration](configuration.md) -- How configuration precedence works diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..00fa4fb --- /dev/null +++ b/docs/index.md @@ -0,0 +1,73 @@ +# errorworks + +**Composable chaos-testing services for LLM and web scraping pipelines.** + +errorworks gives you drop-in fake servers that inject faults, simulate latency, +generate realistic responses, and record metrics -- so you can test how your +client code behaves when things go wrong, before they go wrong in production. + +## Get started in 60 seconds + +```bash +# Install +pip install errorworks + +# Start a fake OpenAI-compatible server +chaosllm serve --preset=realistic + +# In another terminal, make a request +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer test-key" \ + -d '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}] + }' +``` + +Some requests return `200 OK` with generated content. Others return `429`, +`503`, or malformed responses -- exactly what happens in the real world. + +## What's included + +### ChaosLLM + +A fake OpenAI-compatible API server. Point your OpenAI client at it and test how +your code handles rate limits (429), server errors (503/500), connection +timeouts, truncated streams, invalid JSON, and more. + +### ChaosWeb + +A fake web server for scraping resilience tests. Serves HTML pages that +intermittently break with encoding mismatches, truncated content, SSRF redirects, +and other real-world failure modes that trip up web scrapers. + +## Next steps + +
+ +- **Installation** + + Set up errorworks with pip or uv, including development installs. + + [:octicons-arrow-right-24: Installation](getting-started/installation.md) + +- **Quick Start** + + Walk through a complete scenario with ChaosLLM, ChaosWeb, and pytest fixtures. + + [:octicons-arrow-right-24: Quick Start](getting-started/quickstart.md) + +- **Guide** + + Deep dives into presets, configuration, metrics, and testing fixtures. + + [:octicons-arrow-right-24: Guide](guide/chaosllm.md) + +- **Reference** + + CLI commands, HTTP API endpoints, and configuration schema. + + [:octicons-arrow-right-24: Reference](reference/cli.md) + +
diff --git a/docs/reference/api.md b/docs/reference/api.md new file mode 100644 index 0000000..222be91 --- /dev/null +++ b/docs/reference/api.md @@ -0,0 +1,440 @@ +# HTTP API Reference + +Both ChaosLLM and ChaosWeb are Starlette ASGI applications. This page documents every HTTP endpoint exposed by each server. + +## Authentication + +All `/admin/*` endpoints require a Bearer token in the `Authorization` header: + +``` +Authorization: Bearer +``` + +The `admin_token` is auto-generated at startup (printed to the console) or set explicitly via config. Requests without a valid token receive: + +- **401** if the `Authorization: Bearer` header is missing +- **403** if the token does not match + +--- + +## ChaosLLM Endpoints + +### `POST /v1/chat/completions` + +OpenAI-compatible chat completions endpoint. This is the primary endpoint for LLM chaos testing. + +**Request body** (OpenAI format): + +```json +{ + "model": "gpt-4", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ], + "stream": false +} +``` + +**Optional request headers:** + +| Header | Description | +|--------|-------------| +| `X-Fake-Response-Mode` | Override response generation mode (`random`, `template`, `echo`, `preset`). Only honored when `allow_header_overrides` is `true` in config. | +| `X-Fake-Template` | Override template string for template mode. | + +**Success response** (200): + +```json +{ + "id": "chatcmpl-", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Generated response text..." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 25, + "completion_tokens": 42, + "total_tokens": 67 + } +} +``` + +**Error responses** vary by injection type: + +| Injection | Status | Response | +|-----------|--------|----------| +| `rate_limit` | 429 | JSON error with `Retry-After` header | +| `capacity_529` | 529 | JSON error body | +| `service_unavailable` | 503 | JSON error body | +| `bad_gateway` | 502 | JSON error body | +| `gateway_timeout` | 504 | JSON error body | +| `internal_error` | 500 | JSON error body | +| `timeout` | 504 or connection drop | Delays, then responds or disconnects | +| `connection_reset` | N/A | TCP connection reset | +| `connection_stall` | N/A | Stalls, then connection reset | +| `invalid_json` | 200 | Unparseable JSON body | +| `truncated` | 200 | Truncated JSON mid-stream | +| `empty_body` | 200 | Empty response body | +| `missing_fields` | 200 | Valid JSON missing required fields | +| `wrong_content_type` | 200 | HTML body with `text/html` content type | + +**Example curl:** + +```bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}] + }' +``` + +### `POST /openai/deployments/{deployment}/chat/completions` + +Azure OpenAI-compatible endpoint. Same request/response format as `/v1/chat/completions` with an additional `api-version` query parameter. + +**Example curl:** + +```bash +curl -X POST "http://localhost:8000/openai/deployments/my-gpt4/chat/completions?api-version=2024-02-01" \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [{"role": "user", "content": "Hello"}] + }' +``` + +--- + +## ChaosWeb Endpoints + +### `GET /{path}` + +Catch-all content endpoint. Returns HTML pages with configurable error injection, content malformations, and redirect behavior. + +**Optional request headers:** + +| Header | Description | +|--------|-------------| +| `X-Fake-Content-Mode` | Override content generation mode (`random`, `template`, `echo`, `preset`). Only honored when `allow_header_overrides` is `true` in config. | + +**Success response** (200): HTML page with `text/html; charset=utf-8` content type. + +**Error responses** include all ChaosLLM categories plus: + +| Injection | Status | Response | +|-----------|--------|----------| +| `forbidden` | 403 | HTML error page | +| `not_found` | 404 | HTML error page | +| `gone` | 410 | HTML error page | +| `payment_required` | 402 | HTML error page | +| `unavailable_for_legal` | 451 | HTML error page | +| `redirect_loop` | 301 | Chain of 301 redirects via `/redirect` | +| `ssrf_redirect` | 301 | Redirect to private IP / cloud metadata | +| `incomplete_response` | 200 | Partial body then connection drop | +| `encoding_mismatch` | 200 | UTF-8 header with ISO-8859-1 body | +| `truncated_html` | 200 | HTML cut off mid-tag | +| `invalid_encoding` | 200 | Non-decodable bytes in declared encoding | +| `charset_confusion` | 200 | Conflicting charset in header vs meta tag | +| `malformed_meta` | 200 | Invalid `` directives | + +**Example curl:** + +```bash +curl http://localhost:8200/some/page +``` + +### `GET /redirect` + +Internal redirect hop handler used by redirect loop injection. Tracks hops via query parameters (`hop`, `max`, `target`). Not intended for direct use. + +--- + +## Shared Endpoints + +These endpoints are available on both ChaosLLM and ChaosWeb servers. + +### `GET /health` + +Health check endpoint. No authentication required. + +**Response** (200): + +```json +{ + "status": "healthy", + "run_id": "550e8400-e29b-41d4-a716-446655440000", + "started_utc": "2025-01-15T10:30:00+00:00", + "in_burst": false +} +``` + +**Example curl:** + +```bash +curl http://localhost:8000/health +``` + +--- + +### `GET /admin/stats` + +Returns summary statistics for the current run. + +**Response** (200): + +```json +{ + "run_id": "550e8400-e29b-41d4-a716-446655440000", + "started_utc": "2025-01-15T10:30:00+00:00", + "total_requests": 1500, + "requests_by_outcome": { + "success": 1200, + "error_injected": 280, + "error_malformed": 20 + }, + "error_rate": 20.0, + "requests_by_status_code": { + "200": 1220, + "429": 150, + "503": 80, + "500": 50 + }, + "latency_stats": { + "avg_ms": 65.3, + "p50_ms": 52.1, + "p95_ms": 112.7, + "p99_ms": 198.4, + "max_ms": 350.2 + } +} +``` + +**Example curl:** + +```bash +curl -H "Authorization: Bearer $ADMIN_TOKEN" \ + http://localhost:8000/admin/stats +``` + +--- + +### `GET /admin/config` + +Returns the current runtime configuration (error injection, response/content, and latency settings). + +**Response** (200): + +ChaosLLM example: + +```json +{ + "error_injection": { + "rate_limit_pct": 5.0, + "capacity_529_pct": 0.0, + "service_unavailable_pct": 2.0, + "selection_mode": "priority", + "burst": { + "enabled": false, + "interval_sec": 30, + "duration_sec": 5 + } + }, + "response": { + "mode": "random", + "allow_header_overrides": true, + "random": { + "min_words": 10, + "max_words": 100, + "vocabulary": "english" + } + }, + "latency": { + "base_ms": 50, + "jitter_ms": 30 + } +} +``` + +**Example curl:** + +```bash +curl -H "Authorization: Bearer $ADMIN_TOKEN" \ + http://localhost:8000/admin/config +``` + +--- + +### `POST /admin/config` + +Update runtime configuration. Accepts a partial JSON body that is deep-merged with the current configuration. Only the sections you include are modified; omitted sections retain their current values. + +Nested fields within a section are also deep-merged. For example, sending `{"error_injection": {"burst": {"enabled": true}}}` enables burst mode without resetting `interval_sec` or `duration_sec` to defaults. + +After merging, the new configuration is validated through the Pydantic model. If validation fails, a 422 response is returned and no changes are applied. + +**Request body:** + +```json +{ + "error_injection": { + "rate_limit_pct": 25.0, + "burst": { + "enabled": true + } + }, + "latency": { + "base_ms": 100 + } +} +``` + +**Response** (200): + +```json +{ + "status": "updated", + "config": { + "error_injection": { "..." : "full merged config" }, + "response": { "..." : "unchanged" }, + "latency": { "base_ms": 100, "jitter_ms": 30 } + } +} +``` + +**Error responses:** + +| Status | Condition | +|--------|-----------| +| 400 | Request body is not valid JSON or not a JSON object | +| 401 | Missing `Authorization: Bearer` header | +| 403 | Invalid admin token | +| 422 | Merged config fails Pydantic validation | + +**Example curl:** + +```bash +curl -X POST http://localhost:8000/admin/config \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"error_injection": {"rate_limit_pct": 25.0}}' +``` + +--- + +### `POST /admin/reset` + +Reset all metrics and start a new run. Clears the `requests` and `timeseries` tables and generates a new `run_id`. The error injection engine's burst state is also reset. + +**Response** (200): + +```json +{ + "status": "reset", + "new_run_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890" +} +``` + +**Example curl:** + +```bash +curl -X POST http://localhost:8000/admin/reset \ + -H "Authorization: Bearer $ADMIN_TOKEN" +``` + +--- + +### `GET /admin/export` + +Export all raw metrics data for external analysis or archival. Returns the complete request log, timeseries data, and the configuration used for this run. + +**Response** (200): + +```json +{ + "run_id": "550e8400-e29b-41d4-a716-446655440000", + "started_utc": "2025-01-15T10:30:00+00:00", + "requests": [ + { + "request_id": "abc-123", + "timestamp_utc": "2025-01-15T10:30:01+00:00", + "endpoint": "/v1/chat/completions", + "outcome": "success", + "status_code": 200, + "latency_ms": 52.3 + } + ], + "timeseries": [ + { + "bucket_utc": "2025-01-15T10:30:00+00:00", + "requests_total": 45, + "avg_latency_ms": 65.3, + "p99_latency_ms": 198.4 + } + ], + "config": { + "server": { "host": "127.0.0.1", "port": 8000, "workers": 4 }, + "metrics": { "database": "file:chaosllm-metrics?mode=memory&cache=shared" }, + "error_injection": { "..." : "..." }, + "response": { "..." : "..." }, + "latency": { "base_ms": 50, "jitter_ms": 30 } + } +} +``` + +**Example curl:** + +```bash +curl -H "Authorization: Bearer $ADMIN_TOKEN" \ + http://localhost:8000/admin/export +``` + +--- + +## Error Response Format + +### ChaosLLM Error Bodies + +All HTTP-level error injections return OpenAI-compatible error JSON: + +```json +{ + "error": { + "type": "rate_limit_error", + "message": "Rate limit exceeded. Please retry after the specified time.", + "code": "rate_limit" + } +} +``` + +The `type` field maps to OpenAI error types: `rate_limit_error`, `capacity_error`, `server_error`, `permission_error`, `not_found_error`. + +### ChaosWeb Error Bodies + +HTTP-level errors return HTML error pages: + +```html +

429 Too Many Requests -- You are being rate limited.

+``` + +### Admin Error Bodies + +Authentication and validation errors from admin endpoints use a consistent format: + +```json +{ + "error": { + "type": "authentication_error", + "message": "Missing Authorization: Bearer header" + } +} +``` diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 0000000..21407dc --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,243 @@ +# CLI Reference + +Errorworks provides three CLI entry points, all built with [Typer](https://typer.tiangolo.com/). + +## `chaosengine` -- Unified CLI + +The `chaosengine` command aggregates both ChaosLLM and ChaosWeb under a single entry point. It mounts the same Typer apps as subcommands, so all flags are identical to the standalone commands. + +```bash +chaosengine llm serve --preset=gentle +chaosengine llm presets +chaosengine web serve --preset=stress_scraping +chaosengine web presets +``` + +The standalone entry points (`chaosllm`, `chaosweb`) continue to work unchanged. + +--- + +## `chaosllm` -- ChaosLLM Server + +### `chaosllm serve` + +Start the ChaosLLM fake LLM server with OpenAI and Azure OpenAI compatible endpoints. + +**Configuration precedence** (highest to lowest): + +1. Command-line flags +2. Config file (`--config`) +3. Preset (`--preset`) +4. Built-in defaults + +#### Configuration Source Flags + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--preset` | `-p` | `None` | Preset configuration to use. Use `chaosllm presets` to list available presets. | +| `--config` | `-c` | `None` | Path to YAML configuration file. | + +#### Server Binding Flags + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--host` | `-h` | `127.0.0.1` | Host address to bind to. | +| `--port` | `-P` | `8000` | Port to listen on (1-65535). | +| `--workers` | `-w` | `4` (or from preset) | Number of uvicorn workers. | + +#### Metrics Flags + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--database` | `-d` | In-memory SQLite | SQLite database path for metrics storage. | + +#### Error Injection Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--rate-limit-pct` | `0.0` | 429 Rate Limit error percentage (0-100). | +| `--capacity-529-pct` | `0.0` | 529 Capacity error percentage (0-100). | +| `--service-unavailable-pct` | `0.0` | 503 Service Unavailable error percentage (0-100). | +| `--internal-error-pct` | `0.0` | 500 Internal Error percentage (0-100). | +| `--timeout-pct` | `0.0` | Connection timeout percentage (0-100). | +| `--selection-mode` | `priority` | Error selection strategy: `priority` or `weighted`. | + +#### Latency Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--base-ms` | `50` | Base latency in milliseconds. | +| `--jitter-ms` | `30` | Latency jitter in milliseconds (+/-). | + +#### Response Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--response-mode` | `random` | Response generation mode: `random`, `template`, `echo`, `preset`. | + +#### Burst Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--burst-enabled` / `--no-burst` | `False` | Enable burst pattern injection. | +| `--burst-interval-sec` | `30` | Time between burst starts in seconds. | +| `--burst-duration-sec` | `5` | How long each burst lasts in seconds. | + +#### Misc Flags + +| Flag | Short | Description | +|------|-------|-------------| +| `--version` | `-v` | Show version and exit. | + +**Examples:** + +```bash +# Start with defaults +chaosllm serve + +# Use a preset +chaosllm serve --preset=stress_aimd + +# Custom error rates +chaosllm serve --rate-limit-pct=10 --capacity-529-pct=5 + +# Custom port and database +chaosllm serve --port=9000 --database=./my-metrics.db +``` + +### `chaosllm presets` + +List available preset configurations. + +```bash +chaosllm presets +``` + +### `chaosllm show-config` + +Display the effective (merged) configuration. + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--preset` | `-p` | `None` | Preset to show configuration for. | +| `--config` | `-c` | `None` | Config file to show. | +| `--format` | `-f` | `yaml` | Output format: `json` or `yaml`. | + +```bash +chaosllm show-config --preset=stress_aimd --format=json +``` + +--- + +## `chaosweb` -- ChaosWeb Server + +### `chaosweb serve` + +Start the ChaosWeb fake web server for scraping pipeline resilience testing. Serves HTML pages with configurable error injection, content malformations, redirect loops, and SSRF injection. + +**Configuration precedence** is identical to ChaosLLM. + +#### Configuration Source Flags + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--preset` | `-p` | `None` | Preset configuration to use. Use `chaosweb presets` to list available. | +| `--config` | `-c` | `None` | Path to YAML configuration file. | + +#### Server Binding Flags + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--host` | `-h` | `127.0.0.1` | Host address to bind to. | +| `--port` | `-P` | `8200` | Port to listen on (1-65535). | +| `--workers` | `-w` | `4` (or from preset) | Number of uvicorn workers. | + +#### Metrics Flags + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--database` | `-d` | In-memory SQLite | SQLite database path for metrics. | + +#### Error Injection Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--rate-limit-pct` | `0.0` | 429 Rate Limit error percentage (0-100). | +| `--forbidden-pct` | `0.0` | 403 Forbidden error percentage (0-100). | +| `--not-found-pct` | `0.0` | 404 Not Found error percentage (0-100). | +| `--service-unavailable-pct` | `0.0` | 503 Service Unavailable percentage (0-100). | +| `--internal-error-pct` | `0.0` | 500 Internal Error percentage (0-100). | +| `--timeout-pct` | `0.0` | Connection timeout percentage (0-100). | +| `--ssrf-redirect-pct` | `0.0` | SSRF redirect injection percentage (0-100). | +| `--selection-mode` | `priority` | Error selection: `priority` or `weighted`. | + +#### Latency Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--base-ms` | `50` | Base latency in milliseconds. | +| `--jitter-ms` | `30` | Latency jitter in milliseconds (+/-). | + +#### Content Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--content-mode` | `random` | Content generation: `random`, `template`, `echo`, `preset`. | + +#### Burst Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--burst-enabled` / `--no-burst` | `False` | Enable burst pattern injection. | +| `--burst-interval-sec` | `30` | Time between burst starts. | +| `--burst-duration-sec` | `5` | Burst duration in seconds. | + +#### Misc Flags + +| Flag | Short | Description | +|------|-------|-------------| +| `--version` | `-v` | Show version. | + +**Examples:** + +```bash +chaosweb serve +chaosweb serve --preset=stress_scraping +chaosweb serve --rate-limit-pct=10 --forbidden-pct=5 +chaosweb serve --port=9000 --database=./web-metrics.db +``` + +### `chaosweb presets` + +List available preset configurations. + +### `chaosweb show-config` + +Display the effective (merged) configuration. + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--preset` | `-p` | `None` | Preset to show configuration for. | +| `--config` | `-c` | `None` | Config file to show. | +| `--format` | `-f` | `yaml` | Output format: `json` or `yaml`. | + +--- + +## `chaosllm-mcp` -- MCP Metrics Server + +Start the ChaosLLM MCP (Model Context Protocol) server for metrics analysis. Provides Claude-optimized tools for analyzing ChaosLLM metrics and investigating error patterns. + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--database` | `-d` | Auto-discovered | SQLite database path. If not specified, searches for `chaosllm-metrics.db` in the current directory and `./runs/`. | +| `--version` | `-v` | | Show version and exit. | + +```bash +# Auto-discover database +chaosllm-mcp + +# Explicit database path +chaosllm-mcp --database=./my-metrics.db +``` + +If no database is found and none is specified, the command exits with an error suggesting you run `chaosllm serve` first to create one. diff --git a/docs/reference/config-schema.md b/docs/reference/config-schema.md new file mode 100644 index 0000000..5d3173b --- /dev/null +++ b/docs/reference/config-schema.md @@ -0,0 +1,322 @@ +# Configuration Schema Reference + +All configuration models use Pydantic with `frozen=True` (immutable) and `extra="forbid"` (no unknown fields). Runtime updates go through the admin API's `POST /admin/config` endpoint, which creates new model instances rather than mutating existing ones. + +## Shared Configuration + +These models are defined in `errorworks.engine.types` and used by both ChaosLLM and ChaosWeb. + +### ServerConfig + +Server binding and worker configuration. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `host` | `str` | `"127.0.0.1"` | Host address to bind to. Must match pattern `^[a-zA-Z0-9.:\[\]-]+$`. | +| `port` | `int` | `8000` (LLM) / `8200` (Web) | Port to listen on. Range: 1-65535. | +| `workers` | `int` | `4` | Number of uvicorn workers. Must be > 0. | +| `admin_token` | `str` | Auto-generated | Bearer token for `/admin/*` endpoints. Auto-generated via `secrets.token_urlsafe(32)` if not set. | + +**Safety constraint:** Binding to `0.0.0.0`, `::`, or `0:0:0:0:0:0:0:0` is blocked by default. Set `allow_external_bind: true` in the top-level config to override. + +### MetricsConfig + +Metrics storage configuration. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `database` | `str` | In-memory SQLite URI | SQLite database path. Use a file path for persistent storage or a `file:...?mode=memory&cache=shared` URI for in-memory. | +| `timeseries_bucket_sec` | `int` | `1` | Time-series aggregation bucket size in seconds. Must be > 0. | + +Default database URIs: + +- ChaosLLM: `file:chaosllm-metrics?mode=memory&cache=shared` +- ChaosWeb: `file:chaosweb-metrics?mode=memory&cache=shared` + +### LatencyConfig + +Latency simulation configuration. The simulated delay is `(base_ms + uniform(-jitter_ms, +jitter_ms)) / 1000` seconds, clamped to a minimum of 0. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `base_ms` | `int` | `50` | Base latency in milliseconds. Must be >= 0. | +| `jitter_ms` | `int` | `30` | Random jitter added to base latency (+/- ms). Must be >= 0. | + +--- + +## ChaosLLM Configuration + +### ChaosLLMConfig (top-level) + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `server` | `ServerConfig` | See above | Server binding configuration. | +| `metrics` | `MetricsConfig` | See above | Metrics storage configuration. | +| `response` | `ResponseConfig` | See below | Response generation configuration. | +| `latency` | `LatencyConfig` | See above | Latency simulation configuration. | +| `error_injection` | `ErrorInjectionConfig` | See below | Error injection configuration. | +| `preset_name` | `str \| None` | `None` | Preset name used to build this config (set automatically). | +| `allow_external_bind` | `bool` | `false` | Allow binding to all interfaces (`0.0.0.0`). | + +### ErrorInjectionConfig (ChaosLLM) + +All percentage fields are floats in the range 0.0-100.0. A value of `5.0` means 5% of requests. + +#### HTTP-Level Errors + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `rate_limit_pct` | `float` | `0.0` | 429 Rate Limit error percentage (primary AIMD trigger). | +| `capacity_529_pct` | `float` | `0.0` | 529 Model Overloaded error percentage (Azure-specific). | +| `service_unavailable_pct` | `float` | `0.0` | 503 Service Unavailable error percentage. | +| `bad_gateway_pct` | `float` | `0.0` | 502 Bad Gateway error percentage. | +| `gateway_timeout_pct` | `float` | `0.0` | 504 Gateway Timeout error percentage. | +| `internal_error_pct` | `float` | `0.0` | 500 Internal Server Error percentage. | +| `forbidden_pct` | `float` | `0.0` | 403 Forbidden error percentage. | +| `not_found_pct` | `float` | `0.0` | 404 Not Found error percentage. | + +#### Retry-After Header + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `retry_after_sec` | `tuple[int, int]` | `(1, 5)` | Retry-After header value range [min, max] seconds. | + +#### Connection-Level Failures + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `timeout_pct` | `float` | `0.0` | Percentage of requests that hang (trigger client timeout). | +| `timeout_sec` | `tuple[int, int]` | `(30, 60)` | How long to hang before responding [min, max] seconds. | +| `connection_failed_pct` | `float` | `0.0` | Percentage of requests that disconnect after a short lead time. | +| `connection_failed_lead_sec` | `tuple[int, int]` | `(2, 5)` | Lead time before disconnect [min, max] seconds. | +| `connection_stall_pct` | `float` | `0.0` | Percentage of requests that stall the connection then disconnect. | +| `connection_stall_start_sec` | `tuple[int, int]` | `(0, 2)` | Initial delay before stalling [min, max] seconds. | +| `connection_stall_sec` | `tuple[int, int]` | `(30, 60)` | How long to stall before disconnect [min, max] seconds. | +| `connection_reset_pct` | `float` | `0.0` | Percentage of requests that RST the TCP connection. | +| `slow_response_pct` | `float` | `0.0` | Percentage of requests with artificially slow responses. | +| `slow_response_sec` | `tuple[int, int]` | `(10, 30)` | Slow response delay range [min, max] seconds. | + +#### Malformed Response Errors + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `invalid_json_pct` | `float` | `0.0` | Percentage of responses with invalid JSON. | +| `truncated_pct` | `float` | `0.0` | Percentage of responses truncated mid-stream. | +| `empty_body_pct` | `float` | `0.0` | Percentage of responses with empty body. | +| `missing_fields_pct` | `float` | `0.0` | Percentage of responses missing required fields. | +| `wrong_content_type_pct` | `float` | `0.0` | Percentage of responses with wrong Content-Type header. | + +#### Selection Mode + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `selection_mode` | `"priority" \| "weighted"` | `"priority"` | `priority`: errors evaluated in order, first triggered wins. `weighted`: errors selected proportionally by weight. | + +**Validation:** In weighted mode, if total error percentages reach or exceed 100%, a warning is emitted (no successful responses will be generated). + +#### LLMBurstConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | `bool` | `false` | Enable burst pattern injection. | +| `interval_sec` | `int` | `30` | Time between burst starts in seconds. Must be > 0. | +| `duration_sec` | `int` | `5` | How long each burst lasts in seconds. Must be > 0 and < `interval_sec` when enabled. | +| `rate_limit_pct` | `float` | `80.0` | Rate limit percentage during burst (0-100). | +| `capacity_pct` | `float` | `50.0` | Capacity error (529) percentage during burst (0-100). | + +#### Range Field Constraints + +All `tuple[int, int]` range fields must satisfy `min <= max`. They accept both tuples and lists in YAML/JSON input (e.g., `[1, 5]` or `(1, 5)`). + +### ResponseConfig (ChaosLLM) + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `mode` | `"random" \| "template" \| "echo" \| "preset"` | `"random"` | Response generation mode. | +| `allow_header_overrides` | `bool` | `true` | Allow `X-Fake-Response-Mode` and `X-Fake-Template` headers to override response generation. | +| `max_template_length` | `int` | `10000` | Maximum length for template strings (config or header override). Must be > 0. | +| `random` | `RandomResponseConfig` | See below | Settings for random mode. | +| `template` | `TemplateResponseConfig` | See below | Settings for template mode. | +| `preset` | `PresetResponseConfig` | See below | Settings for preset mode. | + +#### RandomResponseConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `min_words` | `int` | `10` | Minimum words in generated response. Must be > 0 and <= `max_words`. | +| `max_words` | `int` | `100` | Maximum words in generated response. Must be > 0. | +| `vocabulary` | `"english" \| "lorem"` | `"english"` | Word source: common English words or Lorem Ipsum. | + +#### TemplateResponseConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `body` | `str` | `'{"result": "ok"}'` | Jinja2 template for response body. Rendered in a `SandboxedEnvironment`. | + +#### PresetResponseConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `file` | `str` | `"./responses.jsonl"` | Path to JSONL file with canned responses. | +| `selection` | `"random" \| "sequential"` | `"random"` | How to select responses from the bank. | + +--- + +## ChaosWeb Configuration + +### ChaosWebConfig (top-level) + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `server` | `ServerConfig` | Port defaults to `8200` | Server binding configuration. | +| `metrics` | `MetricsConfig` | See above | Metrics storage configuration. | +| `content` | `WebContentConfig` | See below | HTML content generation configuration. | +| `latency` | `LatencyConfig` | See above | Latency simulation configuration. | +| `error_injection` | `WebErrorInjectionConfig` | See below | Error injection configuration. | +| `allow_external_bind` | `bool` | `false` | Allow binding to all interfaces (`0.0.0.0`). | +| `preset_name` | `str \| None` | `None` | Preset name used to build this config (set automatically). | + +### WebErrorInjectionConfig (ChaosWeb) + +#### HTTP-Level Errors + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `rate_limit_pct` | `float` | `0.0` | 429 Rate Limit percentage (anti-scraping throttle). | +| `forbidden_pct` | `float` | `0.0` | 403 Forbidden percentage (bot detection). | +| `not_found_pct` | `float` | `0.0` | 404 Not Found percentage (deleted page). | +| `gone_pct` | `float` | `0.0` | 410 Gone percentage (permanent deletion). | +| `payment_required_pct` | `float` | `0.0` | 402 Payment Required percentage (quota exceeded). | +| `unavailable_for_legal_pct` | `float` | `0.0` | 451 Unavailable for Legal Reasons percentage (geo-blocking). | +| `service_unavailable_pct` | `float` | `0.0` | 503 Service Unavailable percentage (maintenance). | +| `bad_gateway_pct` | `float` | `0.0` | 502 Bad Gateway percentage. | +| `gateway_timeout_pct` | `float` | `0.0` | 504 Gateway Timeout percentage. | +| `internal_error_pct` | `float` | `0.0` | 500 Internal Server Error percentage. | + +#### Retry-After Header + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `retry_after_sec` | `tuple[int, int]` | `(1, 30)` | Retry-After header value range [min, max] seconds. | + +#### Connection-Level Failures + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `timeout_pct` | `float` | `0.0` | Percentage of requests that hang (trigger client timeout). | +| `timeout_sec` | `tuple[int, int]` | `(30, 60)` | How long to hang before responding [min, max] seconds. | +| `connection_reset_pct` | `float` | `0.0` | Percentage of requests that RST the TCP connection. | +| `connection_stall_pct` | `float` | `0.0` | Percentage of requests that stall then disconnect. | +| `connection_stall_start_sec` | `tuple[int, int]` | `(0, 2)` | Initial delay before stalling [min, max] seconds. | +| `connection_stall_sec` | `tuple[int, int]` | `(30, 60)` | How long to stall before disconnect [min, max] seconds. | +| `slow_response_pct` | `float` | `0.0` | Percentage of requests with artificially slow responses. | +| `slow_response_sec` | `tuple[int, int]` | `(3, 15)` | Slow response delay range [min, max] seconds. | +| `incomplete_response_pct` | `float` | `0.0` | Percentage of responses that disconnect mid-body. | +| `incomplete_response_bytes` | `tuple[int, int]` | `(100, 1000)` | How many bytes to send before disconnecting [min, max]. | + +#### Content Malformations + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `wrong_content_type_pct` | `float` | `0.0` | Percentage of responses with wrong Content-Type (e.g., `application/pdf`). | +| `encoding_mismatch_pct` | `float` | `0.0` | Percentage with UTF-8 header but ISO-8859-1 body. | +| `truncated_html_pct` | `float` | `0.0` | Percentage with HTML cut off mid-tag. | +| `invalid_encoding_pct` | `float` | `0.0` | Percentage with non-decodable bytes in declared encoding. | +| `charset_confusion_pct` | `float` | `0.0` | Percentage with conflicting charset declarations (header vs meta). | +| `malformed_meta_pct` | `float` | `0.0` | Percentage with invalid `` directives. | + +#### Redirect Injection + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `redirect_loop_pct` | `float` | `0.0` | Percentage of requests that enter redirect loops. | +| `max_redirect_loop_hops` | `int` | `10` | Maximum hops in a redirect loop before terminating. Minimum: 3. | +| `ssrf_redirect_pct` | `float` | `0.0` | Percentage of requests redirected to private IPs (SSRF testing). | + +#### Selection Mode + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `selection_mode` | `"priority" \| "weighted"` | `"priority"` | Error selection strategy. Same semantics as ChaosLLM. | + +#### WebBurstConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | `bool` | `false` | Enable burst pattern injection. | +| `interval_sec` | `int` | `30` | Time between burst starts in seconds. Must be > 0. | +| `duration_sec` | `int` | `5` | How long each burst lasts in seconds. Must be > 0 and < `interval_sec` when enabled. | +| `rate_limit_pct` | `float` | `80.0` | Rate limit (429) percentage during burst (0-100). | +| `forbidden_pct` | `float` | `50.0` | Forbidden (403) percentage during burst (0-100). | + +### WebContentConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `mode` | `"random" \| "template" \| "echo" \| "preset"` | `"random"` | Content generation mode. | +| `allow_header_overrides` | `bool` | `true` | Allow `X-Fake-Content-Mode` header to override content generation. | +| `max_template_length` | `int` | `10000` | Maximum length for template strings. Must be > 0. | +| `default_content_type` | `str` | `"text/html; charset=utf-8"` | Default Content-Type header for successful responses. | +| `random` | `RandomContentConfig` | See below | Settings for random mode. | +| `template` | `TemplateContentConfig` | See below | Settings for template mode. | +| `preset` | `PresetContentConfig` | See below | Settings for preset mode. | + +#### RandomContentConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `min_words` | `int` | `50` | Minimum words in generated HTML body. Must be > 0 and <= `max_words`. | +| `max_words` | `int` | `500` | Maximum words in generated HTML body. Must be > 0. | +| `vocabulary` | `"english" \| "lorem"` | `"english"` | Word source: common English words or Lorem Ipsum. | + +#### TemplateContentConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `body` | `str` | HTML template with `{{ path }}` and `{{ random_words() }}` | Jinja2 template for HTML response body. Rendered in a `SandboxedEnvironment`. | + +#### PresetContentConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `file` | `str` | `"./pages.jsonl"` | Path to JSONL file with HTML page snapshots. | +| `selection` | `"random" \| "sequential"` | `"random"` | How to select pages from the bank. | + +--- + +## YAML Configuration Example + +```yaml +server: + host: 127.0.0.1 + port: 8000 + workers: 4 + admin_token: my-secret-token + +metrics: + database: ./chaosllm-metrics.db + timeseries_bucket_sec: 1 + +error_injection: + rate_limit_pct: 5.0 + service_unavailable_pct: 2.0 + timeout_pct: 1.0 + selection_mode: priority + burst: + enabled: true + interval_sec: 60 + duration_sec: 10 + rate_limit_pct: 80.0 + capacity_pct: 50.0 + +latency: + base_ms: 50 + jitter_ms: 30 + +response: + mode: random + random: + min_words: 20 + max_words: 200 + vocabulary: english +``` diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..a174515 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,63 @@ +site_name: errorworks +site_description: Composable chaos-testing services for LLM and web scraping pipelines +site_url: https://johnm-dta.github.io/errorworks +repo_url: https://github.com/johnm-dta/errorworks +repo_name: johnm-dta/errorworks + +copyright: An open-source project by the Digital Transformation Agency. + +theme: + name: material + palette: + - scheme: default + primary: deep purple + accent: amber + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: deep purple + accent: amber + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.sections + - navigation.expand + - navigation.top + - content.code.copy + - search.suggest + - search.highlight + # logo: assets/logo.png # Uncomment when DTA branding is available + # favicon: assets/favicon.png # Uncomment when DTA branding is available + +nav: + - Home: index.md + - Getting Started: + - Installation: getting-started/installation.md + - Quick Start: getting-started/quickstart.md + - Guide: + - ChaosLLM: guide/chaosllm.md + - ChaosWeb: guide/chaosweb.md + - Presets: guide/presets.md + - Configuration: guide/configuration.md + - Metrics: guide/metrics.md + - Testing Fixtures: guide/testing-fixtures.md + - Reference: + - CLI: reference/cli.md + - HTTP API: reference/api.md + - Configuration Schema: reference/config-schema.md + - Architecture: architecture.md + - Changelog: changelog.md + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.tabbed: + alternate_style: true + - toc: + permalink: true From 068b595ab8323b7e1ce219d0daab5b4023b04acc Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 19:47:46 +1100 Subject: [PATCH 15/17] ci: add docs workflow for GitHub Pages deployment Builds MkDocs site and deploys to gh-pages branch on push to main. Triggers on changes to docs/, mkdocs.yml, or CHANGELOG.md. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/docs.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..e972a40 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,22 @@ +name: Docs + +on: + push: + branches: [main] + paths: + - "docs/**" + - "mkdocs.yml" + - "CHANGELOG.md" + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uv sync --extra docs + - run: cp CHANGELOG.md docs/changelog.md + - run: uv run mkdocs gh-deploy --force From 40a39d3b429deb23e01e05e240b24c9c34ff5629 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 20:08:10 +1100 Subject: [PATCH 16/17] docs: apply DTA brand theme to docs site Switch from generic Material palette to DTA brand colours (navy primary, green accent) with Inter/JetBrains Mono fonts, custom CSS for light/dark modes, and mermaid diagram support. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/stylesheets/extra.css | 176 +++++++++++++++++++++++++++++++++++++ mkdocs.yml | 30 +++++-- 2 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 docs/stylesheets/extra.css diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 0000000..a94d0b2 --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,176 @@ +/* ========================================================================== + DTA Brand Theme for MkDocs Material + Based on Digital Transformation Agency brand guidelines + ========================================================================== */ + +/* -------------------------------------------------------------------------- + DTA Brand Palette + -------------------------------------------------------------------------- + Signature green: #69EBA5 + Primary green: #29B480 + Dark green: #288264 + Primary navy: #1E3C50 + Light green: #C8EBD7 + Body text: #414141 + Secondary gold: #FAD673 + Secondary salmon: #FFC2B0 + Neutrals: #27292B, #70767B, #9FA4A7, #CACCCE, #F4F5F5 + -------------------------------------------------------------------------- */ + +/* -------------------------------------------------------------------------- + Light mode (default scheme) + -------------------------------------------------------------------------- */ +[data-md-color-scheme="default"] { + --md-primary-fg-color: #1E3C50; + --md-primary-fg-color--light: #2a5570; + --md-primary-fg-color--dark: #152d3d; + --md-primary-bg-color: #ffffff; + --md-primary-bg-color--light: #F4F5F5; + + --md-accent-fg-color: #29B480; + --md-accent-fg-color--transparent: rgba(41, 180, 128, 0.1); + --md-accent-bg-color: #ffffff; + + --md-default-fg-color: #414141; + --md-default-fg-color--light: #70767B; + --md-default-fg-color--lighter: #9FA4A7; + --md-default-fg-color--lightest: #CACCCE; + --md-default-bg-color: #ffffff; + --md-default-bg-color--light: #F4F5F5; + --md-default-bg-color--lighter: #F4F5F5; + --md-default-bg-color--lightest: #ffffff; + + --md-typeset-a-color: #29B480; + + --md-code-bg-color: #F4F5F5; + --md-code-fg-color: #27292B; +} + +/* -------------------------------------------------------------------------- + Dark mode (slate scheme) + -------------------------------------------------------------------------- */ +[data-md-color-scheme="slate"] { + --md-primary-fg-color: #1E3C50; + --md-primary-fg-color--light: #2a5570; + --md-primary-fg-color--dark: #152d3d; + --md-primary-bg-color: #ffffff; + --md-primary-bg-color--light: #ffffffb3; + + --md-accent-fg-color: #69EBA5; + --md-accent-fg-color--transparent: rgba(105, 235, 165, 0.1); + + --md-default-bg-color: #1a2631; + --md-default-bg-color--light: #1f3040; + --md-default-bg-color--lighter: #24384a; + --md-default-bg-color--lightest: #2a4155; + + --md-default-fg-color: #e0e4e7; + --md-default-fg-color--light: #9FA4A7; + --md-default-fg-color--lighter: #70767B; + --md-default-fg-color--lightest: #4a5258; + + --md-typeset-a-color: #69EBA5; + + --md-code-bg-color: #1f3040; + --md-code-fg-color: #C8EBD7; +} + +/* -------------------------------------------------------------------------- + Navigation tabs — use navy background, green active indicator + -------------------------------------------------------------------------- */ +.md-tabs { + background-color: #1E3C50; +} + +.md-tabs__link--active, +.md-tabs__link:hover { + color: #69EBA5; +} + +/* -------------------------------------------------------------------------- + Header search bar styling + -------------------------------------------------------------------------- */ +.md-search__input { + background-color: rgba(255, 255, 255, 0.12); +} + +[data-md-color-scheme="default"] .md-search__input::placeholder { + color: rgba(255, 255, 255, 0.6); +} + +/* -------------------------------------------------------------------------- + Content links — green with underline on hover + -------------------------------------------------------------------------- */ +.md-typeset a { + color: var(--md-typeset-a-color); +} + +.md-typeset a:hover { + color: #288264; + text-decoration: underline; +} + +[data-md-color-scheme="slate"] .md-typeset a:hover { + color: #C8EBD7; +} + +/* -------------------------------------------------------------------------- + Admonitions — DTA-flavoured accents + -------------------------------------------------------------------------- */ +.md-typeset .admonition.danger, +.md-typeset details.danger { + border-color: #d32f2f; +} + +.md-typeset .admonition.tip, +.md-typeset details.tip { + border-color: #29B480; +} + +.md-typeset .admonition.info, +.md-typeset details.info { + border-color: #1E3C50; +} + +/* -------------------------------------------------------------------------- + Footer — navy background + -------------------------------------------------------------------------- */ +.md-footer { + background-color: #1E3C50; +} + +.md-footer-meta { + background-color: #152d3d; +} + +/* -------------------------------------------------------------------------- + Navigation sidebar — subtle green active state + -------------------------------------------------------------------------- */ +.md-nav__link--active { + color: #29B480; + font-weight: 600; +} + +[data-md-color-scheme="slate"] .md-nav__link--active { + color: #69EBA5; +} + +/* -------------------------------------------------------------------------- + Code blocks — slightly tinted + -------------------------------------------------------------------------- */ +.md-typeset code { + border-radius: 3px; +} + +/* -------------------------------------------------------------------------- + Tables — DTA-styled header + -------------------------------------------------------------------------- */ +.md-typeset table:not([class]) th { + background-color: #1E3C50; + color: #ffffff; +} + +[data-md-color-scheme="slate"] .md-typeset table:not([class]) th { + background-color: #2a5570; + color: #e0e4e7; +} diff --git a/mkdocs.yml b/mkdocs.yml index a174515..691902b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,24 +10,31 @@ theme: name: material palette: - scheme: default - primary: deep purple - accent: amber + primary: custom + accent: custom toggle: icon: material/brightness-7 name: Switch to dark mode - scheme: slate - primary: deep purple - accent: amber + primary: custom + accent: custom toggle: icon: material/brightness-4 name: Switch to light mode + font: + text: Inter + code: JetBrains Mono features: - navigation.sections - navigation.expand - navigation.top - - content.code.copy + - navigation.footer - search.suggest - search.highlight + - content.tabs.link + - content.code.copy + icon: + repo: fontawesome/brands/github # logo: assets/logo.png # Uncomment when DTA branding is available # favicon: assets/favicon.png # Uncomment when DTA branding is available @@ -53,11 +60,22 @@ nav: markdown_extensions: - admonition - pymdownx.details - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.highlight: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.tabbed: alternate_style: true + - tables + - attr_list + - md_in_html + - footnotes - toc: permalink: true + +extra_css: + - stylesheets/extra.css From 28dac293899acaf069931e521192dbe2fa66ef48 Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Mon, 16 Mar 2026 20:37:30 +1100 Subject: [PATCH 17/17] fix: remove unused imports and lint fixes - Remove unused DEFAULT_MEMORY_DB imports from CLI modules - Remove trailing blank line in test_metrics_store.py - Update uv.lock for docs dependency group Co-Authored-By: Claude Opus 4.6 (1M context) --- src/errorworks/llm/cli.py | 1 - src/errorworks/web/cli.py | 1 - tests/unit/engine/test_metrics_store.py | 1 - uv.lock | 304 +++++++++++++++++++++++- 4 files changed, 303 insertions(+), 4 deletions(-) diff --git a/src/errorworks/llm/cli.py b/src/errorworks/llm/cli.py index 035d6ef..9120b78 100644 --- a/src/errorworks/llm/cli.py +++ b/src/errorworks/llm/cli.py @@ -31,7 +31,6 @@ import yaml from errorworks.llm.config import ( - DEFAULT_MEMORY_DB, list_presets, load_config, ) diff --git a/src/errorworks/web/cli.py b/src/errorworks/web/cli.py index 56dbac5..dfb3a5f 100644 --- a/src/errorworks/web/cli.py +++ b/src/errorworks/web/cli.py @@ -22,7 +22,6 @@ import yaml from errorworks.web.config import ( - DEFAULT_MEMORY_DB, list_presets, load_config, ) diff --git a/tests/unit/engine/test_metrics_store.py b/tests/unit/engine/test_metrics_store.py index 2e24e6e..770fec1 100644 --- a/tests/unit/engine/test_metrics_store.py +++ b/tests/unit/engine/test_metrics_store.py @@ -598,7 +598,6 @@ def test_get_timeseries_limit(self, store: MetricsStore) -> None: assert len(rows) == 3 - # ============================================================================= # Rebuild Timeseries # ============================================================================= diff --git a/uv.lock b/uv.lock index a1247cb..ad93d5f 100644 --- a/uv.lock +++ b/uv.lock @@ -42,6 +42,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, ] +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + +[[package]] +name = "backrefs" +version = "6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/a6/e325ec73b638d3ede4421b5445d4a0b8b219481826cc079d510100af356c/backrefs-6.2.tar.gz", hash = "sha256:f44ff4d48808b243b6c0cdc6231e22195c32f77046018141556c66f8bab72a49", size = 7012303, upload-time = "2026-02-16T19:10:15.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/39/3765df263e08a4df37f4f43cb5aa3c6c17a4bdd42ecfe841e04c26037171/backrefs-6.2-py310-none-any.whl", hash = "sha256:0fdc7b012420b6b144410342caeb8adc54c6866cf12064abc9bb211302e496f8", size = 381075, upload-time = "2026-02-16T19:10:04.322Z" }, + { url = "https://files.pythonhosted.org/packages/0f/f0/35240571e1b67ffb19dafb29ab34150b6f59f93f717b041082cdb1bfceb1/backrefs-6.2-py311-none-any.whl", hash = "sha256:08aa7fae530c6b2361d7bdcbda1a7c454e330cc9dbcd03f5c23205e430e5c3be", size = 392874, upload-time = "2026-02-16T19:10:06.314Z" }, + { url = "https://files.pythonhosted.org/packages/e3/63/77e8c9745b4d227cce9f5e0a6f68041278c5f9b18588b35905f5f19c1beb/backrefs-6.2-py312-none-any.whl", hash = "sha256:c3f4b9cb2af8cda0d87ab4f57800b57b95428488477be164dd2b47be54db0c90", size = 398787, upload-time = "2026-02-16T19:10:08.274Z" }, + { url = "https://files.pythonhosted.org/packages/c5/71/c754b1737ad99102e03fa3235acb6cb6d3ac9d6f596cbc3e5f236705abd8/backrefs-6.2-py313-none-any.whl", hash = "sha256:12df81596ab511f783b7d87c043ce26bc5b0288cf3bb03610fe76b8189282b2b", size = 400747, upload-time = "2026-02-16T19:10:09.791Z" }, + { url = "https://files.pythonhosted.org/packages/af/75/be12ba31a6eb20dccef2320cd8ccb3f7d9013b68ba4c70156259fee9e409/backrefs-6.2-py314-none-any.whl", hash = "sha256:e5f805ae09819caa1aa0623b4a83790e7028604aa2b8c73ba602c4454e665de7", size = 412602, upload-time = "2026-02-16T19:10:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/21/f8/d02f650c47d05034dcd6f9c8cf94f39598b7a89c00ecda0ecb2911bc27e9/backrefs-6.2-py39-none-any.whl", hash = "sha256:664e33cd88c6840b7625b826ecf2555f32d491800900f5a541f772c485f7cda7", size = 381077, upload-time = "2026-02-16T19:10:13.74Z" }, +] + [[package]] name = "certifi" version = "2026.2.25" @@ -117,6 +140,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, ] +[[package]] +name = "charset-normalizer" +version = "3.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/60/e3bec1881450851b087e301bedc3daa9377a4d45f1c26aa90b0b235e38aa/charset_normalizer-3.4.6.tar.gz", hash = "sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6", size = 143363, upload-time = "2026-03-15T18:53:25.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/62/c0815c992c9545347aeea7859b50dc9044d147e2e7278329c6e02ac9a616/charset_normalizer-3.4.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab", size = 295154, upload-time = "2026-03-15T18:50:50.88Z" }, + { url = "https://files.pythonhosted.org/packages/a8/37/bdca6613c2e3c58c7421891d80cc3efa1d32e882f7c4a7ee6039c3fc951a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21", size = 199191, upload-time = "2026-03-15T18:50:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/6c/92/9934d1bbd69f7f398b38c5dae1cbf9cc672e7c34a4adf7b17c0a9c17d15d/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2", size = 218674, upload-time = "2026-03-15T18:50:54.102Z" }, + { url = "https://files.pythonhosted.org/packages/af/90/25f6ab406659286be929fd89ab0e78e38aa183fc374e03aa3c12d730af8a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff", size = 215259, upload-time = "2026-03-15T18:50:55.616Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ef/79a463eb0fff7f96afa04c1d4c51f8fc85426f918db467854bfb6a569ce3/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5", size = 207276, upload-time = "2026-03-15T18:50:57.054Z" }, + { url = "https://files.pythonhosted.org/packages/f7/72/d0426afec4b71dc159fa6b4e68f868cd5a3ecd918fec5813a15d292a7d10/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0", size = 195161, upload-time = "2026-03-15T18:50:58.686Z" }, + { url = "https://files.pythonhosted.org/packages/bf/18/c82b06a68bfcb6ce55e508225d210c7e6a4ea122bfc0748892f3dc4e8e11/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a", size = 203452, upload-time = "2026-03-15T18:51:00.196Z" }, + { url = "https://files.pythonhosted.org/packages/44/d6/0c25979b92f8adafdbb946160348d8d44aa60ce99afdc27df524379875cb/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2", size = 202272, upload-time = "2026-03-15T18:51:01.703Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/7fea3e8fe84136bebbac715dd1221cc25c173c57a699c030ab9b8900cbb7/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5", size = 195622, upload-time = "2026-03-15T18:51:03.526Z" }, + { url = "https://files.pythonhosted.org/packages/57/8a/d6f7fd5cb96c58ef2f681424fbca01264461336d2a7fc875e4446b1f1346/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6", size = 220056, upload-time = "2026-03-15T18:51:05.269Z" }, + { url = "https://files.pythonhosted.org/packages/16/50/478cdda782c8c9c3fb5da3cc72dd7f331f031e7f1363a893cdd6ca0f8de0/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d", size = 203751, upload-time = "2026-03-15T18:51:06.858Z" }, + { url = "https://files.pythonhosted.org/packages/75/fc/cc2fcac943939c8e4d8791abfa139f685e5150cae9f94b60f12520feaa9b/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2", size = 216563, upload-time = "2026-03-15T18:51:08.564Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b7/a4add1d9a5f68f3d037261aecca83abdb0ab15960a3591d340e829b37298/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923", size = 209265, upload-time = "2026-03-15T18:51:10.312Z" }, + { url = "https://files.pythonhosted.org/packages/6c/18/c094561b5d64a24277707698e54b7f67bd17a4f857bbfbb1072bba07c8bf/charset_normalizer-3.4.6-cp312-cp312-win32.whl", hash = "sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4", size = 144229, upload-time = "2026-03-15T18:51:11.694Z" }, + { url = "https://files.pythonhosted.org/packages/ab/20/0567efb3a8fd481b8f34f739ebddc098ed062a59fed41a8d193a61939e8f/charset_normalizer-3.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb", size = 154277, upload-time = "2026-03-15T18:51:13.004Z" }, + { url = "https://files.pythonhosted.org/packages/15/57/28d79b44b51933119e21f65479d0864a8d5893e494cf5daab15df0247c17/charset_normalizer-3.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4", size = 142817, upload-time = "2026-03-15T18:51:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1d/4fdabeef4e231153b6ed7567602f3b68265ec4e5b76d6024cf647d43d981/charset_normalizer-3.4.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f", size = 294823, upload-time = "2026-03-15T18:51:15.755Z" }, + { url = "https://files.pythonhosted.org/packages/47/7b/20e809b89c69d37be748d98e84dce6820bf663cf19cf6b942c951a3e8f41/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843", size = 198527, upload-time = "2026-03-15T18:51:17.177Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/4f8d27527d59c039dce6f7622593cdcd3d70a8504d87d09eb11e9fdc6062/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf", size = 218388, upload-time = "2026-03-15T18:51:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/f6/9b/4770ccb3e491a9bacf1c46cc8b812214fe367c86a96353ccc6daf87b01ec/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8", size = 214563, upload-time = "2026-03-15T18:51:20.374Z" }, + { url = "https://files.pythonhosted.org/packages/2b/58/a199d245894b12db0b957d627516c78e055adc3a0d978bc7f65ddaf7c399/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9", size = 206587, upload-time = "2026-03-15T18:51:21.807Z" }, + { url = "https://files.pythonhosted.org/packages/7e/70/3def227f1ec56f5c69dfc8392b8bd63b11a18ca8178d9211d7cc5e5e4f27/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88", size = 194724, upload-time = "2026-03-15T18:51:23.508Z" }, + { url = "https://files.pythonhosted.org/packages/58/ab/9318352e220c05efd31c2779a23b50969dc94b985a2efa643ed9077bfca5/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84", size = 202956, upload-time = "2026-03-15T18:51:25.239Z" }, + { url = "https://files.pythonhosted.org/packages/75/13/f3550a3ac25b70f87ac98c40d3199a8503676c2f1620efbf8d42095cfc40/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd", size = 201923, upload-time = "2026-03-15T18:51:26.682Z" }, + { url = "https://files.pythonhosted.org/packages/1b/db/c5c643b912740b45e8eec21de1bbab8e7fc085944d37e1e709d3dcd9d72f/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c", size = 195366, upload-time = "2026-03-15T18:51:28.129Z" }, + { url = "https://files.pythonhosted.org/packages/5a/67/3b1c62744f9b2448443e0eb160d8b001c849ec3fef591e012eda6484787c/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194", size = 219752, upload-time = "2026-03-15T18:51:29.556Z" }, + { url = "https://files.pythonhosted.org/packages/f6/98/32ffbaf7f0366ffb0445930b87d103f6b406bc2c271563644bde8a2b1093/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc", size = 203296, upload-time = "2026-03-15T18:51:30.921Z" }, + { url = "https://files.pythonhosted.org/packages/41/12/5d308c1bbe60cabb0c5ef511574a647067e2a1f631bc8634fcafaccd8293/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f", size = 215956, upload-time = "2026-03-15T18:51:32.399Z" }, + { url = "https://files.pythonhosted.org/packages/53/e9/5f85f6c5e20669dbe56b165c67b0260547dea97dba7e187938833d791687/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2", size = 208652, upload-time = "2026-03-15T18:51:34.214Z" }, + { url = "https://files.pythonhosted.org/packages/f1/11/897052ea6af56df3eef3ca94edafee410ca699ca0c7b87960ad19932c55e/charset_normalizer-3.4.6-cp313-cp313-win32.whl", hash = "sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d", size = 143940, upload-time = "2026-03-15T18:51:36.15Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5c/724b6b363603e419829f561c854b87ed7c7e31231a7908708ac086cdf3e2/charset_normalizer-3.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389", size = 154101, upload-time = "2026-03-15T18:51:37.876Z" }, + { url = "https://files.pythonhosted.org/packages/01/a5/7abf15b4c0968e47020f9ca0935fb3274deb87cb288cd187cad92e8cdffd/charset_normalizer-3.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f", size = 143109, upload-time = "2026-03-15T18:51:39.565Z" }, + { url = "https://files.pythonhosted.org/packages/25/6f/ffe1e1259f384594063ea1869bfb6be5cdb8bc81020fc36c3636bc8302a1/charset_normalizer-3.4.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8", size = 294458, upload-time = "2026-03-15T18:51:41.134Z" }, + { url = "https://files.pythonhosted.org/packages/56/60/09bb6c13a8c1016c2ed5c6a6488e4ffef506461aa5161662bd7636936fb1/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421", size = 199277, upload-time = "2026-03-15T18:51:42.953Z" }, + { url = "https://files.pythonhosted.org/packages/00/50/dcfbb72a5138bbefdc3332e8d81a23494bf67998b4b100703fd15fa52d81/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2", size = 218758, upload-time = "2026-03-15T18:51:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/03/b3/d79a9a191bb75f5aa81f3aaaa387ef29ce7cb7a9e5074ba8ea095cc073c2/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30", size = 215299, upload-time = "2026-03-15T18:51:45.871Z" }, + { url = "https://files.pythonhosted.org/packages/76/7e/bc8911719f7084f72fd545f647601ea3532363927f807d296a8c88a62c0d/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db", size = 206811, upload-time = "2026-03-15T18:51:47.308Z" }, + { url = "https://files.pythonhosted.org/packages/e2/40/c430b969d41dda0c465aa36cc7c2c068afb67177bef50905ac371b28ccc7/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8", size = 193706, upload-time = "2026-03-15T18:51:48.849Z" }, + { url = "https://files.pythonhosted.org/packages/48/15/e35e0590af254f7df984de1323640ef375df5761f615b6225ba8deb9799a/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815", size = 202706, upload-time = "2026-03-15T18:51:50.257Z" }, + { url = "https://files.pythonhosted.org/packages/5e/bd/f736f7b9cc5e93a18b794a50346bb16fbfd6b37f99e8f306f7951d27c17c/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a", size = 202497, upload-time = "2026-03-15T18:51:52.012Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ba/2cc9e3e7dfdf7760a6ed8da7446d22536f3d0ce114ac63dee2a5a3599e62/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43", size = 193511, upload-time = "2026-03-15T18:51:53.723Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cb/5be49b5f776e5613be07298c80e1b02a2d900f7a7de807230595c85a8b2e/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0", size = 220133, upload-time = "2026-03-15T18:51:55.333Z" }, + { url = "https://files.pythonhosted.org/packages/83/43/99f1b5dad345accb322c80c7821071554f791a95ee50c1c90041c157ae99/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1", size = 203035, upload-time = "2026-03-15T18:51:56.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/9a/62c2cb6a531483b55dddff1a68b3d891a8b498f3ca555fbcf2978e804d9d/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f", size = 216321, upload-time = "2026-03-15T18:51:58.17Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/94a010ff81e3aec7c293eb82c28f930918e517bc144c9906a060844462eb/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815", size = 208973, upload-time = "2026-03-15T18:51:59.998Z" }, + { url = "https://files.pythonhosted.org/packages/2a/57/4ecff6d4ec8585342f0c71bc03efaa99cb7468f7c91a57b105bcd561cea8/charset_normalizer-3.4.6-cp314-cp314-win32.whl", hash = "sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d", size = 144610, upload-time = "2026-03-15T18:52:02.213Z" }, + { url = "https://files.pythonhosted.org/packages/80/94/8434a02d9d7f168c25767c64671fead8d599744a05d6a6c877144c754246/charset_normalizer-3.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f", size = 154962, upload-time = "2026-03-15T18:52:03.658Z" }, + { url = "https://files.pythonhosted.org/packages/46/4c/48f2cdbfd923026503dfd67ccea45c94fd8fe988d9056b468579c66ed62b/charset_normalizer-3.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e", size = 143595, upload-time = "2026-03-15T18:52:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/31/93/8878be7569f87b14f1d52032946131bcb6ebbd8af3e20446bc04053dc3f1/charset_normalizer-3.4.6-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866", size = 314828, upload-time = "2026-03-15T18:52:06.831Z" }, + { url = "https://files.pythonhosted.org/packages/06/b6/fae511ca98aac69ecc35cde828b0a3d146325dd03d99655ad38fc2cc3293/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc", size = 208138, upload-time = "2026-03-15T18:52:08.239Z" }, + { url = "https://files.pythonhosted.org/packages/54/57/64caf6e1bf07274a1e0b7c160a55ee9e8c9ec32c46846ce59b9c333f7008/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e", size = 224679, upload-time = "2026-03-15T18:52:10.043Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cb/9ff5a25b9273ef160861b41f6937f86fae18b0792fe0a8e75e06acb08f1d/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077", size = 223475, upload-time = "2026-03-15T18:52:11.854Z" }, + { url = "https://files.pythonhosted.org/packages/fc/97/440635fc093b8d7347502a377031f9605a1039c958f3cd18dcacffb37743/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f", size = 215230, upload-time = "2026-03-15T18:52:13.325Z" }, + { url = "https://files.pythonhosted.org/packages/cd/24/afff630feb571a13f07c8539fbb502d2ab494019492aaffc78ef41f1d1d0/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e", size = 199045, upload-time = "2026-03-15T18:52:14.752Z" }, + { url = "https://files.pythonhosted.org/packages/e5/17/d1399ecdaf7e0498c327433e7eefdd862b41236a7e484355b8e0e5ebd64b/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484", size = 211658, upload-time = "2026-03-15T18:52:16.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/38/16baa0affb957b3d880e5ac2144caf3f9d7de7bc4a91842e447fbb5e8b67/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7", size = 210769, upload-time = "2026-03-15T18:52:17.782Z" }, + { url = "https://files.pythonhosted.org/packages/05/34/c531bc6ac4c21da9ddfddb3107be2287188b3ea4b53b70fc58f2a77ac8d8/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff", size = 201328, upload-time = "2026-03-15T18:52:19.553Z" }, + { url = "https://files.pythonhosted.org/packages/fa/73/a5a1e9ca5f234519c1953608a03fe109c306b97fdfb25f09182babad51a7/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e", size = 225302, upload-time = "2026-03-15T18:52:21.043Z" }, + { url = "https://files.pythonhosted.org/packages/ba/f6/cd782923d112d296294dea4bcc7af5a7ae0f86ab79f8fefbda5526b6cfc0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659", size = 211127, upload-time = "2026-03-15T18:52:22.491Z" }, + { url = "https://files.pythonhosted.org/packages/0e/c5/0b6898950627af7d6103a449b22320372c24c6feda91aa24e201a478d161/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602", size = 222840, upload-time = "2026-03-15T18:52:24.113Z" }, + { url = "https://files.pythonhosted.org/packages/7d/25/c4bba773bef442cbdc06111d40daa3de5050a676fa26e85090fc54dd12f0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407", size = 216890, upload-time = "2026-03-15T18:52:25.541Z" }, + { url = "https://files.pythonhosted.org/packages/35/1a/05dacadb0978da72ee287b0143097db12f2e7e8d3ffc4647da07a383b0b7/charset_normalizer-3.4.6-cp314-cp314t-win32.whl", hash = "sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579", size = 155379, upload-time = "2026-03-15T18:52:27.05Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7a/d269d834cb3a76291651256f3b9a5945e81d0a49ab9f4a498964e83c0416/charset_normalizer-3.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4", size = 169043, upload-time = "2026-03-15T18:52:28.502Z" }, + { url = "https://files.pythonhosted.org/packages/23/06/28b29fba521a37a8932c6a84192175c34d49f84a6d4773fa63d05f9aff22/charset_normalizer-3.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c", size = 148523, upload-time = "2026-03-15T18:52:29.956Z" }, + { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" }, +] + [[package]] name = "click" version = "8.3.1" @@ -312,6 +408,9 @@ dev = [ { name = "ruff" }, { name = "types-pyyaml" }, ] +docs = [ + { name = "mkdocs-material" }, +] [package.metadata] requires-dist = [ @@ -319,6 +418,7 @@ requires-dist = [ { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.98,<7" }, { name = "jinja2", specifier = ">=3.1,<4" }, { name = "mcp", specifier = ">=1.0,<2" }, + { name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9,<10" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.19,<2" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.0,<5" }, { name = "pydantic", specifier = ">=2.12,<3" }, @@ -334,7 +434,7 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0,<7" }, { name = "uvicorn", specifier = ">=0.34,<1" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "docs"] [[package]] name = "filelock" @@ -345,6 +445,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/a5/842ae8f0c08b61d6484b52f99a03510a3a72d23141942d216ebe81fefbce/filelock-3.25.2-py3-none-any.whl", hash = "sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70", size = 26759, upload-time = "2026-03-11T20:45:37.437Z" }, ] +[[package]] +name = "ghp-import" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -529,6 +641,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", size = 48755, upload-time = "2026-02-17T16:12:47.943Z" }, ] +[[package]] +name = "markdown" +version = "3.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -638,6 +759,84 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mergedeep" +version = "1.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, +] + +[[package]] +name = "mkdocs" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "ghp-import" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mergedeep" }, + { name = "mkdocs-get-deps" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "watchdog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, +] + +[[package]] +name = "mkdocs-get-deps" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mergedeep" }, + { name = "platformdirs" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, +] + +[[package]] +name = "mkdocs-material" +version = "9.7.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel" }, + { name = "backrefs" }, + { name = "colorama" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "mkdocs" }, + { name = "mkdocs-material-extensions" }, + { name = "paginate" }, + { name = "pygments" }, + { name = "pymdown-extensions" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/76/5c202fecdc45d53e83e03a85bae70c48b6c81e9f87f0bc19a9e9c723bdc0/mkdocs_material-9.7.5.tar.gz", hash = "sha256:f76bdab532bad1d9c57ca7187b37eccf64dd12e1586909307f8856db3be384ea", size = 4097749, upload-time = "2026-03-10T15:43:22.809Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/e1/e8080dcfa95cca267662a6f4afe29237452bdeb5a2a6555ac83646d21915/mkdocs_material-9.7.5-py3-none-any.whl", hash = "sha256:7cf9df2ff121fd098ff6e05c732b0be3699afca9642e2dfe4926c40eb5873eec", size = 9305251, upload-time = "2026-03-10T15:43:19.089Z" }, +] + +[[package]] +name = "mkdocs-material-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847, upload-time = "2023-11-22T19:09:45.208Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" }, +] + [[package]] name = "mypy" version = "1.19.1" @@ -698,6 +897,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] +[[package]] +name = "paginate" +version = "0.5.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252, upload-time = "2024-08-25T14:17:24.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, +] + [[package]] name = "pathspec" version = "1.0.4" @@ -873,6 +1081,19 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pymdown-extensions" +version = "10.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/63/06673d1eb6d8f83c0ea1f677d770e12565fb516928b4109c9e2055656a9e/pymdown_extensions-10.21.tar.gz", hash = "sha256:39f4a020f40773f6b2ff31d2cd2546c2c04d0a6498c31d9c688d2be07e1767d5", size = 853363, upload-time = "2026-02-15T20:44:06.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/2c/5b079febdc65e1c3fb2729bf958d18b45be7113828528e8a0b5850dd819a/pymdown_extensions-10.21-py3-none-any.whl", hash = "sha256:91b879f9f864d49794c2d9534372b10150e6141096c3908a455e45ca72ad9d3f", size = 268877, upload-time = "2026-02-15T20:44:05.464Z" }, +] + [[package]] name = "pytest" version = "9.0.2" @@ -927,6 +1148,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + [[package]] name = "python-discovery" version = "1.1.3" @@ -1020,6 +1253,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "pyyaml-env-tag" +version = "1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -1034,6 +1279,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + [[package]] name = "rich" version = "14.3.3" @@ -1162,6 +1422,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -1251,6 +1520,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + [[package]] name = "uvicorn" version = "0.41.0" @@ -1278,3 +1556,27 @@ sdist = { url = "https://files.pythonhosted.org/packages/aa/92/58199fe10049f9703 wheels = [ { url = "https://files.pythonhosted.org/packages/c6/59/7d02447a55b2e55755011a647479041bc92a82e143f96a8195cb33bd0a1c/virtualenv-21.2.0-py3-none-any.whl", hash = "sha256:1bd755b504931164a5a496d217c014d098426cddc79363ad66ac78125f9d908f", size = 5825084, upload-time = "2026-03-09T17:24:35.378Z" }, ] + +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, + { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, + { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, + { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +]