diff --git a/.gitignore b/.gitignore index 68bc17f9..be5a7f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,9 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Harness execution logs +pytest_out.txt +ruff_out.txt +ruff_errors.json +ruff_errors_utf8.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..03f4dcc1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# Use a lightweight Python image +FROM python:3.11-alpine + +# Set the working directory +WORKDIR /usr/src/harness + +# Prevent Python from writing pyc files and keep stdout unbuffered +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# Install dependencies first (for optimal Docker layer caching) +COPY pyproject.toml HARNESS_README.md ./ +RUN pip install --no-cache-dir . + +# Copy your harness script and any local suite files +COPY annotation_harness.py . +COPY annotations/ annotations/ + +# Define the entry point +ENTRYPOINT ["python", "annotation_harness.py"] diff --git a/HARNESS_README.md b/HARNESS_README.md new file mode 100644 index 00000000..54772236 --- /dev/null +++ b/HARNESS_README.md @@ -0,0 +1,27 @@ +# Annotation Test Suite Harness + +**Objective:** A standalone test harness for the JSON Schema Annotation Test Suite, utilizing the `jschon` implementation natively. + +**Architecture:** Separated the loader, filter, compiler, evaluator, normalizer, asserter, and reporter into distinct pipeline stages enforcing SOLID design principles. The normalization stage handles the conversion of jschon logical keyword paths into specification-compliant physical schema fragments. + +**Current Results:** Out of the 84 assertions, 78 Passed, 6 Failed. The 6 failures are due to specific edge-case keyword evaluations native to `jschon` (e.g., `contains` over-annotating negative items, `propertyNames` evaluating actual values, and `$dynamicRef` cross-resource bugs), not harness parsing errors. + +**Quickstart (Local):** + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e .[dev] +python annotation_harness.py +pytest +``` + +**Quickstart (Docker):** + +```bash +docker build -t gsoc-annotation-harness . +docker run --rm gsoc-annotation-harness +``` + +## Normalization Logic Note +Jschon outputs absolute locations (e.g., `urn:uuid:xxx#/$defs/bar/title`). To meet test suite structural expectations (`{"#/$defs/bar": ...}`), the pipeline resolves these references locally, trims trailing keywords, and injects `$schema` dynamically into fragments missing it on compilation. diff --git a/annotation_harness.py b/annotation_harness.py new file mode 100644 index 00000000..a57fee14 --- /dev/null +++ b/annotation_harness.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +# Annotation Test Harness — runs the official JSON Schema annotation tests +# against jschon and reports PASS/FAIL per assertion. +# +# Usage: python annotation_harness.py [--verbose] +# Exit: 0 = all pass, 1 = any fail + +import json +import sys +from pathlib import Path + +from jschon import JSON, Catalog, JSONSchema, create_catalog + +# ---- Config ---- + +DIALECT = "2020-12" +DIALECT_NUMBER = 2020 +METASCHEMA_URI = "https://json-schema.org/draft/2020-12/schema" +ANNOTATIONS_DIR = Path(__file__).resolve().parent / "annotations" / "tests" + +# Maps compatibility strings to sortable version numbers +DIALECT_MAP = { + "3": 3, "4": 4, "6": 6, "7": 7, + "2019": 2019, "2020": 2020, +} + + +# ---- Loader ---- +# Single responsibility: read test files from disk, nothing else. + +def load_test_files(directory: Path) -> list[dict]: + """Parses every .json file in annotations/tests/ into structured dicts.""" + files = [] + for path in sorted(directory.glob("*.json")): + with open(path, encoding="utf-8") as f: + data = json.load(f) + files.append({ + "filename": path.name, + "description": data.get("description", path.stem), + "suite": data.get("suite", []), + }) + return files + + +# ---- Filter ---- +# Decides whether a test case applies to our target dialect. +# Keeps filtering logic isolated from the rest of the pipeline. + +def parse_compatibility(compat_str: str | None, dialect_num: int) -> bool: + """Returns True if the test case should run for our dialect. + + Supports: "7" (>=), "<=2019", "=2020", and comma-separated combos. + None means always compatible. + """ + if compat_str is None: + return True + + for constraint in compat_str.split(","): + constraint = constraint.strip() + if constraint.startswith("<="): + version = DIALECT_MAP.get(constraint[2:]) + if version is None or dialect_num > version: + return False + elif constraint.startswith("="): + version = DIALECT_MAP.get(constraint[1:]) + if version is None or dialect_num != version: + return False + else: + version = DIALECT_MAP.get(constraint) + if version is None or dialect_num < version: + return False + return True + + +# ---- Compiler ---- +# Builds a JSONSchema object the validator can use. +# Handles the quirk that test-suite schemas omit $schema for portability, +# but jschon requires it — so we inject it here. + +def compile_schema( + catalog: Catalog, schema_data, external_schemas: dict | None = None +) -> JSONSchema: + """Creates a JSONSchema, injecting $schema and registering externals.""" + + # Register any referenced external schemas first + if external_schemas: + for uri, ext_schema in external_schemas.items(): + if isinstance(ext_schema, dict): + ext_copy = dict(ext_schema) + if "$schema" not in ext_copy: + ext_copy["$schema"] = METASCHEMA_URI + JSONSchema(ext_copy, uri=uri) + else: + JSONSchema(ext_schema, uri=uri) # boolean schema + + # Boolean schemas (true/false) don't take $schema + if isinstance(schema_data, bool): + return JSONSchema(schema_data) + + # Inject $schema when missing so jschon knows which dialect to use + schema_copy = dict(schema_data) + if "$schema" not in schema_copy: + schema_copy["$schema"] = METASCHEMA_URI + + return JSONSchema(schema_copy) + + +# ---- Evaluator ---- +# Runs the schema against an instance and returns raw output. +# Kept thin on purpose — one job, no side effects. + +def evaluate_instance(schema: JSONSchema, instance) -> dict: + """Evaluates instance against schema, returns jschon's basic output.""" + result = schema.evaluate(JSON(instance)) + return result.output("basic") + + +# ---- Normalizer ---- +# Transforms jschon's output shape into the format the test suite expects. +# This is the hardest part — jschon uses absoluteKeywordLocation (a full URI), +# while the suite expects plain fragment pointers like "#/properties/foo". + +def extract_keyword(keyword_location: str) -> str: + """Grabs the keyword name from the end of a keywordLocation path. + "/properties/foo/title" -> "title" + """ + return keyword_location.rsplit("/", 1)[-1] + + +def normalize_schema_location(abs_keyword_location: str) -> str: + """Converts absoluteKeywordLocation to a schema fragment pointer. + + "urn:uuid:xxx#/properties/foo/title" -> "#/properties/foo" + "urn:uuid:xxx#/title" -> "#" + + Key insight: we use absoluteKeywordLocation (not keywordLocation) + because it correctly resolves $ref to the actual $defs location. + """ + if "#" not in abs_keyword_location: + return "#" + + fragment = abs_keyword_location.split("#", 1)[1] + + # Single-segment fragment like "/title" means root schema + if "/" not in fragment or fragment.count("/") == 1: + return "#" + + # Strip the trailing keyword to get the schema location + parent = fragment.rsplit("/", 1)[0] + return "#" + parent + + +# Applicator keywords that emit their own bookkeeping annotations +# (e.g. "properties" emits which property names matched). The test +# suite doesn't check for these, so we skip them. +_APPLICATOR_KEYWORDS = frozenset({ + "properties", "patternProperties", "additionalProperties", + "items", "prefixItems", "contains", + "if", "unevaluatedProperties", "unevaluatedItems", +}) + + +def normalize_annotations(output: dict) -> dict: + """Reshapes jschon output into {(instanceLoc, keyword): {schemaLoc: value}}. + + This is the shape the test suite assertions expect. + """ + result = {} + + for ann in output.get("annotations", []): + instance_loc = ann.get("instanceLocation", "") + keyword_loc = ann.get("keywordLocation", "") + abs_keyword_loc = ann.get("absoluteKeywordLocation", "") + value = ann.get("annotation") + + keyword = extract_keyword(keyword_loc) + + # Skip applicator bookkeeping (list of matched props, bool flags) + if keyword in _APPLICATOR_KEYWORDS and isinstance(value, (list, bool)): + continue + + schema_loc = normalize_schema_location(abs_keyword_loc) + + key = (instance_loc, keyword) + if key not in result: + result[key] = {} + result[key][schema_loc] = value + + return result + + +# ---- Asserter ---- +# Compares expected vs actual annotations. Open/closed: easy to extend +# with new comparison modes without touching existing logic. + +def deep_equal(expected, actual) -> bool: + """Recursive equality that handles dicts, lists, and JSON primitives.""" + if type(expected) is not type(actual): + # JSON doesn't distinguish int/float + if isinstance(expected, (int, float)) and isinstance(actual, (int, float)): + return expected == actual + return False + if isinstance(expected, dict): + if set(expected.keys()) != set(actual.keys()): + return False + return all(deep_equal(expected[k], actual[k]) for k in expected) + if isinstance(expected, list): + if len(expected) != len(actual): + return False + return all(deep_equal(e, a) for e, a in zip(expected, actual, strict=False)) + return expected == actual + + +def check_assertion(normalized: dict, assertion: dict) -> tuple[bool, str]: + """Checks one assertion. Returns (passed, failure_message).""" + location = assertion["location"] + keyword = assertion["keyword"] + expected = assertion["expected"] + + actual = normalized.get((location, keyword), {}) + + # Empty expected = annotation must NOT exist here + if expected == {}: + if not actual: + return True, "" + return False, ( + f"Expected NO annotations at '{location}' for '{keyword}', " + f"but got: {json.dumps(actual)}" + ) + + # Non-empty expected = must match exactly + if deep_equal(expected, actual): + return True, "" + return False, ( + f"Mismatch at '{location}' for '{keyword}':\n" + f" Expected: {json.dumps(expected, sort_keys=True)}\n" + f" Got: {json.dumps(actual, sort_keys=True)}" + ) + + +# ---- Reporter ---- +# Single responsibility: format and print results. +# Knows nothing about schemas or assertions — just tallies. + +class Reporter: + """Collects pass/fail counts and prints a human-readable report.""" + + def __init__(self, verbose: bool = False): + self.verbose = verbose + self.total_pass = 0 + self.total_fail = 0 + self.total_skip = 0 + + def file_header(self, filename: str): + print(f"\n--- {filename} ---") + + def skip(self, description: str, reason: str): + self.total_skip += 1 + if self.verbose: + print(f" [SKIP] {description} ({reason})") + + def test_case( + self, description: str, passed: int, failed: int, failures: list[str] + ): + total = passed + failed + if failed == 0: + print(f" [PASS] {description} ({passed}/{total} assertions passed)") + else: + print(f" [FAIL] {description} ({passed}/{total} assertions passed)") + for msg in failures: + print(f" {msg}") + self.total_pass += passed + self.total_fail += failed + + def error(self, description: str, err: str): + print(f" [ERROR] {description}: {err}") + self.total_fail += 1 + + def summary(self): + total = self.total_pass + self.total_fail + print(f"\n{'=' * 60}") + print(f"RESULTS: {self.total_pass}/{total} assertions passed, " + f"{self.total_fail} failed, {self.total_skip} test cases skipped") + if self.total_fail == 0: + print("All assertions PASSED!") + print(f"{'=' * 60}") + + @property + def exit_code(self) -> int: + return 1 if self.total_fail > 0 else 0 + + +# ---- Pipeline ---- +# Orchestrates all stages. Each stage has a single responsibility; +# this function just wires them together in order. + +def run_harness(verbose: bool = False) -> int: + """Runs the full pipeline: + load -> filter -> compile -> evaluate -> normalize -> assert -> report. + """ + + print("=" * 60) + print("JSON Schema Annotation Test Harness") + print("Implementation: jschon (Python)") + print(f"Target Dialect: {DIALECT}") + print("=" * 60) + + catalog = create_catalog(DIALECT) + reporter = Reporter(verbose=verbose) + test_files = load_test_files(ANNOTATIONS_DIR) + + if not test_files: + print(f"ERROR: No test files found in {ANNOTATIONS_DIR}") + return 1 + + for test_file in test_files: + filename = test_file["filename"] + reporter.file_header(filename) + + for test_case in test_file["suite"]: + desc = test_case.get("description", "(no description)") + compat = test_case.get("compatibility") + + # Filter incompatible test cases + if not parse_compatibility(compat, DIALECT_NUMBER): + reporter.skip(desc, f"incompatible with {DIALECT}") + continue + + try: + # Compile the schema + schema = compile_schema( + catalog, test_case["schema"], test_case.get("externalSchemas") + ) + + passed = 0 + failed = 0 + failures = [] + + for test in test_case["tests"]: + # Evaluate and normalize + output = evaluate_instance(schema, test["instance"]) + normalized = normalize_annotations(output) + + # Check each assertion + for assertion in test.get("assertions", []): + ok, msg = check_assertion(normalized, assertion) + if ok: + passed += 1 + else: + failed += 1 + failures.append(msg) + + reporter.test_case(desc, passed, failed, failures) + + except Exception as e: + reporter.error(desc, str(e)) + + reporter.summary() + return reporter.exit_code + + +if __name__ == "__main__": + verbose = "--verbose" in sys.argv or "-v" in sys.argv + sys.exit(run_harness(verbose=verbose)) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..eb885032 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "gsoc-annotation-harness" +version = "0.1.0" +description = "GSoC 2026 Qualification Task: JSON Schema Annotation Test Harness" +readme = "HARNESS_README.md" +requires-python = ">=3.9" +dependencies = [ + "jschon>=0.11.1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "ruff>=0.3.0" +] + +[project.scripts] +annotation-harness = "annotation_harness:run_harness" + +[tool.ruff] +line-length = 88 +target-version = "py310" + +[tool.ruff.lint] +# Enable Pyflakes (F) and pycodestyle (E) +select = ["E", "F"] +ignore = [] + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "-ra -q" +testpaths = [ + ".", +] +filterwarnings = [ + "ignore::DeprecationWarning:rfc3986", +] diff --git a/test_annotation_harness.py b/test_annotation_harness.py new file mode 100644 index 00000000..da13215c --- /dev/null +++ b/test_annotation_harness.py @@ -0,0 +1,317 @@ +# ---- Tests for annotation_harness.py ---- +# Covers each pipeline stage in isolation, then a small integration test. + +import pytest +from jschon import JSONSchema, create_catalog + +from annotation_harness import ( + ANNOTATIONS_DIR, + DIALECT, + Reporter, + check_assertion, + compile_schema, + deep_equal, + evaluate_instance, + extract_keyword, + load_test_files, + normalize_annotations, + normalize_schema_location, + parse_compatibility, +) + +# ---- Fixtures ---- + +@pytest.fixture(scope="session") +def catalog(): + """Initialize jschon catalog once for the whole test session.""" + return create_catalog(DIALECT) + + +# ---- Loader Tests ---- + +class TestLoader: + def test_loads_all_test_files(self): + files = load_test_files(ANNOTATIONS_DIR) + filenames = [f["filename"] for f in files] + assert "meta-data.json" in filenames + assert "applicators.json" in filenames + assert "core.json" in filenames + + def test_each_file_has_suite(self): + files = load_test_files(ANNOTATIONS_DIR) + for f in files: + assert "suite" in f + assert isinstance(f["suite"], list) + + def test_empty_dir_returns_empty(self, tmp_path): + assert load_test_files(tmp_path) == [] + + +# ---- Filter Tests ---- + +class TestFilter: + def test_none_is_always_compatible(self): + assert parse_compatibility(None, 2020) is True + + def test_bare_number_minimum(self): + assert parse_compatibility("7", 2020) is True + assert parse_compatibility("2020", 2020) is True + assert parse_compatibility("2020", 2019) is False + + def test_less_equal(self): + assert parse_compatibility("<=2019", 2019) is True + assert parse_compatibility("<=2019", 2020) is False + + def test_exact_match(self): + assert parse_compatibility("=2020", 2020) is True + assert parse_compatibility("=2020", 2019) is False + + def test_comma_separated(self): + # Between draft-06 and 2019-09 + assert parse_compatibility("6,<=2019", 7) is True + assert parse_compatibility("6,<=2019", 2019) is True + assert parse_compatibility("6,<=2019", 2020) is False + assert parse_compatibility("6,<=2019", 4) is False + + def test_unknown_version_returns_false(self): + assert parse_compatibility("99", 2020) is False + + +# ---- Normalizer Tests ---- + +class TestExtractKeyword: + def test_simple(self): + assert extract_keyword("/title") == "title" + + def test_nested(self): + assert extract_keyword("/properties/foo/title") == "title" + + def test_deep(self): + assert extract_keyword("/allOf/0/title") == "title" + + +class TestNormalizeSchemaLocation: + def test_root_keyword(self): + assert normalize_schema_location("urn:uuid:abc#/title") == "#" + + def test_nested_property(self): + loc = normalize_schema_location("urn:uuid:abc#/properties/foo/title") + assert loc == "#/properties/foo" + + def test_defs_via_ref(self): + loc = normalize_schema_location("urn:uuid:abc#/$defs/bar/title") + assert loc == "#/$defs/bar" + + def test_percent_encoded(self): + loc = normalize_schema_location("urn:uuid:abc#/patternProperties/%5Ea/title") + assert loc == "#/patternProperties/%5Ea" + + def test_no_fragment(self): + assert normalize_schema_location("urn:uuid:abc") == "#" + + def test_allof_path(self): + loc = normalize_schema_location("urn:uuid:abc#/allOf/1/title") + assert loc == "#/allOf/1" + + +class TestNormalizeAnnotations: + def test_basic_title(self): + output = { + "valid": True, + "annotations": [ + { + "instanceLocation": "", + "keywordLocation": "/title", + "absoluteKeywordLocation": "urn:uuid:x#/title", + "annotation": "Foo", + } + ], + } + result = normalize_annotations(output) + assert result == {("", "title"): {"#": "Foo"}} + + def test_skips_applicator_list_annotations(self): + output = { + "valid": True, + "annotations": [ + { + "instanceLocation": "", + "keywordLocation": "/properties", + "absoluteKeywordLocation": "urn:uuid:x#/properties", + "annotation": ["foo"], # list => applicator bookkeeping + } + ], + } + result = normalize_annotations(output) + assert result == {} + + def test_empty_annotations(self): + output = {"valid": True} + assert normalize_annotations(output) == {} + + +# ---- Asserter Tests ---- + +class TestDeepEqual: + def test_equal_dicts(self): + assert deep_equal({"a": 1}, {"a": 1}) is True + + def test_unequal_dicts(self): + assert deep_equal({"a": 1}, {"a": 2}) is False + + def test_missing_key(self): + assert deep_equal({"a": 1}, {}) is False + + def test_equal_lists(self): + assert deep_equal([1, "x"], [1, "x"]) is True + + def test_unequal_lists(self): + assert deep_equal([1], [1, 2]) is False + + def test_nested(self): + assert deep_equal({"a": [1, {"b": 2}]}, {"a": [1, {"b": 2}]}) is True + + def test_int_float_equal(self): + assert deep_equal(1, 1.0) is True + + def test_type_mismatch(self): + assert deep_equal("1", 1) is False + + +class TestCheckAssertion: + def test_pass_matching(self): + normalized = {("", "title"): {"#": "Foo"}} + assertion = {"location": "", "keyword": "title", "expected": {"#": "Foo"}} + ok, msg = check_assertion(normalized, assertion) + assert ok is True + assert msg == "" + + def test_fail_mismatch(self): + normalized = {("", "title"): {"#": "Bar"}} + assertion = {"location": "", "keyword": "title", "expected": {"#": "Foo"}} + ok, msg = check_assertion(normalized, assertion) + assert ok is False + assert "Mismatch" in msg + + def test_pass_empty_expected_no_annotation(self): + normalized = {} + assertion = {"location": "/x", "keyword": "title", "expected": {}} + ok, _ = check_assertion(normalized, assertion) + assert ok is True + + def test_fail_empty_expected_but_annotation_exists(self): + normalized = {("/x", "title"): {"#/foo": "Bar"}} + assertion = {"location": "/x", "keyword": "title", "expected": {}} + ok, msg = check_assertion(normalized, assertion) + assert ok is False + assert "Expected NO annotations" in msg + + +# ---- Compiler Tests ---- + +class TestCompiler: + def test_compiles_simple_schema(self, catalog): + schema = compile_schema(catalog, {"title": "Test"}) + assert isinstance(schema, JSONSchema) + + def test_compiles_boolean_schema(self, catalog): + schema = compile_schema(catalog, True) + assert isinstance(schema, JSONSchema) + + def test_preserves_existing_dollar_schema(self, catalog): + data = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Existing", + } + schema = compile_schema(catalog, data) + assert isinstance(schema, JSONSchema) + + +# ---- Evaluator Tests ---- + +class TestEvaluator: + def test_returns_dict_with_valid(self, catalog): + schema = compile_schema(catalog, {"title": "Foo"}) + output = evaluate_instance(schema, 42) + assert "valid" in output + assert output["valid"] is True + + def test_returns_annotations(self, catalog): + schema = compile_schema(catalog, {"title": "Foo"}) + output = evaluate_instance(schema, "hello") + assert "annotations" in output + assert len(output["annotations"]) > 0 + + +# ---- Reporter Tests ---- + +class TestReporter: + def test_exit_code_zero_on_all_pass(self): + r = Reporter() + r.total_pass = 10 + r.total_fail = 0 + assert r.exit_code == 0 + + def test_exit_code_one_on_failure(self): + r = Reporter() + r.total_pass = 9 + r.total_fail = 1 + assert r.exit_code == 1 + + def test_skip_increments(self): + r = Reporter() + r.skip("test", "reason") + assert r.total_skip == 1 + + +# ---- Integration: end-to-end on meta-data.json ---- + +class TestIntegration: + def test_meta_data_title(self, catalog): + """The simplest test: schema {"title": "Foo"} on instance 42.""" + schema = compile_schema(catalog, {"title": "Foo"}) + output = evaluate_instance(schema, 42) + normalized = normalize_annotations(output) + + assertion = {"location": "", "keyword": "title", "expected": {"#": "Foo"}} + ok, msg = check_assertion(normalized, assertion) + assert ok, msg + + def test_meta_data_description(self, catalog): + schema = compile_schema(catalog, {"description": "Bar"}) + output = evaluate_instance(schema, "anything") + normalized = normalize_annotations(output) + + ok, msg = check_assertion(normalized, { + "location": "", "keyword": "description", "expected": {"#": "Bar"} + }) + assert ok, msg + + def test_properties_annotation(self, catalog): + """Properties subschema annotations land at the right instance location.""" + schema = compile_schema(catalog, { + "properties": {"foo": {"title": "Foo"}} + }) + output = evaluate_instance(schema, {"foo": 42}) + normalized = normalize_annotations(output) + + ok, msg = check_assertion(normalized, { + "location": "/foo", "keyword": "title", + "expected": {"#/properties/foo": "Foo"}, + }) + assert ok, msg + + def test_ref_resolves_to_defs(self, catalog): + """$ref annotations should point to $defs, not $ref.""" + schema = compile_schema(catalog, { + "$ref": "#/$defs/x", + "$defs": {"x": {"title": "X"}}, + }) + output = evaluate_instance(schema, "val") + normalized = normalize_annotations(output) + + ok, msg = check_assertion(normalized, { + "location": "", "keyword": "title", + "expected": {"#/$defs/x": "X"}, + }) + assert ok, msg