Add CI validation against results_v4 schemas

.github/workflows/validate.yml

@@ -0,0 +1,25 @@
+name: Validate results
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install jsonschema
+
+      - name: Validate results against results_v4 schemas
+        run: python scripts/validate_results.py

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "common"]
+	path = common
+	url = git@github.com:openproblems-bio/common_resources.git

README.md

@@ -15,6 +15,24 @@ data/results/<task>/<version>/
 
 The six `*_info` / `results` / `quality_control` files are the results_v4 outputs.
 
+## Validation
+
+Every release is validated in CI against the canonical **results_v4** JSON Schemas,
+which live in the [`common_resources`](https://github.com/openproblems-bio/common_resources)
+repository under `schemas/results_v4/` and are vendored here as the `common/` submodule.
+`metric_info.json` is optional; the other five files are required and each is checked
+against its matching sub-schema.
+
+Run it locally:
+
+```sh
+git submodule update --init --recursive   # fetch the schemas
+pip install jsonschema
+python scripts/validate_results.py
+```
+
+See [`.github/workflows/validate.yml`](.github/workflows/validate.yml).
+
 ## config.json (optional)
 
 A maintainer-editable file that lives next to the results, so task owners can tune how

scripts/validate_results.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Validate every results_v4 release in this repo against the canonical schemas.
+
+The schemas live in the `common/` submodule (openproblems-bio/common_resources,
+`schemas/results_v4/`). Each release directory `<task>/<version>/` holds the six
+per-release JSON files; every present file is validated against its matching
+sub-schema. `metric_info.json` is optional; the other five are required.
+
+Usage:
+    python scripts/validate_results.py [--schema-dir DIR] [--root DIR]
+Exits non-zero if any file is missing, invalid JSON, or fails its schema.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from jsonschema import Draft202012Validator
+from referencing import Registry, Resource
+
+# Per-release file -> schema file in schemas/results_v4. metric_info is optional.
+PARTS = {
+    "task_info.json": "task_info.json",
+    "dataset_info.json": "dataset_info.json",
+    "method_info.json": "method_info.json",
+    "metric_info.json": "metric_info.json",
+    "quality_control.json": "quality_control.json",
+    "results.json": "results.json",
+}
+OPTIONAL = {"metric_info.json"}
+
+
+def load_schemas(schema_dir: Path):
+    if not schema_dir.is_dir():
+        sys.exit(
+            f"Schema dir not found: {schema_dir}\n"
+            "Did you check out submodules? Run: git submodule update --init --recursive"
+        )
+    resources, by_file = [], {}
+    for f in sorted(schema_dir.glob("*.json")):
+        schema = json.loads(f.read_text())
+        sid = schema.get("$id")
+        if not sid:
+            sys.exit(f"Schema {f} is missing a $id")
+        resources.append((sid, Resource.from_contents(schema)))
+        by_file[f.name] = schema
+    if not by_file:
+        sys.exit(f"No schema files found in {schema_dir}")
+    # Refs between schemas resolve locally via the registry, so no network needed.
+    return Registry().with_resources(resources), by_file
+
+
+def main() -> int:
+    here = Path(__file__).resolve().parent
+    repo = here.parent
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--root", type=Path, default=repo, help="results repo root")
+    ap.add_argument(
+        "--schema-dir",
+        type=Path,
+        default=repo / "common" / "schemas" / "results_v4",
+        help="directory holding the results_v4 schema files",
+    )
+    args = ap.parse_args()
+
+    registry, by_file = load_schemas(args.schema_dir)
+    validators = {
+        fname: Draft202012Validator(by_file[sfile], registry=registry)
+        for fname, sfile in PARTS.items()
+        if sfile in by_file
+    }
+
+    releases = sorted(p.parent for p in args.root.glob("*/*/task_info.json"))
+    if not releases:
+        sys.exit("No releases found (expected <task>/<version>/task_info.json).")
+
+    errors = []
+    for rel in releases:
+        name = rel.relative_to(args.root)
+        for fname in PARTS:
+            fpath = rel / fname
+            if not fpath.exists():
+                if fname not in OPTIONAL:
+                    errors.append(f"{name}/{fname}: MISSING")
+                continue
+            try:
+                data = json.loads(fpath.read_text())
+            except json.JSONDecodeError as e:
+                errors.append(f"{name}/{fname}: invalid JSON: {e}")
+                continue
+            for err in sorted(validators[fname].iter_errors(data), key=lambda e: list(e.absolute_path)):
+                loc = "/".join(str(p) for p in err.absolute_path) or "(root)"
+                errors.append(f"{name}/{fname}: {loc}: {err.message}")
+        print(f"checked {name}")
+
+    if errors:
+        print(f"\n{len(errors)} validation error(s):", file=sys.stderr)
+        for e in errors:
+            print(f"  - {e}", file=sys.stderr)
+        return 1
+    print(f"\nAll {len(releases)} release(s) valid against results_v4 schemas.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())