From e79e297cafc546bae8f8c40dcb3bf81125df3fed Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Fri, 26 Jun 2026 23:24:25 +0200 Subject: [PATCH] Add CI validation against results_v4 schemas - Add common_resources as the common/ submodule (pinned to main), which carries the canonical schemas/results_v4 JSON Schemas. - Add scripts/validate_results.py: builds a schema registry from common/schemas/results_v4 (refs resolve locally, no network) and validates each / release file against its sub-schema (metric_info optional). - Add .github/workflows/validate.yml to run it on push/PR. --- .github/workflows/validate.yml | 25 ++++++++ .gitmodules | 3 + README.md | 18 ++++++ common | 1 + scripts/validate_results.py | 106 +++++++++++++++++++++++++++++++++ 5 files changed, 153 insertions(+) create mode 100644 .github/workflows/validate.yml create mode 100644 .gitmodules create mode 160000 common create mode 100644 scripts/validate_results.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..664ed40 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,25 @@ +name: Validate results + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install jsonschema + + - name: Validate results against results_v4 schemas + run: python scripts/validate_results.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d35f6d2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "common"] + path = common + url = git@github.com:openproblems-bio/common_resources.git diff --git a/README.md b/README.md index eb8c206..fc0bbe8 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,24 @@ data/results/// The six `*_info` / `results` / `quality_control` files are the results_v4 outputs. +## Validation + +Every release is validated in CI against the canonical **results_v4** JSON Schemas, +which live in the [`common_resources`](https://github.com/openproblems-bio/common_resources) +repository under `schemas/results_v4/` and are vendored here as the `common/` submodule. +`metric_info.json` is optional; the other five files are required and each is checked +against its matching sub-schema. + +Run it locally: + +```sh +git submodule update --init --recursive # fetch the schemas +pip install jsonschema +python scripts/validate_results.py +``` + +See [`.github/workflows/validate.yml`](.github/workflows/validate.yml). + ## config.json (optional) A maintainer-editable file that lives next to the results, so task owners can tune how diff --git a/common b/common new file mode 160000 index 0000000..011858d --- /dev/null +++ b/common @@ -0,0 +1 @@ +Subproject commit 011858df9679eabedd7d5625d7f255bec2152f66 diff --git a/scripts/validate_results.py b/scripts/validate_results.py new file mode 100644 index 0000000..5387e7a --- /dev/null +++ b/scripts/validate_results.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Validate every results_v4 release in this repo against the canonical schemas. + +The schemas live in the `common/` submodule (openproblems-bio/common_resources, +`schemas/results_v4/`). Each release directory `//` holds the six +per-release JSON files; every present file is validated against its matching +sub-schema. `metric_info.json` is optional; the other five are required. + +Usage: + python scripts/validate_results.py [--schema-dir DIR] [--root DIR] +Exits non-zero if any file is missing, invalid JSON, or fails its schema. +""" +import argparse +import json +import sys +from pathlib import Path + +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +# Per-release file -> schema file in schemas/results_v4. metric_info is optional. +PARTS = { + "task_info.json": "task_info.json", + "dataset_info.json": "dataset_info.json", + "method_info.json": "method_info.json", + "metric_info.json": "metric_info.json", + "quality_control.json": "quality_control.json", + "results.json": "results.json", +} +OPTIONAL = {"metric_info.json"} + + +def load_schemas(schema_dir: Path): + if not schema_dir.is_dir(): + sys.exit( + f"Schema dir not found: {schema_dir}\n" + "Did you check out submodules? Run: git submodule update --init --recursive" + ) + resources, by_file = [], {} + for f in sorted(schema_dir.glob("*.json")): + schema = json.loads(f.read_text()) + sid = schema.get("$id") + if not sid: + sys.exit(f"Schema {f} is missing a $id") + resources.append((sid, Resource.from_contents(schema))) + by_file[f.name] = schema + if not by_file: + sys.exit(f"No schema files found in {schema_dir}") + # Refs between schemas resolve locally via the registry, so no network needed. + return Registry().with_resources(resources), by_file + + +def main() -> int: + here = Path(__file__).resolve().parent + repo = here.parent + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--root", type=Path, default=repo, help="results repo root") + ap.add_argument( + "--schema-dir", + type=Path, + default=repo / "common" / "schemas" / "results_v4", + help="directory holding the results_v4 schema files", + ) + args = ap.parse_args() + + registry, by_file = load_schemas(args.schema_dir) + validators = { + fname: Draft202012Validator(by_file[sfile], registry=registry) + for fname, sfile in PARTS.items() + if sfile in by_file + } + + releases = sorted(p.parent for p in args.root.glob("*/*/task_info.json")) + if not releases: + sys.exit("No releases found (expected //task_info.json).") + + errors = [] + for rel in releases: + name = rel.relative_to(args.root) + for fname in PARTS: + fpath = rel / fname + if not fpath.exists(): + if fname not in OPTIONAL: + errors.append(f"{name}/{fname}: MISSING") + continue + try: + data = json.loads(fpath.read_text()) + except json.JSONDecodeError as e: + errors.append(f"{name}/{fname}: invalid JSON: {e}") + continue + for err in sorted(validators[fname].iter_errors(data), key=lambda e: list(e.absolute_path)): + loc = "/".join(str(p) for p in err.absolute_path) or "(root)" + errors.append(f"{name}/{fname}: {loc}: {err.message}") + print(f"checked {name}") + + if errors: + print(f"\n{len(errors)} validation error(s):", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 1 + print(f"\nAll {len(releases)} release(s) valid against results_v4 schemas.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())