From e79e297cafc546bae8f8c40dcb3bf81125df3fed Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Fri, 26 Jun 2026 23:24:25 +0200 Subject: [PATCH 1/6] Add CI validation against results_v4 schemas - Add common_resources as the common/ submodule (pinned to main), which carries the canonical schemas/results_v4 JSON Schemas. - Add scripts/validate_results.py: builds a schema registry from common/schemas/results_v4 (refs resolve locally, no network) and validates each / release file against its sub-schema (metric_info optional). - Add .github/workflows/validate.yml to run it on push/PR. --- .github/workflows/validate.yml | 25 ++++++++ .gitmodules | 3 + README.md | 18 ++++++ common | 1 + scripts/validate_results.py | 106 +++++++++++++++++++++++++++++++++ 5 files changed, 153 insertions(+) create mode 100644 .github/workflows/validate.yml create mode 100644 .gitmodules create mode 160000 common create mode 100644 scripts/validate_results.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..664ed40 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,25 @@ +name: Validate results + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install jsonschema + + - name: Validate results against results_v4 schemas + run: python scripts/validate_results.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d35f6d2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "common"] + path = common + url = git@github.com:openproblems-bio/common_resources.git diff --git a/README.md b/README.md index eb8c206..fc0bbe8 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,24 @@ data/results/// The six `*_info` / `results` / `quality_control` files are the results_v4 outputs. +## Validation + +Every release is validated in CI against the canonical **results_v4** JSON Schemas, +which live in the [`common_resources`](https://github.com/openproblems-bio/common_resources) +repository under `schemas/results_v4/` and are vendored here as the `common/` submodule. +`metric_info.json` is optional; the other five files are required and each is checked +against its matching sub-schema. + +Run it locally: + +```sh +git submodule update --init --recursive # fetch the schemas +pip install jsonschema +python scripts/validate_results.py +``` + +See [`.github/workflows/validate.yml`](.github/workflows/validate.yml). + ## config.json (optional) A maintainer-editable file that lives next to the results, so task owners can tune how diff --git a/common b/common new file mode 160000 index 0000000..011858d --- /dev/null +++ b/common @@ -0,0 +1 @@ +Subproject commit 011858df9679eabedd7d5625d7f255bec2152f66 diff --git a/scripts/validate_results.py b/scripts/validate_results.py new file mode 100644 index 0000000..5387e7a --- /dev/null +++ b/scripts/validate_results.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Validate every results_v4 release in this repo against the canonical schemas. + +The schemas live in the `common/` submodule (openproblems-bio/common_resources, +`schemas/results_v4/`). Each release directory `//` holds the six +per-release JSON files; every present file is validated against its matching +sub-schema. `metric_info.json` is optional; the other five are required. + +Usage: + python scripts/validate_results.py [--schema-dir DIR] [--root DIR] +Exits non-zero if any file is missing, invalid JSON, or fails its schema. +""" +import argparse +import json +import sys +from pathlib import Path + +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +# Per-release file -> schema file in schemas/results_v4. metric_info is optional. +PARTS = { + "task_info.json": "task_info.json", + "dataset_info.json": "dataset_info.json", + "method_info.json": "method_info.json", + "metric_info.json": "metric_info.json", + "quality_control.json": "quality_control.json", + "results.json": "results.json", +} +OPTIONAL = {"metric_info.json"} + + +def load_schemas(schema_dir: Path): + if not schema_dir.is_dir(): + sys.exit( + f"Schema dir not found: {schema_dir}\n" + "Did you check out submodules? Run: git submodule update --init --recursive" + ) + resources, by_file = [], {} + for f in sorted(schema_dir.glob("*.json")): + schema = json.loads(f.read_text()) + sid = schema.get("$id") + if not sid: + sys.exit(f"Schema {f} is missing a $id") + resources.append((sid, Resource.from_contents(schema))) + by_file[f.name] = schema + if not by_file: + sys.exit(f"No schema files found in {schema_dir}") + # Refs between schemas resolve locally via the registry, so no network needed. + return Registry().with_resources(resources), by_file + + +def main() -> int: + here = Path(__file__).resolve().parent + repo = here.parent + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--root", type=Path, default=repo, help="results repo root") + ap.add_argument( + "--schema-dir", + type=Path, + default=repo / "common" / "schemas" / "results_v4", + help="directory holding the results_v4 schema files", + ) + args = ap.parse_args() + + registry, by_file = load_schemas(args.schema_dir) + validators = { + fname: Draft202012Validator(by_file[sfile], registry=registry) + for fname, sfile in PARTS.items() + if sfile in by_file + } + + releases = sorted(p.parent for p in args.root.glob("*/*/task_info.json")) + if not releases: + sys.exit("No releases found (expected //task_info.json).") + + errors = [] + for rel in releases: + name = rel.relative_to(args.root) + for fname in PARTS: + fpath = rel / fname + if not fpath.exists(): + if fname not in OPTIONAL: + errors.append(f"{name}/{fname}: MISSING") + continue + try: + data = json.loads(fpath.read_text()) + except json.JSONDecodeError as e: + errors.append(f"{name}/{fname}: invalid JSON: {e}") + continue + for err in sorted(validators[fname].iter_errors(data), key=lambda e: list(e.absolute_path)): + loc = "/".join(str(p) for p in err.absolute_path) or "(root)" + errors.append(f"{name}/{fname}: {loc}: {err.message}") + print(f"checked {name}") + + if errors: + print(f"\n{len(errors)} validation error(s):", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 1 + print(f"\nAll {len(releases)} release(s) valid against results_v4 schemas.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From f4005367639ce46454c3678ede28879738fbd255 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Sat, 27 Jun 2026 16:46:40 +0200 Subject: [PATCH 2/6] Add website-preview job to the validate workflow --- .github/workflows/validate.yml | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 664ed40..ef29524 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -23,3 +23,63 @@ jobs: - name: Validate results against results_v4 schemas run: python scripts/validate_results.py + + # On a PR (from a branch in this repo), once validation passes, open or update a + # PR in the website repo that bumps its `data/results` submodule to this PR's head + # commit. Cloudflare then builds a preview of the site with these results, and the + # website's `results-preview.yml` workflow posts the preview URL (and links to the + # changed result pages) back onto this PR. + preview: + needs: validate + runs-on: ubuntu-latest + # Fork PRs can't be previewed: their head commit isn't in this repo (so the + # submodule can't point at it) and secrets aren't available to them. + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: Open/update the website preview PR + env: + GH_TOKEN: ${{ secrets.OP_BOT_TOKEN }} + WEBSITE: openproblems-bio/website-v3 + PR: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "::error::OP_BOT_TOKEN secret is not set; cannot open the website preview PR." + exit 1 + fi + BRANCH="bot/results-pr-${PR}" + + git config --global user.name "openproblems-bot" + git config --global user.email "bot@openproblems.bio" + # Authenticate git (including the ssh-style submodule URL) with the bot token. + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "git@github.com:" + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + + git clone --quiet "https://x-access-token:${GH_TOKEN}@github.com/${WEBSITE}" site + cd site + git checkout -B "$BRANCH" origin/main + + # Point data/results at this PR's head commit. + git submodule sync data/results + git submodule update --init data/results + git -C data/results fetch --quiet origin "$HEAD_SHA" + git -C data/results checkout --quiet "$HEAD_SHA" + git add data/results + + if git diff --cached --quiet; then + echo "data/results already at ${HEAD_SHA}; nothing to update." + else + git commit --quiet -m "preview: results#${PR} (data/results @ ${HEAD_SHA:0:7})" + fi + git push --force --quiet origin "$BRANCH" + + if gh pr view "$BRANCH" -R "$WEBSITE" >/dev/null 2>&1; then + echo "Website preview PR already open for $BRANCH (refreshed)." + else + gh pr create -R "$WEBSITE" --base main --head "$BRANCH" \ + --title "Preview: results#${PR}" \ + --body "Automated preview build for [openproblems-bio/results#${PR}](https://github.com/openproblems-bio/results/pull/${PR}). Bumps the \`data/results\` submodule to that PR's head. **Do not merge** — this is a throwaway preview branch." + fi From 51d4d8981d169d6a8146d4efbdd9cd034ee7c6e6 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Sat, 27 Jun 2026 16:48:41 +0200 Subject: [PATCH 3/6] Soft-skip preview job when OP_BOT_TOKEN is unset Warn and exit 0 instead of failing the PR's checks, so the preview job is a no-op until the bot token is configured. --- .github/workflows/validate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index ef29524..7abdb14 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -47,8 +47,8 @@ jobs: run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then - echo "::error::OP_BOT_TOKEN secret is not set; cannot open the website preview PR." - exit 1 + echo "::warning::OP_BOT_TOKEN secret is not set; skipping the website preview PR." + exit 0 fi BRANCH="bot/results-pr-${PR}" From 3f63a5ceb59d2a0f986fc6df334234304fe3d8d0 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 30 Jun 2026 07:18:50 +0200 Subject: [PATCH 4/6] remove unnecessary code --- .github/workflows/validate.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index c954337..f98edf3 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -46,9 +46,6 @@ jobs: git config --global user.name "openproblems-bot" git config --global user.email "bot@openproblems.bio" - # Authenticate git (including the ssh-style submodule URL) with the bot token. - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "git@github.com:" - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" git clone --quiet "https://x-access-token:${GH_TOKEN}@github.com/${WEBSITE}" site cd site From 184d2fe5a8a7995e5a44a1a9d5745807bf22b0ba Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 30 Jun 2026 14:32:35 +0200 Subject: [PATCH 5/6] fix ci --- .github/workflows/validate.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index f98edf3..c4b3fb3 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -65,10 +65,15 @@ jobs: fi git push --force --quiet origin "$BRANCH" - if gh pr view "$BRANCH" -R "$WEBSITE" >/dev/null 2>&1; then - echo "Website preview PR already open for $BRANCH (refreshed)." - else - gh pr create -R "$WEBSITE" --base main --head "$BRANCH" \ + # The force-push already refreshed any existing preview PR, so create one only + # if it's missing. "already exists" is a success, not a failure. + if gh pr create -R "$WEBSITE" --base main --head "$BRANCH" \ --title "Preview: results#${PR}" \ - --body "Automated preview build for [openproblems-bio/results#${PR}](https://github.com/openproblems-bio/results/pull/${PR}). Bumps the \`data/results\` submodule to that PR's head. **Do not merge** — this is a throwaway preview branch." + --body "Automated preview build for [openproblems-bio/results#${PR}](https://github.com/openproblems-bio/results/pull/${PR}). Bumps the \`data/results\` submodule to that PR's head. **Do not merge** — this is a throwaway preview branch." 2> create.err; then + echo "Opened website preview PR for $BRANCH." + elif grep -q "already exists" create.err; then + echo "Website preview PR already open for $BRANCH (refreshed by force-push)." + else + cat create.err >&2 + exit 1 fi From a4adf18d7ae684c7c7574f63322865ab056fad5c Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 30 Jun 2026 15:43:02 +0200 Subject: [PATCH 6/6] fix capitalisation to test ci --- cell_cell_communication/v1.0.0-ligand_target/task_info.json | 2 +- cell_cell_communication/v1.0.0-source_target/task_info.json | 2 +- dimensionality_reduction/v1.0.0/task_info.json | 2 +- foundation_models/v0.1.0/task_info.json | 2 +- label_projection/v2.0.0/task_info.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cell_cell_communication/v1.0.0-ligand_target/task_info.json b/cell_cell_communication/v1.0.0-ligand_target/task_info.json index 1a098b8..24c8b14 100644 --- a/cell_cell_communication/v1.0.0-ligand_target/task_info.json +++ b/cell_cell_communication/v1.0.0-ligand_target/task_info.json @@ -1,6 +1,6 @@ { "name": "cell_cell_communication", - "label": "Cell-cell communication", + "label": "Cell-Cell Communication", "commit": "c97decf07adb2e3050561d6fa9ae46132be07bef", "summary": "Detect interactions between ligands and target cell types", "description": "\nThe growing availability of single-cell data has sparked an increased\ninterest in the inference of cell-cell communication (CCC),\nwith an ever-growing number of computational tools developed for this purpose.\n\nDifferent tools propose distinct preprocessing steps with diverse\nscoring functions, that are challenging to compare and evaluate.\nFurthermore, each tool typically comes with its own set of prior knowledge.\nTo harmonize these, [Dimitrov et\nal, 2022](https://openproblems.bio/bibliography#dimitrov2022comparison) recently\ndeveloped the [LIANA](https://github.com/saezlab/liana) framework, which was used\nas a foundation for this task.\n\nThe challenges in evaluating the tools are further exacerbated by the\nlack of a gold standard to benchmark the performance of CCC methods. In an\nattempt to address this, Dimitrov et al use alternative data modalities, including\nthe spatial proximity of cell types and\ndownstream cytokine activities, to generate an inferred ground truth. However,\nthese modalities are only approximations of biological reality and come\nwith their own assumptions and limitations. In time, the inclusion of more\ndatasets with known ground truth interactions will become available, from\nwhich the limitations and advantages of the different CCC methods will\nbe better understood.\n\n**This subtask evaluates the methods' ability to predict interactions,\nthe corresponding of cytokines of which, are inferred to be active in\nthe target cell types. This subtask focuses\non the prediction of interactions from steady-state, or single-context,\nsingle-cell data.**\n\n", diff --git a/cell_cell_communication/v1.0.0-source_target/task_info.json b/cell_cell_communication/v1.0.0-source_target/task_info.json index d76b12d..5059f55 100644 --- a/cell_cell_communication/v1.0.0-source_target/task_info.json +++ b/cell_cell_communication/v1.0.0-source_target/task_info.json @@ -1,6 +1,6 @@ { "name": "cell_cell_communication", - "label": "Cell-cell communication", + "label": "Cell-Cell Communication", "commit": "c97decf07adb2e3050561d6fa9ae46132be07bef", "summary": "Detect interactions between source and target cell types", "description": "\nThe growing availability of single-cell data has sparked an increased\ninterest in the inference of cell-cell communication (CCC),\nwith an ever-growing number of computational tools developed for this purpose.\n\nDifferent tools propose distinct preprocessing steps with diverse\nscoring functions, that are challenging to compare and evaluate.\nFurthermore, each tool typically comes with its own set of prior knowledge.\nTo harmonize these, [Dimitrov et\nal, 2022](https://openproblems.bio/bibliography#dimitrov2022comparison) recently\ndeveloped the [LIANA](https://github.com/saezlab/liana) framework, which was used\nas a foundation for this task.\n\nThe challenges in evaluating the tools are further exacerbated by the\nlack of a gold standard to benchmark the performance of CCC methods. In an\nattempt to address this, Dimitrov et al use alternative data modalities, including\nthe spatial proximity of cell types and\ndownstream cytokine activities, to generate an inferred ground truth. However,\nthese modalities are only approximations of biological reality and come\nwith their own assumptions and limitations. In time, the inclusion of more\ndatasets with known ground truth interactions will become available, from\nwhich the limitations and advantages of the different CCC methods will\nbe better understood.\n\n**This subtask evaluates methods in their ability to predict interactions between\nspatially-adjacent source cell types and target cell types. This subtask focuses\non the prediction of interactions from steady-state, or single-context,\nsingle-cell data.**\n\n", diff --git a/dimensionality_reduction/v1.0.0/task_info.json b/dimensionality_reduction/v1.0.0/task_info.json index 30e539e..a5ef218 100644 --- a/dimensionality_reduction/v1.0.0/task_info.json +++ b/dimensionality_reduction/v1.0.0/task_info.json @@ -1,6 +1,6 @@ { "name": "dimensionality_reduction", - "label": "Dimensionality reduction for visualisation", + "label": "Dimensionality Reduction for Visualisation", "commit": "0a0e902bd1482e35418f7816fc91e9bc31a33126", "summary": "Reduction of high-dimensional datasets to 2D for visualization & interpretation", "description": "\nDimensionality reduction is one of the key challenges in single-cell data\nrepresentation. Routine single-cell RNA sequencing (scRNA-seq) experiments measure cells\nin roughly 20,000-30,000 dimensions (i.e., features - mostly gene transcripts but also\nother functional elements encoded in mRNA such as lncRNAs). Since its inception,\nscRNA-seq experiments have been growing in terms of the number of cells measured.\nOriginally, cutting-edge SmartSeq experiments would yield a few hundred cells, at best.\nNow, it is not uncommon to see experiments that yield over [100,000\ncells](https://openproblems.bio/bibliography#tabula2018single) or even [> 1 million\ncells.](https://openproblems.bio/bibliography#cao2020human)\n\nEach *feature* in a dataset functions as a single dimension. While each of the ~30,000\ndimensions measured in each cell contribute to an underlying data structure, the overall\nstructure of the data is challenging to display in few dimensions due to data sparsity\nand the [*\"curse of\ndimensionality\"*](https://en.wikipedia.org/wiki/Curse_of_dimensionality) (distances in\nhigh dimensional data don’t distinguish data points well). Thus, we need to find a way\nto [dimensionally reduce](https://en.wikipedia.org/wiki/Dimensionality_reduction) the\ndata for visualization and interpretation.\n\n", diff --git a/foundation_models/v0.1.0/task_info.json b/foundation_models/v0.1.0/task_info.json index 8aa1702..9e684e3 100644 --- a/foundation_models/v0.1.0/task_info.json +++ b/foundation_models/v0.1.0/task_info.json @@ -1,6 +1,6 @@ { "name": "foundation_models", - "label": "Foundation models", + "label": "Foundation Models", "commit": null, "summary": "Modelling of single-cells to perform multiple tasks using", "description": "Recent developments in deep-learning have led to the creation of several 'foundation models' for single-cell data. These are large models that have been trained on data from millions of cells and am to fully capture the variability in the single-cell landscape. Typically, they use a transformer architecture [@szalata2024transformers] and undergo self-supervised pre-training using masking of parts of the input data. Trained foundation models can then be applied to a variety of downstream tasks, either by directly feeding new data into the model or by fine-tuning to better fit a new dataset or to produce a specific output. The general nature of single-cell foundation models and the large amount of data they have been trained on makes them potentially powerful tools for single-cell analysis but their performance is yet to be fully established.\n\nOpen Problems builds on existing evaluations [@boiarsky2023foundationmodels; @liu2024foundationmodels] of foundation models by incorporating them into our continuous benchmarking framework.\n\nThis overview combines results from the following benchmarks for individual tasks:\n\n- [Label projection](../label_projection)\n- [Batch Integration](../batch_integration)", diff --git a/label_projection/v2.0.0/task_info.json b/label_projection/v2.0.0/task_info.json index ae8254f..a75512b 100644 --- a/label_projection/v2.0.0/task_info.json +++ b/label_projection/v2.0.0/task_info.json @@ -1,6 +1,6 @@ { "name": "label_projection", - "label": "Label projection", + "label": "Label Projection", "commit": null, "summary": "Automated cell type annotation from rich, labeled reference data", "description": "A major challenge for integrating single cell datasets is creating matching\ncell type annotations for each cell. One of the most common strategies for\nannotating cell types is referred to as\n[\"cluster-then-annotate\"](https://www.nature.com/articles/s41576-018-0088-9)\nwhereby cells are aggregated into clusters based on feature similarity and\nthen manually characterized based on differential gene expression or previously\nidentified marker genes. Recently, methods have emerged to build on this\nstrategy and annotate cells using\n[known marker genes](https://www.nature.com/articles/s41592-019-0535-3).\nHowever, these strategies pose a difficulty for integrating atlas-scale\ndatasets as the particular annotations may not match.\n\nTo ensure that the cell type labels in newly generated datasets match\nexisting reference datasets, some methods align cells to a previously\nannotated [reference dataset](https://academic.oup.com/bioinformatics/article/35/22/4688/54802990)\nand then _project_ labels from the reference to the new dataset.\n\nHere, we compare methods for annotation based on a reference dataset.\nThe datasets consist of two or more samples of single cell profiles that\nhave been manually annotated with matching labels. These datasets are then\nsplit into training and test batches, and the task of each method is to\ntrain a cell type classifer on the training set and project those labels\nonto the test set.\n",