diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..fa8120f --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,69 @@ +# OAK adapter benchmarks & selection guide + +`reachable_from` validation resolves ontology graph closures through an OAK +adapter. **Which adapter you pick changes both correctness and performance** — +the wrong choice silently produces incorrect validation or unusable speed. This +directory documents the tradeoffs and provides a repeatable way to re-measure +them. + +- **Correctness is guarded, deterministically, in CI** by + [`tests/test_adapter_parity.py`](../tests/test_adapter_parity.py): local file + adapters must agree on the closure; OLS must not over-return; Ubergraph is a + merged multi-ontology graph that must be prefix-filtered. +- **Performance is measured on demand** (never asserted — it depends on caches + and network) by [`adapter_benchmark.py`](./adapter_benchmark.py), run via + `just benchmark`. Each adapter is benchmarked in its own subprocess, so the + `Peak RAM` column is that adapter's true peak (values are per-process and need + not be monotonic). Latest captured run: + [`results/go_adapter_comparison.md`](./results/go_adapter_comparison.md). + +> Reading the table: closures are measured sequentially per adapter, so the +> `Cold (+part_of)` call runs *after* the `is_a` calls. For adapters that build +> an in-process adjacency index on first traversal (`simpleobo`, `pronto`) that +> column reflects incremental predicate cost, not a cold start — which is why it +> can be far cheaper than `Cold (is_a)`. Adapters with no closure cache (`owl`) +> pay the full cost on every call, so all their columns are similar and large. + +## Adapter selection guide + +| Adapter | Closure cost | Per-term throughput | Scope | Recommended for | +|---|---|---|---|---| +| `sqlite:obo:` | ~ms (indexed `entailed_edge`) | fast | single ontology | **Production default** | +| `pronto:` / bare `.obo` | ~3.6s cold, cached after | high | single ontology | Local file; heavy/repeated traversal | +| `simpleobo:` | ~8–10s cold | moderate | single ontology | Low RAM / fast startup; few closures | +| `ubergraph:` | ~2s warm (one big query) | low (network) | **all ontologies merged** | One-shot cross-ontology closure; **must prefix-filter** for single-ontology validation | +| `owl:` (`.owl`/`.ofn`) | ~145s, no closure cache ❌ | very low | single ontology | **Avoid** for closures (issue #57) | +| `ols:` | truncated to first page ❌ | remote | single ontology | **Avoid** for closures (issue #55); labels only | + +Because `reachable_from` expansion computes one closure and then caches it, the +metric that matters is **load + one cold closure**: `pronto` ≈ 8s, `simpleobo` +≈ 12s (half the RAM), `ubergraph` ≈ 2s warm (but prefix-filter!), `owl` ≈ 160s. + +### Two failure modes to know + +- **`ols:` under-returns (false rejects).** OAK's OLS adapter truncates + descendant/ancestor closures to the first page (~500), so deep terms are + silently dropped and valid data is rejected. See issue #55 (upstream: + `INCATools/ontology-access-kit#893`). +- **`ubergraph:` over-returns (false accepts).** Ubergraph merges many + ontologies, so `descendants(GO:…)` also returns MRO/WBbt/PR/CL/… terms. A + reachable_from enum meaning "GO descendants" must filter by `GO:` prefix or it + will accept foreign-ontology terms. See issue #58. + +## Running the benchmark + +```bash +# full GO, all local file adapters + ubergraph (downloads go.obo / go.owl) +just benchmark +# or directly, with options: +uv run python benchmarks/adapter_benchmark.py --help + +# a subset of adapters against your own ontology, writing the table to a file +uv run python benchmarks/adapter_benchmark.py \ + --adapters simpleobo,pronto --obo path/to/onto.obo --root FOO:0000001 \ + --out benchmarks/results/my_comparison.md +``` + +By default every backend cache (semsql downloads, HTTP cache) is redirected to a +throwaway temp dir so cold/warm numbers are comparable across runs; pass +`--no-isolate-caches` to measure against your real caches instead. diff --git a/benchmarks/adapter_benchmark.py b/benchmarks/adapter_benchmark.py new file mode 100644 index 0000000..030fb6c --- /dev/null +++ b/benchmarks/adapter_benchmark.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Benchmark OAK adapters for reachable_from-style is-a / part_of closures. + +This is an **informational** tool, not a test: timing depends on download +caches, ``requests-cache`` state, network conditions, and the machine. To make +runs comparable rather than reproducible-to-the-millisecond: + +* each adapter is benchmarked in its **own subprocess** (``--worker``), so + ``ru_maxrss`` is that adapter's true peak RAM rather than a process-wide + cumulative peak, and one adapter's caches can't leak into another's; +* every backend cache is redirected to a throwaway temp dir + (cache isolation is on by default; disable with ``--no-isolate-caches``); +* both *cold* (first call) and *warm* (second call) closures are reported -- + the cold/warm gap is itself the interesting signal. + +It regenerates the adapter comparison table published in +``linkml/linkml-term-validator#58``. + +Examples:: + + # full GO, all local file adapters + ubergraph (downloads ontologies) + uv run python benchmarks/adapter_benchmark.py + + # a subset of adapters against your own ontology files + uv run python benchmarks/adapter_benchmark.py \ + --adapters simpleobo,pronto --obo path/to/onto.obo --root FOO:0000001 + +Correctness (that the adapters *agree* on the closure) is guarded separately and +deterministically by ``tests/test_adapter_parity.py``; this script only measures +cost. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import gc +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable + +try: + import resource # POSIX only; absent on Windows +except ImportError: # pragma: no cover - Windows + resource = None # type: ignore[assignment] + +# Source the predicate CURIEs from OAK's vocabulary so they can't drift from the +# constants the adapters (and tests/test_adapter_parity.py) use. +from oaklib.datamodels.vocabulary import IS_A, PART_OF # noqa: E402 + +# GO cellular_component: a deep, real branch (~4k is-a descendants). +DEFAULT_ROOT = "GO:0005575" +GO_OBO_URL = "https://purl.obolibrary.org/obo/go.obo" +GO_OWL_URL = "https://purl.obolibrary.org/obo/go.owl" + +PREDICATE_SETS: dict[str, list[str]] = { + # NB: "is_a" is intentionally first; the ancestor-throughput sample is drawn + # from the is_a descendant set (see _benchmark_adapter). + "is_a": [IS_A], + "is_a+part_of": [IS_A, PART_OF], +} + + +def _max_rss_mb() -> float | None: + if resource is None: # pragma: no cover - Windows + return None + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # ru_maxrss is KiB on Linux but bytes on macOS. + divisor = 1024 * 1024 if sys.platform == "darwin" else 1024 + return peak / divisor + + +def _time(fn: Callable[[], Any]) -> tuple[Any, float]: + gc.collect() + start = time.perf_counter() + out = fn() + return out, time.perf_counter() - start + + +@dataclass +class AdapterResult: + key: str + scope: str + load_s: float | None = None + cold_s: dict[str, float] = field(default_factory=dict) + warm_s: dict[str, float] = field(default_factory=dict) + count: dict[str, int] = field(default_factory=dict) + anc_per_s: float | None = None + rss_mb: float | None = None + error: str | None = None + + +@dataclass +class AdapterSpec: + key: str + scope: str # "single" | "merged (remote)" + build_shorthand: Callable[["BenchContext"], str] + + +class BenchContext: + """Resolves ontology file paths (downloading / converting once).""" + + def __init__(self, args: argparse.Namespace, workdir: Path) -> None: + self.args = args + self.workdir = workdir + self._obo: Path | None = Path(args.obo) if args.obo else None + self._owl: Path | None = Path(args.owl) if args.owl else None + + def _download(self, url: str, dest: Path) -> Path: + if dest.exists(): + return dest + print(f" downloading {url} -> {dest.name} ...", flush=True) + # A User-Agent is required: the default urllib UA is 403'd by the OBO + # hosts / proxy. urlopen honors HTTPS_PROXY/HTTP_PROXY from the env. + req = urllib.request.Request(url, headers={"User-Agent": "linkml-term-validator-benchmark"}) + with urllib.request.urlopen(req, timeout=60) as resp, open(dest, "wb") as fh: # noqa: S310 - trusted OBO PURL + shutil.copyfileobj(resp, fh) + return dest + + def obo_path(self) -> str: + if self._obo is None: + self._obo = self._download(GO_OBO_URL, self.workdir / "go.obo") + return str(self._obo) + + def owl_path(self) -> str: + # The owl adapter (py-horned-owl) reads functional syntax reliably; GO is + # only published as RDF/XML, so download it once and convert to .ofn. + if self._owl is not None: + return str(self._owl) + owl = self._download(GO_OWL_URL, self.workdir / "go.owl") + ofn = self.workdir / "go.ofn" + if not ofn.exists(): + import pyhornedowl + + print(" converting go.owl -> go.ofn (py-horned-owl) ...", flush=True) + onto = pyhornedowl.open_ontology(owl.read_text(), "rdf") + ofn.write_text(onto.save_to_string("ofn")) + self._owl = ofn + return str(ofn) + + +ADAPTER_SPECS: dict[str, AdapterSpec] = { + "simpleobo": AdapterSpec("simpleobo", "single", lambda c: f"simpleobo:{c.obo_path()}"), + "pronto": AdapterSpec("pronto", "single", lambda c: f"pronto:{c.obo_path()}"), + "owl": AdapterSpec("owl", "single", lambda c: f"owl:{c.owl_path()}"), + "ubergraph": AdapterSpec("ubergraph", "merged (remote)", lambda c: "ubergraph:"), +} + + +def _benchmark_adapter(spec: AdapterSpec, ctx: BenchContext, root: str, anc_sample: int) -> AdapterResult: + from oaklib import get_adapter + + res = AdapterResult(key=spec.key, scope=spec.scope) + try: + shorthand = spec.build_shorthand(ctx) + adapter, res.load_s = _time(lambda: get_adapter(shorthand)) + except Exception as e: # pragma: no cover - environment/network dependent + res.error = f"load: {type(e).__name__}: {e}" + return res + + isa_descendants: set[str] = set() + for label, predicates in PREDICATE_SETS.items(): + try: + cold, res.cold_s[label] = _time( + lambda predicates=predicates: set(adapter.descendants(root, predicates=predicates)) + ) + _, res.warm_s[label] = _time( + lambda predicates=predicates: set(adapter.descendants(root, predicates=predicates)) + ) + res.count[label] = len(cold) + if label == "is_a": + isa_descendants = cold + except Exception as e: # pragma: no cover + res.error = f"descendants[{label}]: {type(e).__name__}: {e}" + return res + + # ancestor-closure throughput over a bounded sample of the is_a descendants + # (network adapters are capped so we don't hammer the public endpoint). + cap = min(anc_sample, 25) if "remote" in spec.scope else anc_sample + sample = sorted(isa_descendants)[:cap] + if sample: + try: + _, dt = _time( + lambda: sum(len(set(adapter.ancestors(c, predicates=[IS_A]))) for c in sample) + ) + res.anc_per_s = len(sample) / dt if dt else None + except Exception as e: # pragma: no cover + res.error = f"ancestors: {type(e).__name__}: {e}" + # Own-process peak RSS: meaningful only because each adapter runs in its own + # worker subprocess (see main()). + res.rss_mb = _max_rss_mb() + return res + + +def _fmt(value: float | None, suffix: str = "s") -> str: + return f"{value:.2f}{suffix}" if value is not None else "-" + + +def _render_markdown(results: list[AdapterResult], root: str, meta: dict[str, str]) -> str: + caption = "; ".join(f"{k}: {v}" for k, v in meta.items()) + lines = [ + f"# OAK adapter comparison for reachable_from closures (`descendants({root})`)", + "", + "> Regenerated by `just benchmark` / `benchmarks/adapter_benchmark.py`. " + "Timing is informational (cache/network dependent) and each adapter is measured " + "in its own subprocess; correctness is guarded by `tests/test_adapter_parity.py`.", + "", + f"> Run: {caption}", + "", + "| Adapter | Scope | Load | Cold (is_a) | Warm (is_a) | Cold (+part_of) | " + "Ancestor closures/s | is_a count | Peak RAM |", + "|---|---|---|---|---|---|---|---|---|", + ] + for r in results: + if r.error: + lines.append(f"| `{r.key}` | {r.scope} | ❌ {r.error} | | | | | | |") + continue + lines.append( + "| `{key}` | {scope} | {load} | {cold_isa} | {warm_isa} | {cold_po} | " + "{anc} | {count} | {rss} |".format( + key=r.key, + scope=r.scope, + load=_fmt(r.load_s), + cold_isa=_fmt(r.cold_s.get("is_a")), + warm_isa=_fmt(r.warm_s.get("is_a")), + cold_po=_fmt(r.cold_s.get("is_a+part_of")), + anc=(f"{r.anc_per_s:.0f}/s" if r.anc_per_s is not None else "-"), + count=r.count.get("is_a", "-"), + rss=_fmt(r.rss_mb, " MB"), + ) + ) + lines.append("") + return "\n".join(lines) + + +def _isolate_caches(workdir: Path) -> None: + """Point every backend cache at a throwaway dir so runs are comparable. + + All three are set unconditionally (no ``setdefault``): an inherited value + from the environment would silently defeat the isolation this promises. + """ + cache = workdir / "caches" + cache.mkdir(parents=True, exist_ok=True) + os.environ["PYSTOW_HOME"] = str(cache / "pystow") # semsql / sqlite:obo downloads + os.environ["XDG_CACHE_HOME"] = str(cache / "xdg") + os.environ["OAKLIB_CACHE"] = str(cache / "oaklib") + + +def _run_worker(key: str, args: argparse.Namespace) -> AdapterResult: + """Benchmark a single adapter in a subprocess and return its result.""" + cmd = [ + sys.executable, __file__, "--worker", key, + "--root", args.root, + "--anc-sample", str(args.anc_sample), + "--obo", args.obo, + "--owl", args.owl, + ] + if args.no_isolate_caches: + cmd.append("--no-isolate-caches") + proc = subprocess.run(cmd, capture_output=True, text=True) + for line in proc.stdout.splitlines(): + if line.startswith("__RESULT__"): + return AdapterResult(**json.loads(line[len("__RESULT__"):])) + return AdapterResult( + key=key, + scope=ADAPTER_SPECS[key].scope, + error=f"worker failed (rc={proc.returncode}): {proc.stderr.strip()[-300:] or proc.stdout.strip()[-300:]}", + ) + + +def _resolve_inputs(args: argparse.Namespace, workdir: Path) -> None: + """Download/convert ontology files once so workers reuse them.""" + ctx = BenchContext(args, workdir) + needs_local = any(k in {"simpleobo", "pronto", "owl"} for k in args.keys) + if needs_local and not args.obo: + args.obo = ctx.obo_path() + if "owl" in args.keys and not args.owl: + args.owl = ctx.owl_path() + args.obo = args.obo or "" + args.owl = args.owl or "" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--adapters", default="simpleobo,pronto,owl,ubergraph", + help="comma-separated adapter keys (default: all)") + parser.add_argument("--root", default=DEFAULT_ROOT, help="source node CURIE") + parser.add_argument("--obo", default="", help="local OBO file (default: download GO)") + parser.add_argument("--owl", default="", help="local OWL/OFN file (default: download+convert GO)") + parser.add_argument("--anc-sample", type=int, default=500, help="ancestor-closure sample size") + parser.add_argument("--out", help="write markdown table to this path") + parser.add_argument("--no-isolate-caches", action="store_true", + help="do NOT redirect backend caches to a temp dir (use real caches)") + parser.add_argument("--worker", help=argparse.SUPPRESS) # internal: benchmark one adapter, emit JSON + args = parser.parse_args(argv) + + # -- worker mode: benchmark exactly one adapter, print JSON, exit --------- + if args.worker: + with tempfile.TemporaryDirectory(prefix="oak-bench-w-") as tmp: + if not args.no_isolate_caches: + _isolate_caches(Path(tmp)) + workdir = Path(args.obo).parent if args.obo else Path(tmp) + ctx = BenchContext(args, workdir) + result = _benchmark_adapter(ADAPTER_SPECS[args.worker], ctx, args.root, args.anc_sample) + print("__RESULT__" + json.dumps(dataclasses.asdict(result)), flush=True) + return 0 + + # -- parent mode: resolve inputs once, fan out one subprocess per adapter - + keys = [k.strip() for k in args.adapters.split(",") if k.strip()] + unknown = [k for k in keys if k not in ADAPTER_SPECS] + if unknown: + parser.error(f"unknown adapters: {unknown}; choose from {sorted(ADAPTER_SPECS)}") + args.keys = keys + + with tempfile.TemporaryDirectory(prefix="oak-bench-") as tmp: + _resolve_inputs(args, Path(args.obo).parent if args.obo else Path(tmp)) + + try: + from importlib.metadata import version + + meta = { + "root": args.root, + "oaklib": version("oaklib"), + "py-horned-owl": version("py-horned-owl"), + "caches": "system" if args.no_isolate_caches else "isolated, per-adapter subprocess", + } + except Exception: + meta = {"root": args.root} + + results: list[AdapterResult] = [] + for key in keys: + print(f"=== {key} ===", flush=True) + r = _run_worker(key, args) + results.append(r) + print(f" load={_fmt(r.load_s)} cold_isa={_fmt(r.cold_s.get('is_a'))} " + f"count={r.count.get('is_a', '-')} rss={_fmt(r.rss_mb, ' MB')} err={r.error}", flush=True) + + table = _render_markdown(results, args.root, meta) + print("\n" + table) + if args.out: + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + Path(args.out).write_text(table) + print(f"\nwrote {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/results/go_adapter_comparison.md b/benchmarks/results/go_adapter_comparison.md new file mode 100644 index 0000000..d1c7515 --- /dev/null +++ b/benchmarks/results/go_adapter_comparison.md @@ -0,0 +1,12 @@ +# OAK adapter comparison for reachable_from closures (`descendants(GO:0005575)`) + +> Regenerated by `just benchmark` / `benchmarks/adapter_benchmark.py`. Timing is informational (cache/network dependent) and each adapter is measured in its own subprocess; correctness is guarded by `tests/test_adapter_parity.py`. + +> Run: root: GO:0005575; oaklib: 0.7.1; py-horned-owl: 1.4.0; caches: isolated, per-adapter subprocess + +| Adapter | Scope | Load | Cold (is_a) | Warm (is_a) | Cold (+part_of) | Ancestor closures/s | is_a count | Peak RAM | +|---|---|---|---|---|---|---|---|---| +| `simpleobo` | single | 2.41s | 14.49s | 0.50s | 0.60s | 207/s | 4075 | 416.88 MB | +| `pronto` | single | 5.94s | 4.83s | 0.65s | 0.75s | 2257/s | 4075 | 617.19 MB | +| `owl` | single | 10.63s | 247.78s | 242.95s | 246.75s | 2/s | 4075 | 2201.61 MB | +| `ubergraph` | merged (remote) | 0.01s | 1.68s | 1.20s | 1.56s | 3/s | 35651 | 226.55 MB | diff --git a/project.justfile b/project.justfile index ce6c509..c14e8a4 100644 --- a/project.justfile +++ b/project.justfile @@ -1,5 +1,14 @@ ## Add your own just recipes here. This is imported by the main justfile. +# ============== Benchmark recipes ============== + +# Benchmark OAK adapters for reachable_from closures; regenerates the results table. +# Timing is informational (cache/network dependent); correctness is guarded by +# tests/test_adapter_parity.py. See benchmarks/README.md. +[group('benchmarks')] +benchmark *ARGS: + uv run python benchmarks/adapter_benchmark.py --out benchmarks/results/go_adapter_comparison.md {{ARGS}} + # ============== Notebook recipes ============== # Execute notebooks with papermill and copy to docs for mkdocs-jupyter to render diff --git a/tests/test_adapter_parity.py b/tests/test_adapter_parity.py new file mode 100644 index 0000000..9938e19 --- /dev/null +++ b/tests/test_adapter_parity.py @@ -0,0 +1,230 @@ +"""Behavioral parity + scope invariants for OAK adapters used in reachable_from. + +These tests guard *behavior* that is independent of caches, network latency, and +timing, so they are safe to run in CI: + +* **Local parity** (offline, default CI): the local file adapters ``simpleobo``, + ``pronto`` and ``owl`` must agree on the ``is_a`` and ``is_a + part_of`` + closures computed over the *same* synthetic ontology. If they diverge, term + validation would accept/reject different terms depending only on which backend + the user happened to configure. +* **Remote scope** (``@integration``, network): OLS under-returns (its closure is + a *subset* of the ground truth -- the truncation behind issue #55), and + Ubergraph is a *merged multi-ontology* graph whose ``descendants`` span every + loaded ontology, so it must be prefix-filtered for single-ontology validation + (issue #58). + +Performance is deliberately **not** asserted here -- it depends on download +caches, ``requests-cache`` state, and network conditions. The timing comparison +lives in ``benchmarks/`` and is regenerated on demand via ``just benchmark``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from oaklib import get_adapter +from oaklib.datamodels.vocabulary import IS_A, PART_OF + +# --------------------------------------------------------------------------- # +# Synthetic ontology: a deterministic ternary is_a tree plus a couple of +# part_of cross-links, so that ``is_a + part_of`` closures are strictly larger +# than ``is_a``-only closures for at least one source node. +# --------------------------------------------------------------------------- # + +# Single-source the part_of CURIE from OAK's vocabulary so the synthetic +# ontology and the query predicates can't drift apart. +PART_OF_CURIE = PART_OF +_N = 40 + + +def _cid(i: int) -> str: + return f"TEST:{i:07d}" + + +# child -> is_a parent (ternary tree rooted at node 0) +_ISA_PARENT: dict[int, int] = {i: (i - 1) // 3 for i in range(1, _N)} +# child -> part_of parent (deliberately into a different is_a branch) +_PARTOF_PARENT: dict[int, int] = {_N - 1: 2, _N - 2: 5} + + +def _write_obo(path: Path) -> None: + lines = ["format-version: 1.2", "ontology: synthetic", ""] + # pronto raises KeyError on an undeclared relationship type, so the part_of + # typedef must be declared even though simpleobo/owl tolerate its absence. + lines += ["[Typedef]", f"id: {PART_OF_CURIE}", "name: part of", ""] + for i in range(_N): + lines += ["[Term]", f"id: {_cid(i)}", f"name: term {i}"] + if i in _ISA_PARENT: + lines.append(f"is_a: {_cid(_ISA_PARENT[i])} ! term {_ISA_PARENT[i]}") + if i in _PARTOF_PARENT: + parent = _PARTOF_PARENT[i] + lines.append(f"relationship: {PART_OF_CURIE} {_cid(parent)} ! term {parent}") + lines.append("") + path.write_text("\n".join(lines)) + + +def _write_ofn(path: Path) -> None: + lines = [ + "Prefix(:=)", + "Prefix(TEST:=)", + "Prefix(BFO:=)", + "Prefix(rdfs:=)", + "Ontology(", + f" Declaration(ObjectProperty({PART_OF_CURIE}))", + ] + for i in range(_N): + lines.append(f" Declaration(Class({_cid(i)}))") + lines.append(f' AnnotationAssertion(rdfs:label {_cid(i)} "term {i}")') + if i in _ISA_PARENT: + lines.append(f" SubClassOf({_cid(i)} {_cid(_ISA_PARENT[i])})") + if i in _PARTOF_PARENT: + parent = _PARTOF_PARENT[i] + lines.append( + f" SubClassOf({_cid(i)} ObjectSomeValuesFrom({PART_OF_CURIE} {_cid(parent)}))" + ) + lines.append(")") + path.write_text("\n".join(lines)) + + +def _closure(adapter: Any, node: str, predicates: list[str], *, up: bool = False) -> set[str]: + method = adapter.ancestors if up else adapter.descendants + return set(method(node, predicates=predicates)) + + +@pytest.fixture(scope="session") +def synthetic_paths(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]: + d = tmp_path_factory.mktemp("synthetic_ontology") + obo, ofn = d / "synthetic.obo", d / "synthetic.ofn" + _write_obo(obo) + _write_ofn(ofn) + return obo, ofn + + +@pytest.fixture(scope="session") +def local_adapters(synthetic_paths: tuple[Path, Path]) -> dict[str, Any]: + """Parse the synthetic ontology once and expose the local file adapters.""" + obo, ofn = synthetic_paths + adapters: dict[str, Any] = { + "simpleobo": get_adapter(f"simpleobo:{obo}"), + "pronto": get_adapter(f"pronto:{obo}"), + } + # The owl adapter needs a recent oaklib + py-horned-owl; skip only that one + # if unavailable rather than failing the whole parity check. + try: + owl = get_adapter(f"owl:{ofn}") + if all(hasattr(owl, m) for m in ("descendants", "ancestors")): + adapters["owl"] = owl + except Exception: # pragma: no cover - environment dependent + pass + return adapters + + +# --------------------------------------------------------------------------- # +# Local parity (offline, runs in default CI) +# --------------------------------------------------------------------------- # + +# Roots chosen so the closure is a non-trivial mix of branches, not the whole tree. +_SOURCE_NODES = [_cid(0), _cid(1), _cid(2)] +_PREDICATE_SETS = { + "is_a": [IS_A], + "is_a+part_of": [IS_A, PART_OF], +} + + +@pytest.mark.parametrize("source_node", _SOURCE_NODES) +@pytest.mark.parametrize("pred_label", list(_PREDICATE_SETS)) +@pytest.mark.parametrize("direction", ["descendants", "ancestors"]) +def test_local_adapters_agree( + local_adapters: dict[str, Any], + source_node: str, + pred_label: str, + direction: str, +) -> None: + """simpleobo / pronto / owl must return identical closures for the same query.""" + adapters = local_adapters + if len(adapters) < 2: + pytest.skip("need >=2 local adapters to check parity") + predicates = _PREDICATE_SETS[pred_label] + up = direction == "ancestors" + results = {name: _closure(ad, source_node, predicates, up=up) for name, ad in adapters.items()} + + reference_name, reference = next(iter(results.items())) + for name, value in results.items(): + assert value == reference, ( + f"{direction}({source_node}, {pred_label}): " + f"{name} disagrees with {reference_name}; " + f"only in {name}={sorted(value - reference)}, " + f"only in {reference_name}={sorted(reference - value)}" + ) + + +def test_part_of_broadens_closure(local_adapters: dict[str, Any]) -> None: + """Sanity check that part_of is actually exercised (not a no-op predicate). + + Node 2 has a part_of child (node 39) that is not one of its is_a descendants, + so ``is_a + part_of`` must be a strict superset of ``is_a`` here -- and every + adapter must agree on the broadened set. + """ + adapters = local_adapters + node = _cid(2) + for name, ad in adapters.items(): + isa = _closure(ad, node, [IS_A]) + both = _closure(ad, node, [IS_A, PART_OF]) + assert isa < both, f"{name}: expected part_of to broaden the closure of {node}" + assert _cid(_N - 1) in both, f"{name}: part_of child missing from closure of {node}" + + +# --------------------------------------------------------------------------- # +# Remote scope invariants (network; opt-in via -m integration) +# --------------------------------------------------------------------------- # + +# cellular_component: deep enough that OLS pagination truncation is observable. +_GO_ROOT = "GO:0005575" + + +def _go_scope(curies: set[str]) -> set[str]: + return {c for c in curies if c.startswith("GO:")} + + +@pytest.mark.integration +def test_ols_descendants_are_a_subset_of_ground_truth() -> None: + """OLS never returns *more* than the true closure (issue #55). + + Truncation makes OLS a strict subset in practice, but we assert only the + robust ``subset`` invariant so the test keeps passing once OAK/OLS is fixed. + Exact counts are reported by the benchmark, not asserted here. + """ + ols = get_adapter("ols:go") + ubergraph = get_adapter("ubergraph:") + # This intentionally exercises the *raw* OAK OLS adapter, not LTV's + # DynamicEnumPlugin._ols_descendants fallback: the point is to characterize + # the adapter-level truncation from #55. If a given oaklib/OLS build doesn't + # implement descendants() or yields nothing, there is no adapter closure to + # compare, so skip rather than fail. + try: + ols_desc = _go_scope(_closure(ols, _GO_ROOT, [IS_A])) + except (NotImplementedError, AttributeError) as exc: + pytest.skip(f"OLS adapter does not support descendants(): {exc}") + if not ols_desc: + pytest.skip("OLS adapter returned no GO descendants (implementation-dependent)") + # Both services also surface imported cross-ontology terms (e.g. CL), so + # compare like-for-like in GO scope -- the closure OLS is supposed to serve. + truth = _go_scope(_closure(ubergraph, _GO_ROOT, [IS_A])) + assert ols_desc <= truth, f"OLS returned terms outside ground truth: {sorted(ols_desc - truth)[:10]}" + + +@pytest.mark.integration +def test_ubergraph_is_a_merged_multi_ontology_graph() -> None: + """Ubergraph descendants of a GO term span multiple ontologies (issue #58). + + Consequence for validation: a reachable_from enum meaning "GO descendants" + must prefix-filter, or it will accept foreign-ontology terms (false accepts). + """ + ubergraph = get_adapter("ubergraph:") + raw = _closure(ubergraph, _GO_ROOT, [IS_A]) + prefixes = {c.split(":")[0] for c in raw} + assert prefixes - {"GO"}, "expected cross-ontology descendants, got GO-only" + assert _go_scope(raw) < raw, "expected the GO-scoped subset to be strictly smaller than the raw set"