From 1e3d536526d747af3ed4565f0d76db1981ae28d3 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Tue, 21 Jul 2026 17:28:57 -0700 Subject: [PATCH 1/4] fix(gfql): index AUTO preserves resident polars frames (polars hop O(E) tax) resolve_engine(AUTO) maps polars frames to PANDAS, so create_index coerced-and-replaced the user's polars frames with pandas copies; every later hop(engine='polars') re-converted the full frames per call (O(E), ~220ms at 4M edges) and the pandas-engine index could never fingerprint- match. Resolve AUTO over resident polars frames to POLARS in the index API only (explicit engines unchanged; NOT the global auto-policy change). dgx (warm median 30, parity vs pandas indexed oracle): polars seeded 1-hop 221 -> 0.80ms at 4M (276x); ladder 0.17/0.35/0.80/0.83ms at 0.25M/1M/4M/16M edges vs previously linear 16.6 -> 1751ms. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- CHANGELOG.md | 1 + graphistry/compute/gfql/index/api.py | 25 ++++++++- .../tests/compute/gfql/index/test_index.py | 55 +++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdb8034c9a..11899bb0ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine-selection docs (pandas / polars / cuDF / polars-gpu)**: New :doc:`Choosing a GFQL Engine ` page — a numbers-first, persona-tested guide to the four interchangeable engines. Adds the one-keyword `engine='polars'` speedup (up to ~38× over pandas on real graphs, no GPU), a motivating warm-median comparison table on real public graphs (LiveJournal 35M / Orkut 117M), a decision matrix (workload shape × size × hardware → engine, with the measured ~10K-edge CPU crossover, GPU-work-bound rule, polars-gpu memory-pressure caveat, and GPU-or-error contract), a cuDF-vs-polars-gpu disambiguation (eager-op vs fused-lazy; cuDF is not deprecated), an honest "when *not* to use Polars" section, the differential-parity guarantee, and a methodology + reproducer-script disclosure. Rewrote the top of `gfql/performance.rst` to lead with the engine comparison (de-marketed the prose), wired the new page into the GFQL toctree + recommended paths, and added Polars/polars-gpu to the engine examples in `gfql/quick.rst` and `gfql/about.rst` (previously only pandas/cuDF were documented). Driven by 4-persona doc user-testing (pandas DS, RAPIDS user, perf engineer, skeptical evaluator). ### Fixed +- **GFQL `gfql_index_all()`/`create_index(engine='auto')` no longer swaps polars frames to pandas — the resident index finally serves `engine='polars'` hops**: `resolve_engine(AUTO)` maps polars input frames to PANDAS (the legacy input-format policy), so the AUTO index build coerced-and-REPLACED the user's polars frames with pandas copies. Every later `hop(engine='polars')` then paid a full-frame pandas→polars re-conversion per call — an O(E) tax (~220ms at 4M edges, ~1.7s at 32M on the C3 ladder) — while the pandas-engine index could never serve it (`get_valid` requires engine match + frame identity, and the per-call conversion broke both). The index layer is already engine-polymorphic (numpy sidecar arrays + polars row-gather), so the index API now resolves AUTO over resident polars frames to POLARS and indexes them in place: frames preserved (identity intact), polars-engine index resident and valid. Measured on dgx (warm median of 30, parity-checked vs the pandas indexed oracle every rung): polars seeded 1-hop 221→0.80ms at 4M edges (276×), and the ladder goes from linear (16.6→1751ms over 0.25M→32M) to sub-ms at every rung (0.17/0.35/0.80/0.83ms at 0.25M/1M/4M/16M). Explicit engines are honored unchanged (`gfql_index_all(engine='pandas')` still coerces). Pinned in `test_index.py::TestIndexAutoPreservesPolarsFrames` (frames+index engine preserved, index-engagement spy with pandas-oracle parity, explicit-engine coercion contract). - **GFQL polars engine natively runs `UNWIND` of a carried `collect()` list column**: `... WITH collect(x) AS xs UNWIND xs AS y RETURN ...` (exploding a list-valued column produced by `collect()`, the row-pipeline analogue of the IC6 unwind shape) previously declined on polars — `unwind_polars` only accepted a scalar literal list. Now explodes a `List`-dtype column reference (`with_columns` copy → `filter(list.len() > 0)` → `explode`), matching the pandas oracle exactly: empty-list/null cells → 0 rows, nulls *within* a list survive, source column retained; the pre-filter also makes it independent of the polars-2.0 `empty_as_null` default. Non-list columns, name collisions, unknown identifiers, and nested-list literals still decline (honest NIE, no pandas bridge). Differential fuzz vs pandas over ~3500 collect→UNWIND→{RETURN, grouped, filtered, ORDER BY, aggregate, nested-unwind} queries: 0 disagreements. (The `MATCH ... WITH collect(..) UNWIND .. MATCH` form — IC6 with a trailing re-traversal — is compile-rewritten into WITH→MATCH reentry, a separate path.) Tests in `test_engine_polars_row_pipeline.py`. - **GFQL polars engine natively runs the LDBC IC11/IC6 undirected variable-length bindings table (`-[*1..k]-`), unblocking the cross-alias `WHERE NOT(person=friend)` clause**: a bounded UNDIRECTED variable-length MATCH that materializes a bindings row table (`rows(binding_ops=...)`, emitted whenever a downstream clause — e.g. a cross-alias same-path `WHERE NOT a = b` / `a <> b` — references two node aliases on the path) previously declined on polars (`binding_rows_polars` hard-NIE'd every undirected multihop), while pandas handled it. This was the single residual blocking the official LDBC IC11/IC6 queries on polars (the cross-alias WHERE lowering itself was already native — directed var-length + `WHERE NOT a = b` already matched pandas). `binding_rows_polars` now supports undirected var-length with **`min_hops == 1`** via a doubled-pair join with immediate-backtrack avoidance, an exact port of the pandas oracle `_gfql_multihop_binding_rows` (`avoid_immediate_backtrack=True`): a `__prev__` marker (seeded null, dtype-matched to the id column) drops immediate backtracks each hop (Kleene mask: null prev kept), and the step-pair set reproduces pandas' edge multiplicity exactly — each non-loop edge contributes each directed orientation TWICE (so a length-1 pair appears x2, length-2 x4) while self-loops contribute `(u,u)` x2 only (not double-counted). The multiplicity rule was derived by instrumenting pandas' `step_pairs` (which flow from the var-length `edge_op.execute` hop + `orient_edges`), NOT by reading code alone. Scoped to `min_hops == 1` because that is where the raw-edge reconstruction provably matches pandas: every edge is trivially a length-1 path so the var-length hop's backward pruning removes nothing; `min_hops == 0` (zero-hop, undoubled) and `min_hops >= 2` (backward-pruned / long-walk divergence) still **decline with an honest `NotImplementedError`** rather than risk silent-wrong multiplicities. Differential fuzz vs the pandas oracle over ~2500 random graphs (self-loops, parallel + antiparallel edges, `*1..2`…`*1..5`, all WHERE/RETURN variants): 0 disagreements; the out-of-scope windows decline (never diverge). Tests in `test_engine_polars_binding_rows.py` (parity + multiplicity/backtrack/self-loop/string-id pins + `*0..2`/`*2..3`/`*2..2` decline pins). - **GFQL polars engine natively runs whole-entity `count(DISTINCT n)` / identity aggregation (#1709)**: `g.gfql("MATCH (a {..}) RETURN count(DISTINCT a) AS c")` and its grouped/filtered/traversal variants (`(a)-[]->(b) RETURN count(DISTINCT b)`) previously declined on polars — the Cypher aggregation lowers `count(DISTINCT b)` to the `__gfql_node_id__` identity sentinel, and the polars `lower_expr` `Identifier` branch resolved only the prefixed `alias.__gfql_node_id__` form, not the bare sentinel, so it NIE'd. Fix threads the graph node-id column through a `_NODE_ID` contextvar (published alongside the schema in `_lower_with_schema`, wired through the four callers) so the bare sentinel resolves to the id column **only if present**, else declines (never invents/mis-resolves a column). Differential fuzz vs the pandas oracle: 600 + 300 cases, 0 disagreements. Deliberately still honest-NIE (verified declines, not silent-wrong): whole-entity `collect(b)` (list-of-entities repr not ported, cf #1650) and multi-alias binding-table identity aggregation (bare id absent on connected-pattern binding tables — adjacent to #1273). Tests in `test_engine_polars_row_pipeline.py`. diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index 0ed74c3d4c..02472b2dc9 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -111,13 +111,34 @@ def _check_column(column: Optional[str], expected: str, kind: IndexKind) -> None ) +def _resolve_index_engine(engine: EngineAbstractType, g: Plottable) -> Engine: + """Engine for index build/validation: AUTO keeps resident polars frames polars. + + ``resolve_engine(AUTO)`` maps polars input frames to PANDAS (the legacy + input-format policy), which would make ``create_index`` coerce-and-REPLACE the + user's polars frames with pandas copies — and then every later + ``engine='polars'`` query pays a full-frame pandas->polars re-conversion, an + O(E) tax per call that dwarfs the indexed hop itself. The index layer is + engine-polymorphic (numpy sidecar arrays + polars row-gather), so when the + caller said AUTO and the resident frames are polars, index them in place. + Explicit engines are honored unchanged. + """ + eng = resolve_engine(engine, g) + abstract = EngineAbstract(engine) if isinstance(engine, str) else engine + if abstract == EngineAbstract.AUTO and eng == Engine.PANDAS: + df = g._edges if g._edges is not None else g._nodes + if df is not None and 'polars' in str(type(df).__module__): + return Engine.POLARS + return eng + + def _is_resident_index_valid( g: Plottable, kind: IndexKind, engine: EngineAbstractType = EngineAbstract.AUTO, ) -> bool: """True when a resident index still matches the current graph frames.""" - eng = resolve_engine(engine, g) + eng = _resolve_index_engine(engine, g) registry = get_registry(g) if kind in ADJ_KINDS: src, dst = g._source, g._destination @@ -148,7 +169,7 @@ def create_index( O(E log E) once, amortized over later seeded queries. """ from dataclasses import replace - eng = resolve_engine(engine, g) + eng = _resolve_index_engine(engine, g) # Build over frames already in the target engine so the index arrays land on # the right backend (cupy for cudf, numpy otherwise). No-op when already in-engine. from graphistry.compute.ComputeMixin import _coerce_input_formats diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 0303ce0495..652b8ca674 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -844,3 +844,58 @@ def test_hop_dtype_mismatch_edge_match_matches_scan_error(typed_graph, engine): if engine == "pandas": from graphistry.compute.exceptions import GFQLSchemaError assert isinstance(base_err.value, GFQLSchemaError) + + +class TestIndexAutoPreservesPolarsFrames: + """gfql_index_all(engine='auto') on a polars graph must index in place, NOT + coerce-and-replace the frames with pandas. Regression pin for the C3 "polars + hop is O(E)" mystery: resolve_engine(AUTO) maps polars->PANDAS, so create_index + used to swap the frames to pandas, and every later hop(engine='polars') paid a + full-frame pandas->polars conversion per call (~220ms at 4M edges vs a ~1ms + indexed hop) while the pandas-engine index could never fingerprint-match.""" + + def _pl_graph(self): + pl = pytest.importorskip("polars") + ndf = pl.DataFrame({"id": [0, 1, 2, 3], "type": ["a", "b", "a", "b"]}) + edf = pl.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 3]}) + return graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + + def test_auto_keeps_polars_frames_and_polars_index(self): + from graphistry.Engine import Engine + g = self._pl_graph() + gi = g.gfql_index_all() # AUTO + assert "polars" in type(gi._nodes).__module__ + assert "polars" in type(gi._edges).__module__ + # frames indexed in place: identity preserved so get_valid's `is` check holds + assert gi._edges is g._edges + from graphistry.compute.gfql.index import show_indexes + idx = show_indexes(gi) + assert set(idx["engine"]) == {Engine.POLARS.value} + assert idx["valid"].all() + + def test_auto_polars_hop_engages_index(self, monkeypatch): + # big enough that one seed passes the frontier-fraction cost gate + pl = pytest.importorskip("polars") + import graphistry.compute.gfql.index.api as index_api + rng = np.random.default_rng(0) + n_nodes, m = 2000, 12000 + ndf = pl.DataFrame({"id": np.arange(n_nodes)}) + edf = pl.DataFrame({"src": rng.integers(0, n_nodes, m), "dst": rng.integers(0, n_nodes, m)}) + g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst").gfql_index_all() + hits = [] + orig = index_api.index_seeded_hop + monkeypatch.setattr(index_api, "index_seeded_hop", + lambda *a, **k: (hits.append(1), orig(*a, **k))[1]) + r = g.hop(nodes=pl.DataFrame({"id": [7]}), hops=1, direction="forward", engine="polars") + assert hits, "resident polars index did not serve the polars hop" + # parity vs the pandas indexed oracle on the same data + gp = graphistry.nodes(ndf.to_pandas(), "id").edges(edf.to_pandas(), "src", "dst").gfql_index_all() + rp = gp.hop(nodes=pd.DataFrame({"id": [7]}), hops=1, direction="forward", engine="pandas") + assert sorted(r._nodes.get_column("id").to_list()) == sorted(rp._nodes["id"].tolist()) + + def test_explicit_engine_still_coerces(self): + g = self._pl_graph() + gi = g.gfql_index_all(engine="pandas") + assert isinstance(gi._edges, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} From f876bb65218367f29876829e703d28e66d8e1435 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Tue, 21 Jul 2026 18:27:10 -0700 Subject: [PATCH 2/4] fix(gfql): index AUTO gate is eager-polars-only; nodeless graphs materialize-first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings (plans/review-pr-1767/review.md): B1 edges-only polars graph crashed in the NODE_ID build (materialize under AUTO synthesizes pandas; polars gathers ran on a pandas frame) — gfql_index_all now materializes + engine-aligns BEFORE any build so all indexes land valid, and create_index re-aligns for direct calls. M1 LazyFrame frames under AUTO crashed — the gate now requires EAGER polars DataFrames on ALL present frames (LazyFrame/mixed keep the legacy pandas path). Spy test asserts the index SERVED (non-None), not merely was called. AUTO-query trade-off documented in CHANGELOG (deferred to #1743 by policy hold). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- CHANGELOG.md | 2 +- graphistry/compute/gfql/index/api.py | 19 ++++++++++-- .../tests/compute/gfql/index/test_index.py | 31 +++++++++++++++++-- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11899bb0ec..1dbd8e3467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine-selection docs (pandas / polars / cuDF / polars-gpu)**: New :doc:`Choosing a GFQL Engine ` page — a numbers-first, persona-tested guide to the four interchangeable engines. Adds the one-keyword `engine='polars'` speedup (up to ~38× over pandas on real graphs, no GPU), a motivating warm-median comparison table on real public graphs (LiveJournal 35M / Orkut 117M), a decision matrix (workload shape × size × hardware → engine, with the measured ~10K-edge CPU crossover, GPU-work-bound rule, polars-gpu memory-pressure caveat, and GPU-or-error contract), a cuDF-vs-polars-gpu disambiguation (eager-op vs fused-lazy; cuDF is not deprecated), an honest "when *not* to use Polars" section, the differential-parity guarantee, and a methodology + reproducer-script disclosure. Rewrote the top of `gfql/performance.rst` to lead with the engine comparison (de-marketed the prose), wired the new page into the GFQL toctree + recommended paths, and added Polars/polars-gpu to the engine examples in `gfql/quick.rst` and `gfql/about.rst` (previously only pandas/cuDF were documented). Driven by 4-persona doc user-testing (pandas DS, RAPIDS user, perf engineer, skeptical evaluator). ### Fixed -- **GFQL `gfql_index_all()`/`create_index(engine='auto')` no longer swaps polars frames to pandas — the resident index finally serves `engine='polars'` hops**: `resolve_engine(AUTO)` maps polars input frames to PANDAS (the legacy input-format policy), so the AUTO index build coerced-and-REPLACED the user's polars frames with pandas copies. Every later `hop(engine='polars')` then paid a full-frame pandas→polars re-conversion per call — an O(E) tax (~220ms at 4M edges, ~1.7s at 32M on the C3 ladder) — while the pandas-engine index could never serve it (`get_valid` requires engine match + frame identity, and the per-call conversion broke both). The index layer is already engine-polymorphic (numpy sidecar arrays + polars row-gather), so the index API now resolves AUTO over resident polars frames to POLARS and indexes them in place: frames preserved (identity intact), polars-engine index resident and valid. Measured on dgx (warm median of 30, parity-checked vs the pandas indexed oracle every rung): polars seeded 1-hop 221→0.80ms at 4M edges (276×), and the ladder goes from linear (16.6→1751ms over 0.25M→32M) to sub-ms at every rung (0.17/0.35/0.80/0.83ms at 0.25M/1M/4M/16M). Explicit engines are honored unchanged (`gfql_index_all(engine='pandas')` still coerces). Pinned in `test_index.py::TestIndexAutoPreservesPolarsFrames` (frames+index engine preserved, index-engagement spy with pandas-oracle parity, explicit-engine coercion contract). +- **GFQL `gfql_index_all()`/`create_index(engine='auto')` no longer swaps polars frames to pandas — the resident index finally serves `engine='polars'` hops**: `resolve_engine(AUTO)` maps polars input frames to PANDAS (the legacy input-format policy), so the AUTO index build coerced-and-REPLACED the user's polars frames with pandas copies. Every later `hop(engine='polars')` then paid a full-frame pandas→polars re-conversion per call — an O(E) tax (~220ms at 4M edges, ~1.7s at 32M on the C3 ladder) — while the pandas-engine index could never serve it (`get_valid` requires engine match + frame identity, and the per-call conversion broke both). The index layer is already engine-polymorphic (numpy sidecar arrays + polars row-gather), so the index API now resolves AUTO over resident polars frames to POLARS and indexes them in place: frames preserved (identity intact), polars-engine index resident and valid. Measured on dgx (warm median of 30, parity-checked vs the pandas indexed oracle every rung): polars seeded 1-hop 221→0.80ms at 4M edges (276×), and the ladder goes from linear (16.6→1751ms over 0.25M→32M) to sub-ms at every rung (0.17/0.35/0.80/0.83ms at 0.25M/1M/4M/16M). Explicit engines are honored unchanged (`gfql_index_all(engine='pandas')` still coerces); LazyFrame or mixed-engine frames keep the legacy pandas path (eager-polars-only gate); edges-only polars graphs materialize + engine-align BEFORE index builds so all three indexes land valid. Known trade-off, deliberate: `engine='auto'` QUERIES on a polars graph still resolve pandas (the #1743 routing policy is unchanged), so they re-coerce per call and scan — the accelerated surface is explicit `engine='polars'`; AUTO-first users can `gfql_index_all(engine='pandas')` to keep the old behavior. Pinned in `test_index.py::TestIndexAutoPreservesPolarsFrames` (frames+index engine preserved, index-engagement spy with pandas-oracle parity, explicit-engine coercion contract). - **GFQL polars engine natively runs `UNWIND` of a carried `collect()` list column**: `... WITH collect(x) AS xs UNWIND xs AS y RETURN ...` (exploding a list-valued column produced by `collect()`, the row-pipeline analogue of the IC6 unwind shape) previously declined on polars — `unwind_polars` only accepted a scalar literal list. Now explodes a `List`-dtype column reference (`with_columns` copy → `filter(list.len() > 0)` → `explode`), matching the pandas oracle exactly: empty-list/null cells → 0 rows, nulls *within* a list survive, source column retained; the pre-filter also makes it independent of the polars-2.0 `empty_as_null` default. Non-list columns, name collisions, unknown identifiers, and nested-list literals still decline (honest NIE, no pandas bridge). Differential fuzz vs pandas over ~3500 collect→UNWIND→{RETURN, grouped, filtered, ORDER BY, aggregate, nested-unwind} queries: 0 disagreements. (The `MATCH ... WITH collect(..) UNWIND .. MATCH` form — IC6 with a trailing re-traversal — is compile-rewritten into WITH→MATCH reentry, a separate path.) Tests in `test_engine_polars_row_pipeline.py`. - **GFQL polars engine natively runs the LDBC IC11/IC6 undirected variable-length bindings table (`-[*1..k]-`), unblocking the cross-alias `WHERE NOT(person=friend)` clause**: a bounded UNDIRECTED variable-length MATCH that materializes a bindings row table (`rows(binding_ops=...)`, emitted whenever a downstream clause — e.g. a cross-alias same-path `WHERE NOT a = b` / `a <> b` — references two node aliases on the path) previously declined on polars (`binding_rows_polars` hard-NIE'd every undirected multihop), while pandas handled it. This was the single residual blocking the official LDBC IC11/IC6 queries on polars (the cross-alias WHERE lowering itself was already native — directed var-length + `WHERE NOT a = b` already matched pandas). `binding_rows_polars` now supports undirected var-length with **`min_hops == 1`** via a doubled-pair join with immediate-backtrack avoidance, an exact port of the pandas oracle `_gfql_multihop_binding_rows` (`avoid_immediate_backtrack=True`): a `__prev__` marker (seeded null, dtype-matched to the id column) drops immediate backtracks each hop (Kleene mask: null prev kept), and the step-pair set reproduces pandas' edge multiplicity exactly — each non-loop edge contributes each directed orientation TWICE (so a length-1 pair appears x2, length-2 x4) while self-loops contribute `(u,u)` x2 only (not double-counted). The multiplicity rule was derived by instrumenting pandas' `step_pairs` (which flow from the var-length `edge_op.execute` hop + `orient_edges`), NOT by reading code alone. Scoped to `min_hops == 1` because that is where the raw-edge reconstruction provably matches pandas: every edge is trivially a length-1 path so the var-length hop's backward pruning removes nothing; `min_hops == 0` (zero-hop, undoubled) and `min_hops >= 2` (backward-pruned / long-walk divergence) still **decline with an honest `NotImplementedError`** rather than risk silent-wrong multiplicities. Differential fuzz vs the pandas oracle over ~2500 random graphs (self-loops, parallel + antiparallel edges, `*1..2`…`*1..5`, all WHERE/RETURN variants): 0 disagreements; the out-of-scope windows decline (never diverge). Tests in `test_engine_polars_binding_rows.py` (parity + multiplicity/backtrack/self-loop/string-id pins + `*0..2`/`*2..3`/`*2..2` decline pins). - **GFQL polars engine natively runs whole-entity `count(DISTINCT n)` / identity aggregation (#1709)**: `g.gfql("MATCH (a {..}) RETURN count(DISTINCT a) AS c")` and its grouped/filtered/traversal variants (`(a)-[]->(b) RETURN count(DISTINCT b)`) previously declined on polars — the Cypher aggregation lowers `count(DISTINCT b)` to the `__gfql_node_id__` identity sentinel, and the polars `lower_expr` `Identifier` branch resolved only the prefixed `alias.__gfql_node_id__` form, not the bare sentinel, so it NIE'd. Fix threads the graph node-id column through a `_NODE_ID` contextvar (published alongside the schema in `_lower_with_schema`, wired through the four callers) so the bare sentinel resolves to the id column **only if present**, else declines (never invents/mis-resolves a column). Differential fuzz vs the pandas oracle: 600 + 300 cases, 0 disagreements. Deliberately still honest-NIE (verified declines, not silent-wrong): whole-entity `collect(b)` (list-of-entities repr not ported, cf #1650) and multi-alias binding-table identity aggregation (bare id absent on connected-pattern binding tables — adjacent to #1273). Tests in `test_engine_polars_row_pipeline.py`. diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index 02472b2dc9..86e315c288 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -123,11 +123,15 @@ def _resolve_index_engine(engine: EngineAbstractType, g: Plottable) -> Engine: caller said AUTO and the resident frames are polars, index them in place. Explicit engines are honored unchanged. """ + def _is_eager_polars(df: object) -> bool: + # EAGER frames only: LazyFrame passes the module check but no index/coercion + # path can gather rows from it — those graphs keep the legacy pandas path. + return df is not None and 'polars' in str(type(df).__module__) and type(df).__name__ == 'DataFrame' eng = resolve_engine(engine, g) abstract = EngineAbstract(engine) if isinstance(engine, str) else engine if abstract == EngineAbstract.AUTO and eng == Engine.PANDAS: - df = g._edges if g._edges is not None else g._nodes - if df is not None and 'polars' in str(type(df).__module__): + frames = [f for f in (g._edges, g._nodes) if f is not None] + if frames and all(_is_eager_polars(f) for f in frames): return Engine.POLARS return eng @@ -193,6 +197,11 @@ def create_index( if kind == NODE_ID: g2 = g.materialize_nodes() if g._nodes is None else g + if g._nodes is None: + # materialize_nodes synthesizes under its own AUTO rules (pandas for polars + # edges), so re-align the frames with the engine this index builds in — + # otherwise build_node_id_index runs polars gathers on a pandas frame. + g2 = _coerce_input_formats(g2, eng) node_col = g2._node assert node_col is not None and g2._nodes is not None _check_column(column, node_col, kind) @@ -277,6 +286,12 @@ def gfql_index_all(g: Plottable, per-id semantics), so this convenience SKIPS it rather than raising — seeded traversal stays correct via the un-indexed node materialization path. (Explicit ``create_index(NODE_ID)`` still raises, since the caller asked for it specifically.)""" + if g._nodes is None: + # Materialize + engine-align BEFORE any index builds: materialize_nodes + # synthesizes (and can rebind edges) under its own AUTO rules, which would + # invalidate adjacency indexes built first (identity fingerprint). + from graphistry.compute.ComputeMixin import _coerce_input_formats + g = _coerce_input_formats(g.materialize_nodes(), _resolve_index_engine(engine, g)) g = gfql_index_edges(g, "both", engine=engine) try: g = create_index(g, NODE_ID, engine=engine) diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 652b8ca674..fa8c545bd5 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -884,10 +884,14 @@ def test_auto_polars_hop_engages_index(self, monkeypatch): g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst").gfql_index_all() hits = [] orig = index_api.index_seeded_hop - monkeypatch.setattr(index_api, "index_seeded_hop", - lambda *a, **k: (hits.append(1), orig(*a, **k))[1]) + + def spy(*a, **k): + out = orig(*a, **k) + hits.append(out is not None) # None = scan fallback; only a real serve counts + return out + monkeypatch.setattr(index_api, "index_seeded_hop", spy) r = g.hop(nodes=pl.DataFrame({"id": [7]}), hops=1, direction="forward", engine="polars") - assert hits, "resident polars index did not serve the polars hop" + assert any(hits), "resident polars index did not SERVE the polars hop (call alone is not service)" # parity vs the pandas indexed oracle on the same data gp = graphistry.nodes(ndf.to_pandas(), "id").edges(edf.to_pandas(), "src", "dst").gfql_index_all() rp = gp.hop(nodes=pd.DataFrame({"id": [7]}), hops=1, direction="forward", engine="pandas") @@ -899,3 +903,24 @@ def test_explicit_engine_still_coerces(self): assert isinstance(gi._edges, pd.DataFrame) from graphistry.compute.gfql.index import show_indexes assert set(show_indexes(gi)["engine"]) == {"pandas"} + + + def test_nodeless_polars_graph_indexes(self): + # B1 pin: edges-only polars graph — materialize_nodes synthesizes under its + # own AUTO rules; the NODE_ID build must re-align engines, not crash. + pl = pytest.importorskip("polars") + g = graphistry.edges(pl.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 3]}), "src", "dst") + gi = g.gfql_index_all() + from graphistry.compute.gfql.index import show_indexes + idx = show_indexes(gi) + assert len(idx) >= 2 and idx["valid"].all() + + def test_lazyframe_auto_keeps_legacy_pandas_path(self): + # M1 pin: LazyFrame frames under AUTO must coerce to pandas like master + # (the eager-polars-only gate), never crash in a polars index build. + pl = pytest.importorskip("polars") + g = graphistry.nodes(pl.LazyFrame({"id": [0, 1, 2]}), "id").edges( + pl.LazyFrame({"src": [0, 1], "dst": [1, 2]}), "src", "dst") + gi = g.gfql_index_all() + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} From 26026e4c5b6e00b49da6ff6cecc357d8d864e6a7 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Wed, 22 Jul 2026 03:09:38 -0700 Subject: [PATCH 3/4] fix(gfql): keep index AUTO source-native --- CHANGELOG.md | 2 +- graphistry/compute/gfql/index/api.py | 19 ++--- .../tests/compute/gfql/index/test_index.py | 70 +++++++++++++++++-- 3 files changed, 70 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dbd8e3467..d9057d8fcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine-selection docs (pandas / polars / cuDF / polars-gpu)**: New :doc:`Choosing a GFQL Engine ` page — a numbers-first, persona-tested guide to the four interchangeable engines. Adds the one-keyword `engine='polars'` speedup (up to ~38× over pandas on real graphs, no GPU), a motivating warm-median comparison table on real public graphs (LiveJournal 35M / Orkut 117M), a decision matrix (workload shape × size × hardware → engine, with the measured ~10K-edge CPU crossover, GPU-work-bound rule, polars-gpu memory-pressure caveat, and GPU-or-error contract), a cuDF-vs-polars-gpu disambiguation (eager-op vs fused-lazy; cuDF is not deprecated), an honest "when *not* to use Polars" section, the differential-parity guarantee, and a methodology + reproducer-script disclosure. Rewrote the top of `gfql/performance.rst` to lead with the engine comparison (de-marketed the prose), wired the new page into the GFQL toctree + recommended paths, and added Polars/polars-gpu to the engine examples in `gfql/quick.rst` and `gfql/about.rst` (previously only pandas/cuDF were documented). Driven by 4-persona doc user-testing (pandas DS, RAPIDS user, perf engineer, skeptical evaluator). ### Fixed -- **GFQL `gfql_index_all()`/`create_index(engine='auto')` no longer swaps polars frames to pandas — the resident index finally serves `engine='polars'` hops**: `resolve_engine(AUTO)` maps polars input frames to PANDAS (the legacy input-format policy), so the AUTO index build coerced-and-REPLACED the user's polars frames with pandas copies. Every later `hop(engine='polars')` then paid a full-frame pandas→polars re-conversion per call — an O(E) tax (~220ms at 4M edges, ~1.7s at 32M on the C3 ladder) — while the pandas-engine index could never serve it (`get_valid` requires engine match + frame identity, and the per-call conversion broke both). The index layer is already engine-polymorphic (numpy sidecar arrays + polars row-gather), so the index API now resolves AUTO over resident polars frames to POLARS and indexes them in place: frames preserved (identity intact), polars-engine index resident and valid. Measured on dgx (warm median of 30, parity-checked vs the pandas indexed oracle every rung): polars seeded 1-hop 221→0.80ms at 4M edges (276×), and the ladder goes from linear (16.6→1751ms over 0.25M→32M) to sub-ms at every rung (0.17/0.35/0.80/0.83ms at 0.25M/1M/4M/16M). Explicit engines are honored unchanged (`gfql_index_all(engine='pandas')` still coerces); LazyFrame or mixed-engine frames keep the legacy pandas path (eager-polars-only gate); edges-only polars graphs materialize + engine-align BEFORE index builds so all three indexes land valid. Known trade-off, deliberate: `engine='auto'` QUERIES on a polars graph still resolve pandas (the #1743 routing policy is unchanged), so they re-coerce per call and scan — the accelerated surface is explicit `engine='polars'`; AUTO-first users can `gfql_index_all(engine='pandas')` to keep the old behavior. Pinned in `test_index.py::TestIndexAutoPreservesPolarsFrames` (frames+index engine preserved, index-engagement spy with pandas-oracle parity, explicit-engine coercion contract). +- **GFQL resident-index construction preserves eager polars node+edge frames under `engine='auto'`**: `gfql_index_all()`/`create_index()` now select polars only when both resident frames are eager polars, avoiding the legacy pandas conversion and allowing later explicit `engine='polars'` hops to serve the resident index. Pandas and cuDF remain source-native; mixed, LazyFrame, and missing-frame inputs keep the legacy fallback; explicit engines and the global AUTO query-routing policy are unchanged. Behavioral coverage pins frame/index preservation, actual index service with pandas-oracle parity, and every fallback boundary. - **GFQL polars engine natively runs `UNWIND` of a carried `collect()` list column**: `... WITH collect(x) AS xs UNWIND xs AS y RETURN ...` (exploding a list-valued column produced by `collect()`, the row-pipeline analogue of the IC6 unwind shape) previously declined on polars — `unwind_polars` only accepted a scalar literal list. Now explodes a `List`-dtype column reference (`with_columns` copy → `filter(list.len() > 0)` → `explode`), matching the pandas oracle exactly: empty-list/null cells → 0 rows, nulls *within* a list survive, source column retained; the pre-filter also makes it independent of the polars-2.0 `empty_as_null` default. Non-list columns, name collisions, unknown identifiers, and nested-list literals still decline (honest NIE, no pandas bridge). Differential fuzz vs pandas over ~3500 collect→UNWIND→{RETURN, grouped, filtered, ORDER BY, aggregate, nested-unwind} queries: 0 disagreements. (The `MATCH ... WITH collect(..) UNWIND .. MATCH` form — IC6 with a trailing re-traversal — is compile-rewritten into WITH→MATCH reentry, a separate path.) Tests in `test_engine_polars_row_pipeline.py`. - **GFQL polars engine natively runs the LDBC IC11/IC6 undirected variable-length bindings table (`-[*1..k]-`), unblocking the cross-alias `WHERE NOT(person=friend)` clause**: a bounded UNDIRECTED variable-length MATCH that materializes a bindings row table (`rows(binding_ops=...)`, emitted whenever a downstream clause — e.g. a cross-alias same-path `WHERE NOT a = b` / `a <> b` — references two node aliases on the path) previously declined on polars (`binding_rows_polars` hard-NIE'd every undirected multihop), while pandas handled it. This was the single residual blocking the official LDBC IC11/IC6 queries on polars (the cross-alias WHERE lowering itself was already native — directed var-length + `WHERE NOT a = b` already matched pandas). `binding_rows_polars` now supports undirected var-length with **`min_hops == 1`** via a doubled-pair join with immediate-backtrack avoidance, an exact port of the pandas oracle `_gfql_multihop_binding_rows` (`avoid_immediate_backtrack=True`): a `__prev__` marker (seeded null, dtype-matched to the id column) drops immediate backtracks each hop (Kleene mask: null prev kept), and the step-pair set reproduces pandas' edge multiplicity exactly — each non-loop edge contributes each directed orientation TWICE (so a length-1 pair appears x2, length-2 x4) while self-loops contribute `(u,u)` x2 only (not double-counted). The multiplicity rule was derived by instrumenting pandas' `step_pairs` (which flow from the var-length `edge_op.execute` hop + `orient_edges`), NOT by reading code alone. Scoped to `min_hops == 1` because that is where the raw-edge reconstruction provably matches pandas: every edge is trivially a length-1 path so the var-length hop's backward pruning removes nothing; `min_hops == 0` (zero-hop, undoubled) and `min_hops >= 2` (backward-pruned / long-walk divergence) still **decline with an honest `NotImplementedError`** rather than risk silent-wrong multiplicities. Differential fuzz vs the pandas oracle over ~2500 random graphs (self-loops, parallel + antiparallel edges, `*1..2`…`*1..5`, all WHERE/RETURN variants): 0 disagreements; the out-of-scope windows decline (never diverge). Tests in `test_engine_polars_binding_rows.py` (parity + multiplicity/backtrack/self-loop/string-id pins + `*0..2`/`*2..3`/`*2..2` decline pins). - **GFQL polars engine natively runs whole-entity `count(DISTINCT n)` / identity aggregation (#1709)**: `g.gfql("MATCH (a {..}) RETURN count(DISTINCT a) AS c")` and its grouped/filtered/traversal variants (`(a)-[]->(b) RETURN count(DISTINCT b)`) previously declined on polars — the Cypher aggregation lowers `count(DISTINCT b)` to the `__gfql_node_id__` identity sentinel, and the polars `lower_expr` `Identifier` branch resolved only the prefixed `alias.__gfql_node_id__` form, not the bare sentinel, so it NIE'd. Fix threads the graph node-id column through a `_NODE_ID` contextvar (published alongside the schema in `_lower_with_schema`, wired through the four callers) so the bare sentinel resolves to the id column **only if present**, else declines (never invents/mis-resolves a column). Differential fuzz vs the pandas oracle: 600 + 300 cases, 0 disagreements. Deliberately still honest-NIE (verified declines, not silent-wrong): whole-entity `collect(b)` (list-of-entities repr not ported, cf #1650) and multi-alias binding-table identity aggregation (bare id absent on connected-pattern binding tables — adjacent to #1273). Tests in `test_engine_polars_row_pipeline.py`. diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index 86e315c288..1571df6151 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -130,8 +130,12 @@ def _is_eager_polars(df: object) -> bool: eng = resolve_engine(engine, g) abstract = EngineAbstract(engine) if isinstance(engine, str) else engine if abstract == EngineAbstract.AUTO and eng == Engine.PANDAS: - frames = [f for f in (g._edges, g._nodes) if f is not None] - if frames and all(_is_eager_polars(f) for f in frames): + if ( + g._edges is not None + and g._nodes is not None + and _is_eager_polars(g._edges) + and _is_eager_polars(g._nodes) + ): return Engine.POLARS return eng @@ -197,11 +201,6 @@ def create_index( if kind == NODE_ID: g2 = g.materialize_nodes() if g._nodes is None else g - if g._nodes is None: - # materialize_nodes synthesizes under its own AUTO rules (pandas for polars - # edges), so re-align the frames with the engine this index builds in — - # otherwise build_node_id_index runs polars gathers on a pandas frame. - g2 = _coerce_input_formats(g2, eng) node_col = g2._node assert node_col is not None and g2._nodes is not None _check_column(column, node_col, kind) @@ -286,12 +285,6 @@ def gfql_index_all(g: Plottable, per-id semantics), so this convenience SKIPS it rather than raising — seeded traversal stays correct via the un-indexed node materialization path. (Explicit ``create_index(NODE_ID)`` still raises, since the caller asked for it specifically.)""" - if g._nodes is None: - # Materialize + engine-align BEFORE any index builds: materialize_nodes - # synthesizes (and can rebind edges) under its own AUTO rules, which would - # invalidate adjacency indexes built first (identity fingerprint). - from graphistry.compute.ComputeMixin import _coerce_input_formats - g = _coerce_input_formats(g.materialize_nodes(), _resolve_index_engine(engine, g)) g = gfql_index_edges(g, "both", engine=engine) try: g = create_index(g, NODE_ID, engine=engine) diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index fa8c545bd5..8ae9fb9ff5 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -904,16 +904,26 @@ def test_explicit_engine_still_coerces(self): from graphistry.compute.gfql.index import show_indexes assert set(show_indexes(gi)["engine"]) == {"pandas"} - - def test_nodeless_polars_graph_indexes(self): - # B1 pin: edges-only polars graph — materialize_nodes synthesizes under its - # own AUTO rules; the NODE_ID build must re-align engines, not crash. + def test_edges_only_polars_keeps_legacy_pandas_path(self): pl = pytest.importorskip("polars") - g = graphistry.edges(pl.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 3]}), "src", "dst") + g = graphistry.edges( + pl.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 3]}), "src", "dst" + ) gi = g.gfql_index_all() + assert isinstance(gi._nodes, pd.DataFrame) + assert isinstance(gi._edges, pd.DataFrame) from graphistry.compute.gfql.index import show_indexes - idx = show_indexes(gi) - assert len(idx) >= 2 and idx["valid"].all() + assert set(show_indexes(gi)["engine"]) == {"pandas"} + + def test_nodes_only_polars_keeps_legacy_pandas_path(self): + pl = pytest.importorskip("polars") + gi = ( + graphistry.nodes(pl.DataFrame({"id": [0, 1, 2]}), "id") + .create_index("node_id") + ) + assert isinstance(gi._nodes, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} def test_lazyframe_auto_keeps_legacy_pandas_path(self): # M1 pin: LazyFrame frames under AUTO must coerce to pandas like master @@ -924,3 +934,49 @@ def test_lazyframe_auto_keeps_legacy_pandas_path(self): gi = g.gfql_index_all() from graphistry.compute.gfql.index import show_indexes assert set(show_indexes(gi)["engine"]) == {"pandas"} + + def test_pandas_auto_stays_pandas(self): + ndf = pd.DataFrame({"id": [0, 1, 2]}) + edf = pd.DataFrame({"src": [0, 1], "dst": [1, 2]}) + gi = ( + graphistry.nodes(ndf, "id") + .edges(edf, "src", "dst") + .gfql_index_all() + ) + assert isinstance(gi._nodes, pd.DataFrame) + assert isinstance(gi._edges, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} + + def test_cudf_auto_stays_cudf(self): + cudf = pytest.importorskip("cudf") + ndf = cudf.DataFrame({"id": [0, 1, 2]}) + edf = cudf.DataFrame({"src": [0, 1], "dst": [1, 2]}) + gi = ( + graphistry.nodes(ndf, "id") + .edges(edf, "src", "dst") + .gfql_index_all() + ) + assert isinstance(gi._nodes, cudf.DataFrame) + assert isinstance(gi._edges, cudf.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"cudf"} + + @pytest.mark.parametrize("polars_side", ["nodes", "edges"]) + def test_mixed_eager_frames_keep_legacy_pandas_path(self, polars_side): + pl = pytest.importorskip("polars") + if polars_side == "nodes": + ndf = pl.DataFrame({"id": [0, 1, 2]}) + edf = pd.DataFrame({"src": [0, 1], "dst": [1, 2]}) + else: + ndf = pd.DataFrame({"id": [0, 1, 2]}) + edf = pl.DataFrame({"src": [0, 1], "dst": [1, 2]}) + gi = ( + graphistry.nodes(ndf, "id") + .edges(edf, "src", "dst") + .gfql_index_all() + ) + assert isinstance(gi._nodes, pd.DataFrame) + assert isinstance(gi._edges, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} From d12c9f4082bfb17b14f98fa2cc6fb02294045110 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Thu, 23 Jul 2026 01:20:59 -0700 Subject: [PATCH 4/4] chore: retrigger PR checks after GitHub ref desync