From 01dc75a791cf33cffee504e732e4f67628b40279 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Thu, 23 Jul 2026 01:46:46 -0700 Subject: [PATCH] fix(gfql): keep AUTO index construction source-native --- CHANGELOG.md | 1 + graphistry/compute/gfql/index/api.py | 33 ++++- .../tests/compute/gfql/index/test_index.py | 136 ++++++++++++++++++ 3 files changed, 168 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 346f1d8214..aa88924f79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **GFQL engine-selection docs (pandas / polars / cuDF / polars-gpu)**: New :doc:`Choosing a GFQL Engine ` page — a numbers-first, persona-tested guide to the four interchangeable engines. Adds the one-keyword `engine='polars'` speedup (up to ~38× over pandas on real graphs, no GPU), a motivating warm-median comparison table on real public graphs (LiveJournal 35M / Orkut 117M), a decision matrix (workload shape × size × hardware → engine, with the measured ~10K-edge CPU crossover, GPU-work-bound rule, polars-gpu memory-pressure caveat, and GPU-or-error contract), a cuDF-vs-polars-gpu disambiguation (eager-op vs fused-lazy; cuDF is not deprecated), an honest "when *not* to use Polars" section, the differential-parity guarantee, and a methodology + reproducer-script disclosure. Rewrote the top of `gfql/performance.rst` to lead with the engine comparison (de-marketed the prose), wired the new page into the GFQL toctree + recommended paths, and added Polars/polars-gpu to the engine examples in `gfql/quick.rst` and `gfql/about.rst` (previously only pandas/cuDF were documented). Driven by 4-persona doc user-testing (pandas DS, RAPIDS user, perf engineer, skeptical evaluator). ### Fixed +- **GFQL resident-index construction preserves eager polars node+edge frames under `engine='auto'`**: `gfql_index_all()`/`create_index()` now select polars only when both resident frames are eager polars, avoiding the legacy pandas conversion and allowing later explicit `engine='polars'` hops to serve the resident index. Pandas and cuDF remain source-native; mixed, LazyFrame, and missing-frame inputs keep the legacy fallback; explicit engines and the global AUTO query-routing policy are unchanged. Behavioral coverage pins frame/index preservation, actual index service with pandas-oracle parity, and every fallback boundary. - **GFQL polars engine natively runs `UNWIND` of a carried `collect()` list column**: `... WITH collect(x) AS xs UNWIND xs AS y RETURN ...` (exploding a list-valued column produced by `collect()`, the row-pipeline analogue of the IC6 unwind shape) previously declined on polars — `unwind_polars` only accepted a scalar literal list. Now explodes a `List`-dtype column reference (`with_columns` copy → `filter(list.len() > 0)` → `explode`), matching the pandas oracle exactly: empty-list/null cells → 0 rows, nulls *within* a list survive, source column retained; the pre-filter also makes it independent of the polars-2.0 `empty_as_null` default. Non-list columns, name collisions, unknown identifiers, and nested-list literals still decline (honest NIE, no pandas bridge). Differential fuzz vs pandas over ~3500 collect→UNWIND→{RETURN, grouped, filtered, ORDER BY, aggregate, nested-unwind} queries: 0 disagreements. (The `MATCH ... WITH collect(..) UNWIND .. MATCH` form — IC6 with a trailing re-traversal — is compile-rewritten into WITH→MATCH reentry, a separate path.) Tests in `test_engine_polars_row_pipeline.py`. - **GFQL polars engine natively runs the LDBC IC11/IC6 undirected variable-length bindings table (`-[*1..k]-`), unblocking the cross-alias `WHERE NOT(person=friend)` clause**: a bounded UNDIRECTED variable-length MATCH that materializes a bindings row table (`rows(binding_ops=...)`, emitted whenever a downstream clause — e.g. a cross-alias same-path `WHERE NOT a = b` / `a <> b` — references two node aliases on the path) previously declined on polars (`binding_rows_polars` hard-NIE'd every undirected multihop), while pandas handled it. This was the single residual blocking the official LDBC IC11/IC6 queries on polars (the cross-alias WHERE lowering itself was already native — directed var-length + `WHERE NOT a = b` already matched pandas). `binding_rows_polars` now supports undirected var-length with **`min_hops == 1`** via a doubled-pair join with immediate-backtrack avoidance, an exact port of the pandas oracle `_gfql_multihop_binding_rows` (`avoid_immediate_backtrack=True`): a `__prev__` marker (seeded null, dtype-matched to the id column) drops immediate backtracks each hop (Kleene mask: null prev kept), and the step-pair set reproduces pandas' edge multiplicity exactly — each non-loop edge contributes each directed orientation TWICE (so a length-1 pair appears x2, length-2 x4) while self-loops contribute `(u,u)` x2 only (not double-counted). The multiplicity rule was derived by instrumenting pandas' `step_pairs` (which flow from the var-length `edge_op.execute` hop + `orient_edges`), NOT by reading code alone. Scoped to `min_hops == 1` because that is where the raw-edge reconstruction provably matches pandas: every edge is trivially a length-1 path so the var-length hop's backward pruning removes nothing; `min_hops == 0` (zero-hop, undoubled) and `min_hops >= 2` (backward-pruned / long-walk divergence) still **decline with an honest `NotImplementedError`** rather than risk silent-wrong multiplicities. Differential fuzz vs the pandas oracle over ~2500 random graphs (self-loops, parallel + antiparallel edges, `*1..2`…`*1..5`, all WHERE/RETURN variants): 0 disagreements; the out-of-scope windows decline (never diverge). Tests in `test_engine_polars_binding_rows.py` (parity + multiplicity/backtrack/self-loop/string-id pins + `*0..2`/`*2..3`/`*2..2` decline pins). - **GFQL polars engine natively runs whole-entity `count(DISTINCT n)` / identity aggregation (#1709)**: `g.gfql("MATCH (a {..}) RETURN count(DISTINCT a) AS c")` and its grouped/filtered/traversal variants (`(a)-[]->(b) RETURN count(DISTINCT b)`) previously declined on polars — the Cypher aggregation lowers `count(DISTINCT b)` to the `__gfql_node_id__` identity sentinel, and the polars `lower_expr` `Identifier` branch resolved only the prefixed `alias.__gfql_node_id__` form, not the bare sentinel, so it NIE'd. Fix threads the graph node-id column through a `_NODE_ID` contextvar (published alongside the schema in `_lower_with_schema`, wired through the four callers) so the bare sentinel resolves to the id column **only if present**, else declines (never invents/mis-resolves a column). Differential fuzz vs the pandas oracle: 600 + 300 cases, 0 disagreements. Deliberately still honest-NIE (verified declines, not silent-wrong): whole-entity `collect(b)` (list-of-entities repr not ported, cf #1650) and multi-alias binding-table identity aggregation (bare id absent on connected-pattern binding tables — adjacent to #1273). Tests in `test_engine_polars_row_pipeline.py`. diff --git a/graphistry/compute/gfql/index/api.py b/graphistry/compute/gfql/index/api.py index 0ed74c3d4c..1571df6151 100644 --- a/graphistry/compute/gfql/index/api.py +++ b/graphistry/compute/gfql/index/api.py @@ -111,13 +111,42 @@ def _check_column(column: Optional[str], expected: str, kind: IndexKind) -> None ) +def _resolve_index_engine(engine: EngineAbstractType, g: Plottable) -> Engine: + """Engine for index build/validation: AUTO keeps resident polars frames polars. + + ``resolve_engine(AUTO)`` maps polars input frames to PANDAS (the legacy + input-format policy), which would make ``create_index`` coerce-and-REPLACE the + user's polars frames with pandas copies — and then every later + ``engine='polars'`` query pays a full-frame pandas->polars re-conversion, an + O(E) tax per call that dwarfs the indexed hop itself. The index layer is + engine-polymorphic (numpy sidecar arrays + polars row-gather), so when the + caller said AUTO and the resident frames are polars, index them in place. + Explicit engines are honored unchanged. + """ + def _is_eager_polars(df: object) -> bool: + # EAGER frames only: LazyFrame passes the module check but no index/coercion + # path can gather rows from it — those graphs keep the legacy pandas path. + return df is not None and 'polars' in str(type(df).__module__) and type(df).__name__ == 'DataFrame' + eng = resolve_engine(engine, g) + abstract = EngineAbstract(engine) if isinstance(engine, str) else engine + if abstract == EngineAbstract.AUTO and eng == Engine.PANDAS: + if ( + g._edges is not None + and g._nodes is not None + and _is_eager_polars(g._edges) + and _is_eager_polars(g._nodes) + ): + return Engine.POLARS + return eng + + def _is_resident_index_valid( g: Plottable, kind: IndexKind, engine: EngineAbstractType = EngineAbstract.AUTO, ) -> bool: """True when a resident index still matches the current graph frames.""" - eng = resolve_engine(engine, g) + eng = _resolve_index_engine(engine, g) registry = get_registry(g) if kind in ADJ_KINDS: src, dst = g._source, g._destination @@ -148,7 +177,7 @@ def create_index( O(E log E) once, amortized over later seeded queries. """ from dataclasses import replace - eng = resolve_engine(engine, g) + eng = _resolve_index_engine(engine, g) # Build over frames already in the target engine so the index arrays land on # the right backend (cupy for cudf, numpy otherwise). No-op when already in-engine. from graphistry.compute.ComputeMixin import _coerce_input_formats diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 0303ce0495..8ae9fb9ff5 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -844,3 +844,139 @@ def test_hop_dtype_mismatch_edge_match_matches_scan_error(typed_graph, engine): if engine == "pandas": from graphistry.compute.exceptions import GFQLSchemaError assert isinstance(base_err.value, GFQLSchemaError) + + +class TestIndexAutoPreservesPolarsFrames: + """gfql_index_all(engine='auto') on a polars graph must index in place, NOT + coerce-and-replace the frames with pandas. Regression pin for the C3 "polars + hop is O(E)" mystery: resolve_engine(AUTO) maps polars->PANDAS, so create_index + used to swap the frames to pandas, and every later hop(engine='polars') paid a + full-frame pandas->polars conversion per call (~220ms at 4M edges vs a ~1ms + indexed hop) while the pandas-engine index could never fingerprint-match.""" + + def _pl_graph(self): + pl = pytest.importorskip("polars") + ndf = pl.DataFrame({"id": [0, 1, 2, 3], "type": ["a", "b", "a", "b"]}) + edf = pl.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 3]}) + return graphistry.nodes(ndf, "id").edges(edf, "src", "dst") + + def test_auto_keeps_polars_frames_and_polars_index(self): + from graphistry.Engine import Engine + g = self._pl_graph() + gi = g.gfql_index_all() # AUTO + assert "polars" in type(gi._nodes).__module__ + assert "polars" in type(gi._edges).__module__ + # frames indexed in place: identity preserved so get_valid's `is` check holds + assert gi._edges is g._edges + from graphistry.compute.gfql.index import show_indexes + idx = show_indexes(gi) + assert set(idx["engine"]) == {Engine.POLARS.value} + assert idx["valid"].all() + + def test_auto_polars_hop_engages_index(self, monkeypatch): + # big enough that one seed passes the frontier-fraction cost gate + pl = pytest.importorskip("polars") + import graphistry.compute.gfql.index.api as index_api + rng = np.random.default_rng(0) + n_nodes, m = 2000, 12000 + ndf = pl.DataFrame({"id": np.arange(n_nodes)}) + edf = pl.DataFrame({"src": rng.integers(0, n_nodes, m), "dst": rng.integers(0, n_nodes, m)}) + g = graphistry.nodes(ndf, "id").edges(edf, "src", "dst").gfql_index_all() + hits = [] + orig = index_api.index_seeded_hop + + def spy(*a, **k): + out = orig(*a, **k) + hits.append(out is not None) # None = scan fallback; only a real serve counts + return out + monkeypatch.setattr(index_api, "index_seeded_hop", spy) + r = g.hop(nodes=pl.DataFrame({"id": [7]}), hops=1, direction="forward", engine="polars") + assert any(hits), "resident polars index did not SERVE the polars hop (call alone is not service)" + # parity vs the pandas indexed oracle on the same data + gp = graphistry.nodes(ndf.to_pandas(), "id").edges(edf.to_pandas(), "src", "dst").gfql_index_all() + rp = gp.hop(nodes=pd.DataFrame({"id": [7]}), hops=1, direction="forward", engine="pandas") + assert sorted(r._nodes.get_column("id").to_list()) == sorted(rp._nodes["id"].tolist()) + + def test_explicit_engine_still_coerces(self): + g = self._pl_graph() + gi = g.gfql_index_all(engine="pandas") + assert isinstance(gi._edges, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} + + def test_edges_only_polars_keeps_legacy_pandas_path(self): + pl = pytest.importorskip("polars") + g = graphistry.edges( + pl.DataFrame({"src": [0, 1, 2], "dst": [1, 2, 3]}), "src", "dst" + ) + gi = g.gfql_index_all() + assert isinstance(gi._nodes, pd.DataFrame) + assert isinstance(gi._edges, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} + + def test_nodes_only_polars_keeps_legacy_pandas_path(self): + pl = pytest.importorskip("polars") + gi = ( + graphistry.nodes(pl.DataFrame({"id": [0, 1, 2]}), "id") + .create_index("node_id") + ) + assert isinstance(gi._nodes, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} + + def test_lazyframe_auto_keeps_legacy_pandas_path(self): + # M1 pin: LazyFrame frames under AUTO must coerce to pandas like master + # (the eager-polars-only gate), never crash in a polars index build. + pl = pytest.importorskip("polars") + g = graphistry.nodes(pl.LazyFrame({"id": [0, 1, 2]}), "id").edges( + pl.LazyFrame({"src": [0, 1], "dst": [1, 2]}), "src", "dst") + gi = g.gfql_index_all() + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} + + def test_pandas_auto_stays_pandas(self): + ndf = pd.DataFrame({"id": [0, 1, 2]}) + edf = pd.DataFrame({"src": [0, 1], "dst": [1, 2]}) + gi = ( + graphistry.nodes(ndf, "id") + .edges(edf, "src", "dst") + .gfql_index_all() + ) + assert isinstance(gi._nodes, pd.DataFrame) + assert isinstance(gi._edges, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"} + + def test_cudf_auto_stays_cudf(self): + cudf = pytest.importorskip("cudf") + ndf = cudf.DataFrame({"id": [0, 1, 2]}) + edf = cudf.DataFrame({"src": [0, 1], "dst": [1, 2]}) + gi = ( + graphistry.nodes(ndf, "id") + .edges(edf, "src", "dst") + .gfql_index_all() + ) + assert isinstance(gi._nodes, cudf.DataFrame) + assert isinstance(gi._edges, cudf.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"cudf"} + + @pytest.mark.parametrize("polars_side", ["nodes", "edges"]) + def test_mixed_eager_frames_keep_legacy_pandas_path(self, polars_side): + pl = pytest.importorskip("polars") + if polars_side == "nodes": + ndf = pl.DataFrame({"id": [0, 1, 2]}) + edf = pd.DataFrame({"src": [0, 1], "dst": [1, 2]}) + else: + ndf = pd.DataFrame({"id": [0, 1, 2]}) + edf = pl.DataFrame({"src": [0, 1], "dst": [1, 2]}) + gi = ( + graphistry.nodes(ndf, "id") + .edges(edf, "src", "dst") + .gfql_index_all() + ) + assert isinstance(gi._nodes, pd.DataFrame) + assert isinstance(gi._edges, pd.DataFrame) + from graphistry.compute.gfql.index import show_indexes + assert set(show_indexes(gi)["engine"]) == {"pandas"}