From 98ce057d6544426ee436bed9c4fd6463f47336f1 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:50:47 +0000 Subject: [PATCH 01/21] test(extract): add perl fixtures and RED language tests Add tests/fixtures/sample.pl + sample_module.pm and 17 Perl tests in tests/test_languages.py covering packages (incl. mid-file switch), subs, use/require imports (pragma exclusion), @ISA/use parent/use base inheritance, static + bare-sub calls (INFERRED, second-pass) and the untyped method-call negative case. Extractor (S3) and .pl/.pm dispatch (S4) not implemented yet, so all 17 fail (RED); existing suite stays green. --- tests/fixtures/sample.pl | 27 +++++ tests/fixtures/sample_module.pm | 46 +++++++++ tests/test_languages.py | 173 ++++++++++++++++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 tests/fixtures/sample.pl create mode 100644 tests/fixtures/sample_module.pm diff --git a/tests/fixtures/sample.pl b/tests/fixtures/sample.pl new file mode 100644 index 000000000..102f75bf6 --- /dev/null +++ b/tests/fixtures/sample.pl @@ -0,0 +1,27 @@ +#!/usr/bin/perl +use strict; +use warnings; +use Acme::Widget; + +package Acme::Helper; + +sub emit { + my ($text) = @_; + print "$text\n"; + return length $text; +} + +sub format { + my ($text) = @_; + return uc $text; +} + +package main; + +sub run_report { + my $widget = Acme::Widget->new(); + $widget->render(); + Acme::Helper::emit("done"); +} + +run_report(); diff --git a/tests/fixtures/sample_module.pm b/tests/fixtures/sample_module.pm new file mode 100644 index 000000000..02e0eaa20 --- /dev/null +++ b/tests/fixtures/sample_module.pm @@ -0,0 +1,46 @@ +package Acme::Widget; + +use strict; +use warnings; +use Scalar::Util qw(blessed); +use Carp; +require Acme::Helper; + +use parent -norequire, 'Acme::Role'; +use base 'Acme::Mixin'; + +our @ISA = ('Acme::Base'); + +sub new { + my ($class, %args) = @_; + my $self = { %args }; + return bless $self, $class; +} + +sub render { + my $self = shift; + my $line = format_line(); + Acme::Helper::emit($line); + $self->update(); + return $line; +} + +sub format_line { + return "rendered"; +} + +sub update { + my $self = shift; + $self->{dirty} = 1; + return $self; +} + +package Acme::Widget::Inner; + +use POSIX qw(floor); + +sub tick { + return floor(1.5); +} + +1; diff --git a/tests/test_languages.py b/tests/test_languages.py index f610fbc40..f4e1124fd 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -2945,3 +2945,176 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): r = _corpus("cpp_samedir/Alpha.h", "cpp_samedir/Beta.h") dups = _nodes_with_label(r, "Dup") assert len(dups) == 2, f"same-dir distinct Dups must stay distinct, got {[n['id'] for n in dups]}" + + +# ── Perl ────────────────────────────────────────────────────────────────────── +# RED (S2): the `extract_perl` extractor does not exist yet (S3) and the .pl/.pm +# dispatch is not registered yet (S4). The `extract_perl` import is kept INSIDE +# each test so importing this module still succeeds and the existing suite stays +# GREEN — only the Perl tests fail (ImportError until S3, empty graph until S4). + +def test_perl_no_error(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert "error" not in r + +def test_perl_finds_packages(): + """Both the leading `package Acme::Widget` and the mid-file + `package Acme::Widget::Inner` switch must surface as nodes.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + labels = _labels(r) + assert any("Acme::Widget" in l for l in labels) + assert any("Acme::Widget::Inner" in l for l in labels) + +def test_perl_finds_subs(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + labels = _labels(r) + for name in ("new", "render", "format_line", "update", "tick"): + assert any(name in l for l in labels), f"missing sub {name!r}" + +def test_perl_finds_imports(): + """`use Module` / `require Module` become imports edges; the qw-list form + (`use Scalar::Util qw(blessed)`) and bareword `require` both count.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert "imports" in _relations(r) + import_edges = _edges_with_relation(r, "imports", "imports_from") + targets = " ".join(str(e.get("target", "")) for e in import_edges).lower() + assert "scalar" in targets or "util" in targets, "Scalar::Util import missing" + assert "carp" in targets, "Carp import missing" + assert "helper" in targets, "require Acme::Helper import missing" + +def test_perl_import_edges_have_import_context(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + assert import_edges + assert all(e.get("context") == "import" for e in import_edges) + +def test_perl_import_edges_are_extracted(): + """`use`/`require` are literal in source -> EXTRACTED confidence.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + assert import_edges + assert all(e.get("confidence") == "EXTRACTED" for e in import_edges) + +def test_perl_pragmas_not_imported(): + """`use strict` / `use warnings` are pragmas, not module dependencies, and + must NOT create imports edges (matches the pragma-exclusion in other langs).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + targets = " ".join(str(e.get("target", "")) for e in import_edges).lower() + assert "strict" not in targets, "`use strict` pragma must not be an import" + assert "warnings" not in targets, "`use warnings` pragma must not be an import" + +def test_perl_isa_inherits_edge(): + """`our @ISA = ('Acme::Base')` must emit an inherits edge to Acme::Base.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + found = any( + "Acme::Widget" in node_by_id.get(e["source"], "") + and "Acme::Base" in node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + ) + assert found, "Acme::Widget should have inherits edge to Acme::Base (our @ISA)" + +def test_perl_use_parent_inherits_edge(): + """`use parent -norequire, 'Acme::Role'` must emit an inherits edge; the + `-norequire` flag is not a parent.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + parents = { + node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + and "Acme::Widget" in node_by_id.get(e["source"], "") + } + assert any("Acme::Role" in p for p in parents), "use parent target Acme::Role missing" + assert not any("norequire" in p for p in parents), "-norequire flag must not be a parent" + +def test_perl_use_base_inherits_edge(): + """`use base 'Acme::Mixin'` must emit an inherits edge to Acme::Mixin.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + parents = { + node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + and "Acme::Widget" in node_by_id.get(e["source"], "") + } + assert any("Acme::Mixin" in p for p in parents), "use base target Acme::Mixin missing" + +def test_perl_inherits_edges_are_extracted(): + """@ISA / use parent / use base are literal in source -> EXTRACTED.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert inherits + assert all(e.get("confidence") == "EXTRACTED" for e in inherits) + +def test_perl_no_dangling_edges(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"dangling edge source: {e}" + + +# Perl call resolution runs in the top-level `extract()` second pass (raw_calls -> +# name-matched calls), so these drive the public multi-file dispatch, like the +# ObjC cross-file tests. RED until BOTH S3 (extractor) and S4 (.pl/.pm dispatch). + +def _perl_corpus(): + from graphify.extract import extract + return extract([FIXTURES / "sample.pl", FIXTURES / "sample_module.pm"], parallel=False) + +def test_perl_bare_sub_call_edge(): + """`format_line()` bare call inside `render` (same file/package) must resolve + to a calls edge render -> format_line.""" + r = _perl_corpus() + calls = _calls(r) + assert any("render" in src and "format_line" in tgt for src, tgt in calls), \ + f"expected render->format_line calls edge, got {calls}" + +def test_perl_static_qualified_call_edge(): + """`Acme::Helper::emit(...)` static, package-qualified call must resolve to a + calls edge targeting the emit sub.""" + r = _perl_corpus() + calls = _calls(r) + assert any("emit" in tgt for _src, tgt in calls), \ + f"expected a calls edge into Acme::Helper::emit, got {calls}" + +def test_perl_call_edges_are_inferred(): + """Second-pass name-matched calls carry INFERRED confidence.""" + r = _perl_corpus() + call_edges = [e for e in r["edges"] if e["relation"] == "calls"] + assert call_edges + assert all(e.get("confidence") == "INFERRED" for e in call_edges), \ + f"perl calls should be INFERRED (second-pass), got {[e.get('confidence') for e in call_edges]}" + +def test_perl_call_edges_have_call_context(): + r = _perl_corpus() + call_edges = _edges_with_relation(r, "calls") + assert call_edges + assert all(e.get("context") == "call" for e in call_edges) + +def test_perl_method_call_no_edge(): + """`$self->update()` / `$widget->render()` are untyped method dispatch and must + NOT create calls edges, even though `update` and `render` are defined subs + (naive name-matching would wrongly connect them).""" + r = _perl_corpus() + calls = _calls(r) + # Positive guard: the corpus must actually produce static/bare calls, so this + # test is not a vacuous pass on an empty graph (RED until S3+S4). + assert any("format_line" in tgt or "emit" in tgt for _src, tgt in calls), \ + f"expected the static/bare calls to resolve before asserting the negatives, got {calls}" + assert not any("update" in tgt for _src, tgt in calls), \ + f"method call $self->update() must not produce a calls edge, got {calls}" + assert not any(tgt == "render" or tgt.endswith("::render") or tgt.endswith(".render") + for _src, tgt in calls), \ + f"method call $widget->render() must not produce a calls edge, got {calls}" From 8e6f78d683d04766b21da2a121f7d3ae3701f1af Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:00:16 +0000 Subject: [PATCH 02/21] feat(extract): add perl extractor (packages, subs, imports, inherits) Bespoke tree-sitter-perl extractor under extractors/perl.py, re-exported via the extract.py facade and registered in LANGUAGE_EXTRACTORS. Emits package + sub nodes (mid-file package switches tracked), imports edges for use/require (pragmas and use parent/base excluded), and inherits edges for our \@ISA / use parent / use base. Calls are handed to the shared second pass as raw_calls (INFERRED name-matching); method calls are flagged is_member_call so that pass drops them (untyped, unresolvable). tree-sitter-perl>=1.2.0,<2.0 added as a regular dependency. --- graphify/extract.py | 1 + graphify/extractors/__init__.py | 2 + graphify/extractors/perl.py | 230 ++++++++++++++++++++++++++++++++ pyproject.toml | 1 + uv.lock | 18 ++- 5 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 graphify/extractors/perl.py diff --git a/graphify/extract.py b/graphify/extract.py index dbef91ea0..6909db4ca 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -44,6 +44,7 @@ from graphify.extractors.json_config import extract_json # noqa: F401 from graphify.extractors.markdown import extract_markdown # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 +from graphify.extractors.perl import extract_perl # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada517094..601ed547d 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -24,6 +24,7 @@ from graphify.extractors.objc import extract_objc from graphify.extractors.pascal import extract_pascal from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form +from graphify.extractors.perl import extract_perl from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest from graphify.extractors.razor import extract_razor from graphify.extractors.rust import extract_rust @@ -52,6 +53,7 @@ "markdown": extract_markdown, "objc": extract_objc, "pascal": extract_pascal, + "perl": extract_perl, "powershell": extract_powershell, "powershell_manifest": extract_powershell_manifest, "razor": extract_razor, diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py new file mode 100644 index 000000000..6b40740d2 --- /dev/null +++ b/graphify/extractors/perl.py @@ -0,0 +1,230 @@ +"""Perl extractor: packages, subs, imports (use/require), inheritance (@ISA/parent/base). + +Deliberately untyped, consistent with the other language extractors. Calls are +handed to the shared cross-file second pass as ``raw_calls`` (INFERRED by +name-matching); method calls (``$obj->meth()``) are marked ``is_member_call`` so +that pass drops them — without receiver types they are unresolvable and naive +name-matching would wire spurious edges to same-named subs. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from graphify.extractors.base import _file_stem, _make_id + +# ``use strict`` & friends are compiler pragmas, not module dependencies — they +# must not become imports edges (matches the pragma-exclusion in other langs). +_PERL_PRAGMAS: frozenset[str] = frozenset({ + "strict", "warnings", "utf8", "constant", "vars", "lib", "feature", + "integer", "bytes", "overload", "mro", "autodie", "diagnostics", + "sort", "subs", "attributes", "fields", "encoding", "if", "less", +}) + +# ``use parent`` / ``use base`` declare inheritance, not an import. +_PERL_INHERIT_PRAGMAS: frozenset[str] = frozenset({"parent", "base"}) + +# Perl builtins that surface as function_call_expression callees; excluding them +# keeps them from accumulating spurious calls edges as god-nodes. +_PERL_BUILTINS: frozenset[str] = frozenset({ + "print", "printf", "say", "sprintf", "die", "warn", "return", "bless", + "shift", "unshift", "push", "pop", "splice", "map", "grep", "sort", + "reverse", "join", "split", "keys", "values", "each", "exists", "delete", + "defined", "ref", "scalar", "wantarray", "length", "uc", "lc", "ucfirst", + "lcfirst", "chomp", "chop", "chr", "ord", "abs", "int", "sqrt", "eval", + "local", "my", "our", "sub", "do", "require", "use", "no", +}) + + +def extract_perl(path: Path) -> dict: + """Extract packages, subs, imports and inheritance from a .pl/.pm/.t file.""" + try: + import tree_sitter_perl as tsperl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree_sitter_perl not installed"} + + try: + language = Language(tsperl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = _file_stem(path) + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + sub_bodies: list[tuple[str, Any]] = [] + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", context: str | None = None) -> None: + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0} + if context: + edge["context"] = context + edges.append(edge) + + file_nid = _make_id(str(path)) + add_node(file_nid, path.name, 1) + + def _package_name(node) -> str | None: + """`package Foo::Bar;` -> the second `package`-typed child is the name.""" + names = [c for c in node.children if c.type == "package"] + return _text(names[1]) if len(names) >= 2 else None + + def _string_parents(node) -> list[str]: + """Every string / qw-word parent named under an inheritance construct. + + Reads string_literal (`'Foo'`) and quoted_word_list (`qw(Foo Bar)`) + content anywhere below ``node``; autoquoted barewords like ``-norequire`` + are a different node type and are intentionally skipped. + """ + out: list[str] = [] + + def visit(n) -> None: + if n.type in ("string_literal", "interpolated_string_literal", "quoted_word_list"): + for c in n.children: + if c.type == "string_content": + out.extend(_text(c).split()) + return + for c in n.children: + visit(c) + + visit(node) + return out + + def add_inherits(pkg_nid: str, parent_name: str, line: int) -> None: + parent_nid = _make_id(parent_name) + add_node(parent_nid, parent_name, line) + add_edge(pkg_nid, parent_nid, "inherits", line, context="inherit") + + current_pkg_nid: str | None = None + + def handle_use(node, line: int) -> None: + # In a use_statement the `use` keyword is its own node type, so the + # module/pragma name is the first (and only) `package`-typed child — + # unlike package_statement, where `package` is also the keyword's type. + pkgs = [c for c in node.children if c.type == "package"] + if not pkgs: + return + module = _text(pkgs[0]) + if module in _PERL_INHERIT_PRAGMAS: + if current_pkg_nid: + for parent in _string_parents(node): + add_inherits(current_pkg_nid, parent, line) + return + if module in _PERL_PRAGMAS: + return + add_edge(file_nid, _make_id(module), "imports", line, context="import") + + def handle_require(req_node, line: int) -> None: + for c in req_node.children: + if c.type == "bareword": + add_edge(file_nid, _make_id(_text(c)), "imports", line, context="import") + return + + def handle_isa(assign_node, line: int) -> None: + """`our @ISA = (...)` -> inherits edges to each named parent.""" + if not current_pkg_nid: + return + is_isa = False + for child in assign_node.children: + if child.type == "variable_declaration": + for sub in child.children: + if sub.type == "array" and _text(sub) == "@ISA": + is_isa = True + if not is_isa: + return + for parent in _string_parents(assign_node): + add_inherits(current_pkg_nid, parent, line) + + def walk_statements(node) -> None: + nonlocal current_pkg_nid + for child in node.children: + line = child.start_point[0] + 1 + if child.type == "package_statement": + name = _package_name(child) + if name: + pkg_nid = _make_id(stem, name) + add_node(pkg_nid, name, line) + add_edge(file_nid, pkg_nid, "contains", line) + current_pkg_nid = pkg_nid + elif child.type == "use_statement": + handle_use(child, line) + elif child.type == "subroutine_declaration_statement": + name = None + block = None + for c in child.children: + if c.type == "bareword" and name is None: + name = _text(c) + elif c.type == "block": + block = c + if name: + container = current_pkg_nid or file_nid + sub_nid = _make_id(container, name) + add_node(sub_nid, f"{name}()", line) + add_edge(container, sub_nid, "contains", line) + if block is not None: + sub_bodies.append((sub_nid, block)) + elif child.type == "expression_statement": + for c in child.children: + if c.type == "require_expression": + handle_require(c, line) + elif c.type == "assignment_expression": + handle_isa(c, line) + + walk_statements(root) + + raw_calls: list[dict] = [] + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "function_call_expression": + for c in node.children: + if c.type == "function": + callee = _text(c).split("::")[-1] + if callee and callee not in _PERL_BUILTINS: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": False, + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + elif node.type == "method_call_expression": + for c in node.children: + if c.type == "method": + callee = _text(c).split("::")[-1] + if callee: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": True, + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + for child in node.children: + walk_calls(child, caller_nid) + + for caller_nid, body in sub_bodies: + walk_calls(body, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] == "imports")] + return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, + "input_tokens": 0, "output_tokens": 0} diff --git a/pyproject.toml b/pyproject.toml index 510e77469..3efaad19f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "tree-sitter-fortran>=0.6,<0.8", "tree-sitter-bash>=0.23,<0.27", "tree-sitter-json>=0.23,<0.26", + "tree-sitter-perl>=1.2.0,<2.0", ] [project.urls] diff --git a/uv.lock b/uv.lock index 088ebbbdc..c1b8de73e 100644 --- a/uv.lock +++ b/uv.lock @@ -1090,7 +1090,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.9.6" +version = "0.9.12" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -1114,6 +1114,7 @@ dependencies = [ { name = "tree-sitter-kotlin" }, { name = "tree-sitter-lua" }, { name = "tree-sitter-objc" }, + { name = "tree-sitter-perl" }, { name = "tree-sitter-php" }, { name = "tree-sitter-powershell" }, { name = "tree-sitter-python" }, @@ -1314,6 +1315,7 @@ requires-dist = [ { name = "tree-sitter-objc", specifier = ">=3.0,<4.0" }, { name = "tree-sitter-pascal", marker = "extra == 'all'" }, { name = "tree-sitter-pascal", marker = "extra == 'pascal'" }, + { name = "tree-sitter-perl", specifier = ">=1.2.0,<2.0" }, { name = "tree-sitter-php", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-powershell", specifier = ">=0.26,<0.28" }, { name = "tree-sitter-python", specifier = ">=0.23,<0.26" }, @@ -4754,6 +4756,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/ad/e5381365ed5b247cad2e047682b081ccbb79f84bb5c1b48642ef3385f1d3/tree_sitter_pascal-0.11.0-cp38-abi3-win_arm64.whl", hash = "sha256:9d22b974a47765f1c98629338d82d1df89c35c270a73f08dda8fbd49538e32d4", size = 124914, upload-time = "2026-07-01T03:53:15.191Z" }, ] +[[package]] +name = "tree-sitter-perl" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/44/c211fda4f42c01791391928b23192b50d160b14e676f2c1fcce1850d76d9/tree_sitter_perl-1.2.1.tar.gz", hash = "sha256:7b5ce3fe9cd7d23b1619ed49d5539b106e301b68dd7139bcae6b9d4ad1e3a1d3", size = 1749866, upload-time = "2026-06-30T18:53:23.646Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/29/4cdb0da188ce7864507317e70db6c9dc045f41d6250f0865aef03294eeb2/tree_sitter_perl-1.2.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5e77a13168ae5699ee9942aaa928920b9a2d8ce051934550ba2a4b547833a8ba", size = 347325, upload-time = "2026-06-30T18:53:14.467Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b3/7489ac765c9d5e2a8ef75f43caaa5f1df3cbc93bbc5c0f49559bc4c2e280/tree_sitter_perl-1.2.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a18f09b6b76d9bb52da49f3bb4ed7fe4db0857fbf21d1aad78021abaddb47d72", size = 408980, upload-time = "2026-06-30T18:53:16.211Z" }, + { url = "https://files.pythonhosted.org/packages/a3/50/7340015653da77bc5d12e45678c9339cabf389aba7404d3f5e0514e04acc/tree_sitter_perl-1.2.1-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6cd149c5f5fb5f82d337d5764d8626eecb03b00604c8c0c7ffc4e165e26b273", size = 385181, upload-time = "2026-06-30T18:53:17.741Z" }, + { url = "https://files.pythonhosted.org/packages/8b/77/76486f28c0d837d1424d3a314736ca2b6916a56b5adf073d6e038fb03522/tree_sitter_perl-1.2.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:66361e2d61467843aeb9ef1233a461e2880ed294a368d81e4b898b33c76913fe", size = 370909, upload-time = "2026-06-30T18:53:19.223Z" }, + { url = "https://files.pythonhosted.org/packages/7f/cd/6fbecae40fb5f78bcb95432fc41d6eb0b6a246e43bb63237544c92aa271a/tree_sitter_perl-1.2.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0cfd166f0f11de834b96d99432a86b54444ccbd270d9e19eed06c2de9d6f932e", size = 373213, upload-time = "2026-06-30T18:53:20.769Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/a17bd77d350cc7344baf6f3f568f297a1d1a88d2afbf3024a8e86618e96f/tree_sitter_perl-1.2.1-cp39-abi3-win_amd64.whl", hash = "sha256:168704111ed36461b1d1da1a36b16332e2c8e2693b5daec57bd8abada1e02f58", size = 320105, upload-time = "2026-06-30T18:53:22.14Z" }, +] + [[package]] name = "tree-sitter-php" version = "0.24.1" From a651d173ab7093a57e90b66fb576b41f5ad61e7a Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:24:17 +0000 Subject: [PATCH 03/21] fix(extract): package-aware perl call resolution, inherit stubs, builtin filter, edge dedup Review findings on the Perl extractor (S3b): - F1/F3: raw_calls now carry callee_package (qualifier) + caller_package (enclosing package). The shared second pass package-aware-filters candidates: a qualified Pkg::sub() binds only to a sub in that package; a bare call prefers the caller's own package, else requires unique import evidence. No unique match drops the edge (zero-edge, not a wrong-package guess). Additive + guarded: only Perl raw_calls carry the fields, so every other language's resolution is unchanged. - F4: external inheritance parents (RTL / other module) are emitted as stubs with empty source_file instead of falsely claiming the child file. - F5: builtin filter extended to the perlfunc core (open/close/system/exec/ index/substr/time/sleep/mkdir/unlink/...). - F6: edges deduped on (source, target, relation, context). Second-pass call tests remain RED until S4 wires .pl/.pm dispatch. --- graphify/extract.py | 46 +++++++++++++++ graphify/extractors/perl.py | 80 ++++++++++++++++++++----- tests/test_languages.py | 114 ++++++++++++++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 14 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 6909db4ca..02d104035 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4606,6 +4606,19 @@ def extract( # file is real ONLY if the caller imported it. So a cross-file call from one # of these files with no import evidence is gated below (#1659). _JS_TS_CALL_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") + + # Enclosing-package label per node id, for the package-aware call resolution + # below. Only consulted for raw_calls that carry package qualifiers (Perl): + # a sub's direct container is its package node, whose label IS the package + # name (e.g. "Acme::Widget"), so the `contains` edge target->source gives + # sub_id -> package label. Other languages never set the package fields, so + # this map is built but never read for them. + _label_by_nid = {n["id"]: n.get("label", "") for n in all_nodes} + pkg_label_by_nid: dict[str, str] = {} + for e in all_edges: + if e.get("relation") == "contains": + pkg_label_by_nid[e["target"]] = _label_by_nid.get(e["source"], "") + for rc in all_raw_calls: callee = rc.get("callee", "") if not callee: @@ -4672,6 +4685,39 @@ def _has_import_evidence(candidate_id: str) -> bool: or (candidate_file_nid is not None and candidate_file_nid in imported_modules) ) + # Package-aware pre-filter for languages that tag calls with their + # package (Perl). Additive + guarded: a raw_call only reaches this branch + # when it carries a package field, which no other language sets, so every + # other language's candidate set is untouched (the existing 314 tests + # prove the unqualified path is unchanged). Zero-edge over a wrong guess: + # an unresolvable qualifier or a foreign-package bare call is dropped, not + # bound to a same-named sub in the wrong package (#F1/#F3). + callee_package = rc.get("callee_package") + caller_package = rc.get("caller_package") + if callee_package is not None or caller_package is not None: + if callee_package is not None: + # Qualified call `Pkg::sub()`: bind only to a sub whose enclosing + # package matches the qualifier. None or several -> drop. + candidates = [ + c for c in candidates + if pkg_label_by_nid.get(c) == callee_package + ] + else: + # Bare call: prefer the caller's own package. If the sub is + # defined there, that's the target. Otherwise it can only be an + # imported sub — require unique import evidence, else drop (never + # bind a same-named sub from an unrelated package). + same_pkg = [ + c for c in candidates + if pkg_label_by_nid.get(c) == caller_package + ] + if same_pkg: + candidates = same_pkg + else: + candidates = [c for c in candidates if _has_import_evidence(c)] + if len(candidates) != 1: + continue + if len(candidates) == 1: tgt = candidates[0] has_import_evidence = _has_import_evidence(tgt) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 6b40740d2..e1ce68b6b 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -25,14 +25,32 @@ _PERL_INHERIT_PRAGMAS: frozenset[str] = frozenset({"parent", "base"}) # Perl builtins that surface as function_call_expression callees; excluding them -# keeps them from accumulating spurious calls edges as god-nodes. +# keeps them from accumulating spurious calls edges as god-nodes. Kept to the +# canonical perlfunc core so a user sub that happens to share a name with a +# builtin still resolves against real definitions in the corpus. _PERL_BUILTINS: frozenset[str] = frozenset({ - "print", "printf", "say", "sprintf", "die", "warn", "return", "bless", + # I/O & formatting + "print", "printf", "say", "sprintf", "open", "close", "read", "write", + "binmode", "eof", "seek", "tell", "sysread", "syswrite", "readline", + # process / system + "system", "exec", "fork", "wait", "waitpid", "kill", "sleep", "time", + "exit", "die", "warn", + # filesystem + "mkdir", "rmdir", "unlink", "rename", "chdir", "chmod", "chown", "stat", + "lstat", "opendir", "readdir", "closedir", "glob", + # list / hash ops "shift", "unshift", "push", "pop", "splice", "map", "grep", "sort", "reverse", "join", "split", "keys", "values", "each", "exists", "delete", - "defined", "ref", "scalar", "wantarray", "length", "uc", "lc", "ucfirst", - "lcfirst", "chomp", "chop", "chr", "ord", "abs", "int", "sqrt", "eval", - "local", "my", "our", "sub", "do", "require", "use", "no", + "wantarray", + # string ops + "index", "rindex", "substr", "length", "uc", "lc", "ucfirst", "lcfirst", + "chomp", "chop", "chr", "ord", "hex", "oct", "sprintf", "pack", "unpack", + "quotemeta", + # math + "abs", "int", "sqrt", "sin", "cos", "atan2", "exp", "log", "rand", "srand", + # scalars / refs / types + "defined", "ref", "scalar", "bless", "return", "eval", "local", "my", + "our", "sub", "do", "require", "use", "no", "wantarray", }) @@ -58,7 +76,12 @@ def extract_perl(path: Path) -> dict: nodes: list[dict] = [] edges: list[dict] = [] seen_ids: set[str] = set() - sub_bodies: list[tuple[str, Any]] = [] + # Dedup edges on their identity tuple so a repeated construct (e.g. two + # `use Foo;` in one file) yields a single edge instead of parallel dups. + seen_edges: set[tuple[str, str, str, str | None]] = set() + # sub_nid, body block, and the sub's enclosing package name (for the shared + # second pass's package-aware call resolution). + sub_bodies: list[tuple[str, Any, str | None]] = [] def _text(node) -> str: return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") @@ -69,8 +92,27 @@ def add_node(nid: str, label: str, line: int) -> None: nodes.append({"id": nid, "label": label, "file_type": "code", "source_file": str_path, "source_location": f"L{line}"}) + def add_stub_node(nid: str, label: str) -> None: + """External inheritance target (base class defined elsewhere / in the RTL). + + Emitted with an empty ``source_file`` so it does not falsely claim this + child file as the parent's source and so the corpus-level cross-file + rewire can collapse it onto the real definition; ``origin_file`` keeps + distinct same-named stubs apart in the colliding-id pass. Mirrors the + external-stub shape used by go/julia/objc. + """ + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": "", "source_location": "", + "origin_file": str_path}) + def add_edge(src: str, tgt: str, relation: str, line: int, confidence: str = "EXTRACTED", context: str | None = None) -> None: + key = (src, tgt, relation, context) + if key in seen_edges: + return + seen_edges.add(key) edge = {"source": src, "target": tgt, "relation": relation, "confidence": confidence, "source_file": str_path, "source_location": f"L{line}", "weight": 1.0} @@ -109,10 +151,11 @@ def visit(n) -> None: def add_inherits(pkg_nid: str, parent_name: str, line: int) -> None: parent_nid = _make_id(parent_name) - add_node(parent_nid, parent_name, line) + add_stub_node(parent_nid, parent_name) add_edge(pkg_nid, parent_nid, "inherits", line, context="inherit") current_pkg_nid: str | None = None + current_pkg_name: str | None = None def handle_use(node, line: int) -> None: # In a use_statement the `use` keyword is its own node type, so the @@ -153,7 +196,7 @@ def handle_isa(assign_node, line: int) -> None: add_inherits(current_pkg_nid, parent, line) def walk_statements(node) -> None: - nonlocal current_pkg_nid + nonlocal current_pkg_nid, current_pkg_name for child in node.children: line = child.start_point[0] + 1 if child.type == "package_statement": @@ -163,6 +206,7 @@ def walk_statements(node) -> None: add_node(pkg_nid, name, line) add_edge(file_nid, pkg_nid, "contains", line) current_pkg_nid = pkg_nid + current_pkg_name = name elif child.type == "use_statement": handle_use(child, line) elif child.type == "subroutine_declaration_statement": @@ -179,7 +223,7 @@ def walk_statements(node) -> None: add_node(sub_nid, f"{name}()", line) add_edge(container, sub_nid, "contains", line) if block is not None: - sub_bodies.append((sub_nid, block)) + sub_bodies.append((sub_nid, block, current_pkg_name)) elif child.type == "expression_statement": for c in child.children: if c.type == "require_expression": @@ -191,15 +235,23 @@ def walk_statements(node) -> None: raw_calls: list[dict] = [] - def walk_calls(node, caller_nid: str) -> None: + def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: if node.type == "function_call_expression": for c in node.children: if c.type == "function": - callee = _text(c).split("::")[-1] + # `Acme::Helper::emit` -> callee `emit`, callee_package + # `Acme::Helper`; a bare `emit` -> callee `emit`, no package. + # The qualifier + caller package let the shared second pass + # bind to the right same-named sub instead of any `emit()`. + parts = _text(c).split("::") + callee = parts[-1] + callee_package = "::".join(parts[:-1]) or None if callee and callee not in _PERL_BUILTINS: raw_calls.append({ "caller_nid": caller_nid, "callee": callee, + "callee_package": callee_package, + "caller_package": caller_package, "is_member_call": False, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -219,10 +271,10 @@ def walk_calls(node, caller_nid: str) -> None: }) break for child in node.children: - walk_calls(child, caller_nid) + walk_calls(child, caller_nid, caller_package) - for caller_nid, body in sub_bodies: - walk_calls(body, caller_nid) + for caller_nid, body, caller_package in sub_bodies: + walk_calls(body, caller_nid, caller_package) clean_edges = [e for e in edges if e["source"] in seen_ids and (e["target"] in seen_ids or e["relation"] == "imports")] diff --git a/tests/test_languages.py b/tests/test_languages.py index f4e1124fd..effa2b2c9 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3064,6 +3064,74 @@ def test_perl_no_dangling_edges(): for e in r["edges"]: assert e["source"] in node_ids, f"dangling edge source: {e}" +def test_perl_inherit_stub_has_no_source_file(): + """Inheritance parents defined outside this file (RTL / another module) must + not claim the child file as their source; they are external stubs with an + empty source_file (mirrors go/julia/objc).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + stubs = [n for n in r["nodes"] + if n["label"] in ("Acme::Base", "Acme::Role", "Acme::Mixin")] + assert stubs, "expected external inheritance stub nodes" + for n in stubs: + assert n.get("source_file") == "", ( + f"external parent stub {n['label']!r} must have empty source_file, " + f"got {n.get('source_file')!r}" + ) + +def test_perl_builtins_not_called(tmp_path): + """Perl core builtins (open/close/system/exec/index/…) surface as call + expressions but are not user subs; they must not become raw-call edges.""" + from graphify.extract import extract_perl + src = tmp_path / "builtins.pm" + src.write_text( + "package B;\n" + "sub run {\n" + " open(my $fh, '<', 'f');\n" + " system('ls');\n" + " my $i = index('abc', 'b');\n" + " my $n = substr('abc', 1);\n" + " close($fh);\n" + "}\n" + "1;\n" + ) + r = extract_perl(src) + callees = {rc["callee"] for rc in r["raw_calls"]} + for b in ("open", "system", "index", "substr", "close"): + assert b not in callees, f"builtin {b!r} must not be a raw call, got {callees}" + +def test_perl_duplicate_use_dedup(tmp_path): + """Two identical `use Foo;` statements must dedup to a single imports edge.""" + from graphify.extract import extract_perl + src = tmp_path / "dup.pm" + src.write_text( + "package D;\n" + "use Foo;\n" + "use Foo;\n" + "1;\n" + ) + r = extract_perl(src) + foo_imports = [e for e in r["edges"] + if e["relation"] == "imports" and "foo" in str(e["target"]).lower()] + assert len(foo_imports) == 1, ( + f"duplicate `use Foo;` must dedup to one edge, got {len(foo_imports)}: {foo_imports}" + ) + +def test_perl_raw_calls_carry_package_qualifiers(): + """The extractor half of package-aware resolution: a bare call carries its + caller's package (and no callee_package); a qualified `Pkg::sub()` carries + the qualifier as callee_package. The shared second pass reads these to bind + the right same-named sub.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + by_callee = {rc["callee"]: rc for rc in r["raw_calls"] if not rc.get("is_member_call")} + bare = by_callee["format_line"] + assert bare["callee_package"] is None, bare + assert bare["caller_package"] == "Acme::Widget", bare + qualified = by_callee["emit"] + assert qualified["callee_package"] == "Acme::Helper", qualified + assert qualified["caller_package"] == "Acme::Widget", qualified + # Perl call resolution runs in the top-level `extract()` second pass (raw_calls -> # name-matched calls), so these drive the public multi-file dispatch, like the @@ -3118,3 +3186,49 @@ def test_perl_method_call_no_edge(): assert not any(tgt == "render" or tgt.endswith("::render") or tgt.endswith(".render") for _src, tgt in calls), \ f"method call $widget->render() must not produce a calls edge, got {calls}" + + +# Package-aware second-pass resolution (F1/F3). Same S4 dependency as the corpus +# tests above: the shared pass reads the raw_calls' package qualifiers, but no +# calls edge exists until the .pl/.pm dispatch is registered — RED until S4. + +def _calls_with_target_pkg(r): + """(caller_label, target_label, target_enclosing_package_label) per calls edge.""" + label = {n["id"]: n.get("label", "") for n in r["nodes"]} + contained_by = {e["target"]: e["source"] for e in r["edges"] if e["relation"] == "contains"} + return [ + (label.get(e["source"], e["source"]), + label.get(e["target"], e["target"]), + label.get(contained_by.get(e["target"]), "")) + for e in r["edges"] if e["relation"] == "calls" + ] + +def test_perl_qualified_call_binds_correct_package(tmp_path): + """Two packages define `emit`; the package-qualified call `P::A::emit()` must + bind only to P::A's emit, never the same-named sub in P::B.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "b.pm").write_text("package P::B;\nsub emit { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text("package P::C;\nsub run { P::A::emit(); }\n1;\n") + r = extract([tmp_path / "a.pm", tmp_path / "b.pm", tmp_path / "c.pm"], parallel=False) + emit_calls = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "emit" in t] + assert emit_calls, f"expected a calls edge into emit, got none: {_calls_with_target_pkg(r)}" + assert all(p == "P::A" for (_s, _t, p) in emit_calls), \ + f"qualified P::A::emit() must bind only to P::A's emit, got {emit_calls}" + +def test_perl_bare_foreign_package_call_no_edge(tmp_path): + """A bare `helper()` call resolves to a same-package sub, never to a + same-named sub in a DIFFERENT, un-imported package (zero-edge, not a guess).""" + from graphify.extract import extract + (tmp_path / "x.pm").write_text("package X;\nsub helper { return 1; }\n1;\n") + (tmp_path / "y.pm").write_text( + "package Y;\nsub run { helper(); other(); }\nsub other { return 2; }\n1;\n" + ) + r = extract([tmp_path / "x.pm", tmp_path / "y.pm"], parallel=False) + calls = _calls(r) + # Positive guard: the same-package bare call must resolve, so the negative + # assertion below is not a vacuous pass on an empty (pre-S4) graph. + assert any("run" in s and "other" in t for s, t in calls), \ + f"expected run->other same-package resolution, got {calls}" + assert not any("helper" in t for _s, t in calls), \ + f"bare helper() must not bind to X::helper without import evidence, got {calls}" From f1feb0316d0d46bae9e9ef531b95af153fd44d57 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:32:05 +0000 Subject: [PATCH 04/21] feat(extract): dispatch .pl/.pm and perl shebang to perl extractor Register Perl in the four extension/dispatch tables so .pl/.pm files and extensionless #!/usr/bin/perl scripts reach extract_perl: - _DISPATCH: .pl/.pm -> extract_perl - _SHEBANG_DISPATCH: perl -> extract_perl (detect already routes perl shebang) - suffix->language map: .pl/.pm -> perl (language stats) - detect.CODE_EXTENSIONS: .pl/.pm (watch._WATCHED_EXTENSIONS inherits) .t deliberately excluded (ambiguous across ecosystems). Moves the perl example in test_extract from the unsupported-shebang test (now extract_perl) to the supported-dispatch test; tcsh takes its place as a known-but-unmapped interpreter. --- graphify/detect.py | 2 +- graphify/extract.py | 6 +++++- tests/test_extract.py | 11 ++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 8c0f64a8f..2b6dd02c7 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -27,7 +27,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.pl', '.pm', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index 02d104035..f707b7c0b 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1786,6 +1786,7 @@ def _lang_is_case_insensitive(source_file: object) -> bool: ".lua": "lua", ".luau": "lua", ".zig": "zig", ".ex": "elixir", ".exs": "elixir", + ".pl": "perl", ".pm": "perl", ".jl": "julia", ".dart": "dart", ".sh": "shell", ".bash": "shell", @@ -3778,6 +3779,8 @@ def add_existing_edge(edge: dict) -> None: ".psd1": extract_powershell_manifest, ".ex": extract_elixir, ".exs": extract_elixir, + ".pl": extract_perl, + ".pm": extract_perl, ".m": extract_objc, ".mm": extract_objc, ".jl": extract_julia, @@ -3854,7 +3857,7 @@ def add_existing_edge(edge: dict) -> None: # routes them to the CODE path via _shebang_interpreter; _get_extractor must # honor the same signal or these files are classified as code and then silently # dropped by extraction. Only interpreters with a real extractor are mapped — -# detect's wider set (perl, fish, tcsh, Rscript) stays unmapped and skipped. +# detect's wider set (fish, tcsh, Rscript) stays unmapped and skipped. _SHEBANG_DISPATCH: dict[str, Any] = { "python": extract_python, "python2": extract_python, @@ -3870,6 +3873,7 @@ def add_existing_edge(edge: dict) -> None: "lua": extract_lua, "php": extract_php, "julia": extract_julia, + "perl": extract_perl, } diff --git a/tests/test_extract.py b/tests/test_extract.py index 6247ec59c..eeffd8934 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1618,6 +1618,11 @@ def test_extensionless_shebang_via_dispatch(tmp_path): split.write_text("#!/usr/bin/env -S bash -eu\necho hi\n") assert _get_extractor(split) is extract_bash + from graphify.extract import extract_perl + perltool = tmp_path / "legacy" + perltool.write_text("#!/usr/bin/perl\nprint 1;\n") + assert _get_extractor(perltool) is extract_perl + def test_extensionless_without_usable_shebang_stays_unsupported(tmp_path): from graphify.extract import _get_extractor @@ -1628,9 +1633,9 @@ def test_extensionless_without_usable_shebang_stays_unsupported(tmp_path): # Interpreter known to detect but with no AST extractor: stays skipped # rather than being mis-parsed by a wrong grammar. - perl = tmp_path / "legacy" - perl.write_text("#!/usr/bin/env perl\nprint 1;\n") - assert _get_extractor(perl) is None + tcsh = tmp_path / "legacy" + tcsh.write_text("#!/usr/bin/env tcsh\necho 1\n") + assert _get_extractor(tcsh) is None def test_extract_extensionless_bash_cli_end_to_end(tmp_path): From 5cc35648a62bd4922d450c0cb4299435466a14a6 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:40:47 +0000 Subject: [PATCH 05/21] fix(extract): bare perl call resolves to sub in use-imported package The package-aware bare-call fallback only accepted direct symbol/module import evidence, but `use P::A;` emits an imports edge to the module label id (_make_id('P::A')), never to the sub id. A bare emit() call to a sub defined in a use-imported foreign package therefore dropped even though the import unambiguously names its package. Bridge it with _has_package_import_evidence: the candidate sub's enclosing package, re-idized the same way the use target is, must be among the caller file's imports. Strictly scoped to the Perl/package branch; ambiguity (two imported packages, same sub name) still drops. Constraint: zero-edge over a guess preserved for the ambiguous case. --- graphify/extract.py | 14 +++++++++++++- tests/test_languages.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/graphify/extract.py b/graphify/extract.py index f707b7c0b..dd82cd7f2 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4689,6 +4689,15 @@ def _has_import_evidence(candidate_id: str) -> bool: or (candidate_file_nid is not None and candidate_file_nid in imported_modules) ) + def _has_package_import_evidence(candidate_id: str) -> bool: + # Perl-only: `use P::A;` emits an imports edge to the MODULE label id + # (`_make_id('P::A')`), never to the sub id — so a bare call to an + # imported package's sub has no direct symbol/module evidence above. + # Bridge it: the candidate sub's enclosing package, re-idized the same + # way the `use` target is, must be among the caller file's imports. + pkg = pkg_label_by_nid.get(candidate_id, "") + return bool(pkg) and _make_id(pkg) in imported_symbols + # Package-aware pre-filter for languages that tag calls with their # package (Perl). Additive + guarded: a raw_call only reaches this branch # when it carries a package field, which no other language sets, so every @@ -4718,7 +4727,10 @@ def _has_import_evidence(candidate_id: str) -> bool: if same_pkg: candidates = same_pkg else: - candidates = [c for c in candidates if _has_import_evidence(c)] + candidates = [ + c for c in candidates + if _has_import_evidence(c) or _has_package_import_evidence(c) + ] if len(candidates) != 1: continue diff --git a/tests/test_languages.py b/tests/test_languages.py index effa2b2c9..2888b43dc 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3232,3 +3232,40 @@ def test_perl_bare_foreign_package_call_no_edge(tmp_path): f"expected run->other same-package resolution, got {calls}" assert not any("helper" in t for _s, t in calls), \ f"bare helper() must not bind to X::helper without import evidence, got {calls}" + + +def test_perl_bare_call_binds_imported_package(tmp_path): + """A bare `emit()` call resolves to a sub in a FOREIGN package when that + package is pulled in with `use P::A;` — the `use` import is the evidence that + disambiguates the otherwise-foreign sub. `use P::A;` emits an imports edge to + the module label (`_make_id('P::A')`), not to the sub id, so plain + symbol-import evidence misses it; package-import evidence must bridge the gap.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nsub run { emit(); }\n1;\n" + ) + r = extract([tmp_path / "a.pm", tmp_path / "c.pm"], parallel=False) + emit_calls = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "emit" in t] + assert emit_calls, \ + f"bare emit() with `use P::A;` must bind to P::A::emit, got none: {_calls_with_target_pkg(r)}" + assert all(p == "P::A" for (_s, _t, p) in emit_calls), \ + f"emit() must bind only to the imported P::A::emit, got {emit_calls}" + + +def test_perl_bare_call_two_imported_packages_ambiguous_no_edge(tmp_path): + """Two imported packages both define `emit`; a bare `emit()` is ambiguous — + the `use` import evidence points at two candidates, so the edge is dropped + (zero-edge over a guess), same discipline as the qualified-drop path.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "b.pm").write_text("package P::B;\nsub emit { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nuse P::B;\nsub run { emit(); }\n1;\n" + ) + r = extract( + [tmp_path / "a.pm", tmp_path / "b.pm", tmp_path / "c.pm"], parallel=False + ) + calls = _calls(r) + assert not any("emit" in t for _s, t in calls), \ + f"bare emit() imported from two packages is ambiguous and must drop, got {calls}" From cbed7449240602b0255440fdd7cb1815df9401d1 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:09:41 +0000 Subject: [PATCH 06/21] fix(extract): repoint perl imports to in-corpus packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `use Foo::Bar;` / `require Foo::Bar;` emit an imports edge to a bare module-label id (`_make_id('Foo::Bar')`) that matches no node when Foo::Bar is a package defined elsewhere in the corpus — its package node's id is `_make_id(stem, 'Foo::Bar')`. Add `_resolve_perl_imports`, run in the top-level `extract()` AFTER the shared cross-file call pass, to re-point those edges onto the real package node keyed by package LABEL (multi-package files have unrelated stems). External modules keep their bare, dangling target, matching how other languages leave unresolved external imports. Ordering is load-bearing: `_has_package_import_evidence` reads imports targets as bare module-label ids to bind a bare call to an imported package's sub, so the re-point must run after call resolution, not before. On the Foswiki core corpus (378 files): dangling imports 671 -> 174 (497 in-corpus edges re-pointed; the 174 remaining are genuine externals). --- graphify/extract.py | 11 ++++++++- graphify/extractors/perl.py | 48 ++++++++++++++++++++++++++++++++++++ tests/test_languages.py | 49 +++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/graphify/extract.py b/graphify/extract.py index dd82cd7f2..c12610ec0 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -44,7 +44,7 @@ from graphify.extractors.json_config import extract_json # noqa: F401 from graphify.extractors.markdown import extract_markdown # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 -from graphify.extractors.perl import extract_perl # noqa: F401 +from graphify.extractors.perl import _resolve_perl_imports, extract_perl # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 @@ -4839,6 +4839,15 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # a new language plugs in without editing this body (#1356 Swift, #1446 Python). run_language_resolvers(paths, per_file, all_nodes, all_edges) + # Re-point dangling in-corpus Perl `imports` edges onto the real package node. + # Runs AFTER the call pass above so `_has_package_import_evidence` still sees the + # bare module-label ids it needs to bind bare calls to imported packages (#S5b). + try: + _resolve_perl_imports(all_nodes, all_edges) + except Exception as exc: + import logging + logging.getLogger(__name__).warning("Perl import resolution failed, skipping: %s", exc) + # Relativize source_file fields so paths are portable across machines (#555) for item in all_nodes + all_edges: sf = item.get("source_file") diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index e1ce68b6b..9353e894f 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -280,3 +280,51 @@ def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: (e["target"] in seen_ids or e["relation"] == "imports")] return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, "input_tokens": 0, "output_tokens": 0} + + +def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: + """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. + + ``use Foo::Bar;`` / ``require Foo::Bar;`` emit an imports edge to a bare + module-label id (``_make_id('Foo::Bar')``) that matches no node: when Foo::Bar + is defined in the corpus its package node's id is ``_make_id(stem, 'Foo::Bar')`` + (and a ``package Foo::Bar;`` may live in a file whose stem is unrelated — a + multi-package file — so we key on the package LABEL, not the file stem). Bridge + module-label -> real package id so in-corpus imports connect instead of + dangling. External modules (POSIX, Carp, …) have no in-corpus package node, so + their edge keeps its bare target — matching the dangling-stub behavior other + languages leave on unresolved external imports. + + Runs AFTER the shared cross-file call pass: ``_has_package_import_evidence`` + (extract.py) reads imports targets as bare module-label ids to bind a bare call + to an imported package's sub, so re-pointing before it would break that binding. + Mutates ``all_edges`` in place; the bare target was never a node, so there is + nothing to prune. + """ + # module-label id -> package node id. First package with a given fully- + # qualified label wins (deterministic in node order); a bare `use Foo` can't + # disambiguate a re-opened same-name package, and picking one beats dangling. + pkg_by_label_id: dict[str, str] = {} + for node in all_nodes: + src = str(node.get("source_file") or "") + if not src.endswith((".pl", ".pm")): + continue + label = node.get("label", "") + nid = node.get("id", "") + # package nodes only: skip the file node (label == basename) and subs + # (label ends in `()`); neither keys an import target. + if not label or not nid or label.endswith(")") or label == Path(src).name: + continue + pkg_by_label_id.setdefault(_make_id(label), nid) + if not pkg_by_label_id: + return + for edge in all_edges: + if edge.get("relation") != "imports": + continue + # Scope to Perl imports only, so a same-named module-label id in another + # language's imports edge is never re-pointed onto a Perl package node. + if not str(edge.get("source_file") or "").endswith((".pl", ".pm")): + continue + real = pkg_by_label_id.get(edge.get("target")) + if real and real != edge["target"]: + edge["target"] = real diff --git a/tests/test_languages.py b/tests/test_languages.py index 2888b43dc..5ffd367cc 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3269,3 +3269,52 @@ def test_perl_bare_call_two_imported_packages_ambiguous_no_edge(tmp_path): calls = _calls(r) assert not any("emit" in t for _s, t in calls), \ f"bare emit() imported from two packages is ambiguous and must drop, got {calls}" + + +# In-corpus import re-pointer (S5b Befund A): a `use`/`require` whose module is a +# package defined elsewhere in the corpus must re-point its imports edge from the +# bare module-label id onto the real package node; external modules stay dangling. + +def test_perl_import_repoints_to_in_corpus_package(tmp_path): + """`use Acme::Helper;` from another file must re-point its imports edge onto + the real Acme::Helper package node (id `_make_id(stem, 'Acme::Helper')`), + not dangle on the bare module-label id `_make_id('Acme::Helper')`. An external + `use POSIX;` has no in-corpus package, so its edge keeps the bare (node-less) + target — mirroring how other languages leave external imports dangling.""" + from graphify.extract import extract + from graphify.extractors.base import _make_id + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + (tmp_path / "main.pm").write_text( + "package Main;\nuse Acme::Helper;\nuse POSIX qw(floor);\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", tmp_path / "main.pm"], parallel=False) + node_ids = {n["id"] for n in r["nodes"]} + pkg_nodes = [n for n in r["nodes"] if n.get("label") == "Acme::Helper"] + assert len(pkg_nodes) == 1, f"expected one Acme::Helper package node, got {pkg_nodes}" + pkg_id = pkg_nodes[0]["id"] + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"use Acme::Helper must re-point to package node {pkg_id}, got {[e['target'] for e in imports]}" + bare_helper = _make_id("Acme::Helper") + assert bare_helper != pkg_id + assert not any(e["target"] == bare_helper for e in imports), \ + "the in-corpus import must no longer dangle on the bare module-label id" + posix_id = _make_id("POSIX") + assert any(e["target"] == posix_id for e in imports), "external POSIX import edge missing" + assert posix_id not in node_ids, \ + "external POSIX must stay a dangling label-id stub, not become a node" + + +def test_perl_require_repoints_to_in_corpus_package(tmp_path): + """`require Acme::Helper;` (bareword require form) re-points the same way as + `use` when the module is an in-corpus package.""" + from graphify.extract import extract + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + (tmp_path / "main.pm").write_text( + "package Main;\nrequire Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", tmp_path / "main.pm"], parallel=False) + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"require Acme::Helper must re-point to package node {pkg_id}, got {[e['target'] for e in imports]}" From 9e0829d4f210e11519782ce329342f384e88e4a1 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:10:30 +0000 Subject: [PATCH 07/21] fix(extract): treat indirect-object new as member call `my $cgi = new CGI();` (indirect-object constructor syntax, == `CGI->new`) parses as ambiguous_function_call_expression(function 'new', function_call_expression 'CGI()'); the inner `CGI()` was recorded as a bare function call, so a same-named sub in the corpus got a spurious `calls` edge. Detect the leading `new` on the parent node and mark the inner call `is_member_call` (edge-less), like any other untyped `$obj->meth()` dispatch. On the Foswiki core corpus this removes exactly the one wrong edge (prepareBody -> CGI, EXTRACTED) and changes nothing else. --- graphify/extractors/perl.py | 14 +++++++++++++- tests/test_languages.py | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 9353e894f..afa8036a0 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -237,6 +237,18 @@ def walk_statements(node) -> None: def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: if node.type == "function_call_expression": + # Indirect-object constructor `new CLASS(...)` parses as + # ambiguous_function_call_expression(function 'new', function_call_expression + # 'CLASS()'); `new CLASS` == CLASS->new, an untyped member dispatch. Mark + # it a member call (edge-less) so it is not wired to a sub named CLASS. + parent = node.parent + indirect_new = ( + parent is not None + and parent.type == "ambiguous_function_call_expression" + and bool(parent.children) + and parent.children[0].type == "function" + and _text(parent.children[0]) == "new" + ) for c in node.children: if c.type == "function": # `Acme::Helper::emit` -> callee `emit`, callee_package @@ -252,7 +264,7 @@ def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: "callee": callee, "callee_package": callee_package, "caller_package": caller_package, - "is_member_call": False, + "is_member_call": indirect_new, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) diff --git a/tests/test_languages.py b/tests/test_languages.py index 5ffd367cc..cb5411785 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3188,6 +3188,27 @@ def test_perl_method_call_no_edge(): f"method call $widget->render() must not produce a calls edge, got {calls}" +def test_perl_indirect_object_new_no_call_edge(tmp_path): + """`my $x = new Widget();` is indirect-object constructor syntax (== Widget->new), + an untyped member dispatch; it must NOT bind to a same-named sub `Widget` as if + `Widget()` were a direct function call.""" + from graphify.extract import extract + (tmp_path / "m.pm").write_text( + "package M;\n" + "sub Widget { return 1; }\n" + "sub helper { return 2; }\n" + "sub run { helper(); my $x = new Widget(); }\n" + "1;\n" + ) + r = extract([tmp_path / "m.pm"], parallel=False) + calls = _calls(r) + # Positive guard: the plain bare call resolves, so the negative isn't vacuous. + assert any("run" in s and "helper" in t for s, t in calls), \ + f"expected run->helper bare-call resolution, got {calls}" + assert not any("Widget" in t for _s, t in calls), \ + f"indirect-object `new Widget()` must not create a calls edge, got {calls}" + + # Package-aware second-pass resolution (F1/F3). Same S4 dependency as the corpus # tests above: the shared pass reads the raw_calls' package qualifiers, but no # calls edge exists until the .pl/.pm dispatch is registered — RED until S4. From e2766f81b514751b9965d37ca333d9e4e30b72fa Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:28:28 +0000 Subject: [PATCH 08/21] fix(extract): only repoint perl imports on unique package label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A re-opened Perl package declared under the same fully-qualified label in two files (Foswiki-real: package Assert; in AssertOn.pm AND AssertOff.pm) is ambiguous: a bare `use P::A;` cannot say which file it means. The old setdefault map let the first package node in node order win, producing a guessed cross-file import edge — worse than dangling. Collect label_id -> list[node_ids] and re-point only when exactly one package node carries the label; >1 candidate stays dangling on the bare module-label id (zero-edge over a guess). Finding F1 (P1) from Review-3 of the S5b import re-pointer. --- graphify/extractors/perl.py | 20 +++++++++++--------- tests/test_languages.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index afa8036a0..f38c14b8d 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -313,10 +313,12 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: Mutates ``all_edges`` in place; the bare target was never a node, so there is nothing to prune. """ - # module-label id -> package node id. First package with a given fully- - # qualified label wins (deterministic in node order); a bare `use Foo` can't - # disambiguate a re-opened same-name package, and picking one beats dangling. - pkg_by_label_id: dict[str, str] = {} + # module-label id -> list of package node ids carrying that fully-qualified + # label. A bare `use Foo` cannot disambiguate a package re-opened under the + # same label across files (Foswiki-real: `package Assert;` in AssertOn.pm AND + # AssertOff.pm), so we re-point ONLY when exactly one package node matches; + # >1 candidate stays dangling (zero-edge over a guessed cross-file binding). + pkg_ids_by_label_id: dict[str, list[str]] = {} for node in all_nodes: src = str(node.get("source_file") or "") if not src.endswith((".pl", ".pm")): @@ -327,8 +329,8 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: # (label ends in `()`); neither keys an import target. if not label or not nid or label.endswith(")") or label == Path(src).name: continue - pkg_by_label_id.setdefault(_make_id(label), nid) - if not pkg_by_label_id: + pkg_ids_by_label_id.setdefault(_make_id(label), []).append(nid) + if not pkg_ids_by_label_id: return for edge in all_edges: if edge.get("relation") != "imports": @@ -337,6 +339,6 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: # language's imports edge is never re-pointed onto a Perl package node. if not str(edge.get("source_file") or "").endswith((".pl", ".pm")): continue - real = pkg_by_label_id.get(edge.get("target")) - if real and real != edge["target"]: - edge["target"] = real + candidates = pkg_ids_by_label_id.get(edge.get("target")) + if candidates and len(candidates) == 1 and candidates[0] != edge["target"]: + edge["target"] = candidates[0] diff --git a/tests/test_languages.py b/tests/test_languages.py index cb5411785..95cc3b9f2 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3339,3 +3339,31 @@ def test_perl_require_repoints_to_in_corpus_package(tmp_path): imports = [e for e in r["edges"] if e["relation"] == "imports"] assert any(e["target"] == pkg_id for e in imports), \ f"require Acme::Helper must re-point to package node {pkg_id}, got {[e['target'] for e in imports]}" + + +def test_perl_import_duplicate_package_label_stays_dangling(tmp_path): + """A re-opened package declared under the SAME fully-qualified label in two + files (Foswiki-real: `package Assert;` in both AssertOn.pm and AssertOff.pm) + is ambiguous: a bare `use P::A;` cannot say which file it means. Binding it to + one file arbitrarily is a guessed edge — worse than dangling — so the imports + edge must stay on the bare module-label id (no package node) when >1 package + node carries the label. Only a unique label re-points (zero-edge over a guess).""" + from graphify.extract import extract + from graphify.extractors.base import _make_id + (tmp_path / "a1.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "a2.pm").write_text("package P::A;\nsub other { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nsub run { return 1; }\n1;\n" + ) + r = extract( + [tmp_path / "a1.pm", tmp_path / "a2.pm", tmp_path / "c.pm"], parallel=False + ) + pkg_nodes = [n for n in r["nodes"] if n.get("label") == "P::A"] + assert len(pkg_nodes) == 2, f"expected two P::A package nodes, got {pkg_nodes}" + pkg_ids = {n["id"] for n in pkg_nodes} + imports = [e for e in r["edges"] if e["relation"] == "imports"] + bare_id = _make_id("P::A") + assert any(e["target"] == bare_id for e in imports), \ + f"ambiguous use P::A; must stay dangling on bare id {bare_id}, got {[e['target'] for e in imports]}" + assert not any(e["target"] in pkg_ids for e in imports), \ + f"ambiguous use P::A; must NOT bind to either P::A file node, got {[e['target'] for e in imports]}" From 1d678cfc0d757e1e2ac1fdea79227d18c974c546 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:29:29 +0000 Subject: [PATCH 09/21] fix(extract): scope perl import re-pointer by extractor provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-pointer scoped package nodes and imports edges by .pl/.pm suffix, so extensionless #!/usr/bin/perl scripts — dispatched to extract_perl by shebang since S4 — had their imports never re-pointed (underreporting). extract() now passes the set of Perl source paths (where _get_extractor is extract_perl) into _resolve_perl_imports, which scopes on membership instead of suffix. Keyed on str(path) to match the nodes' source_file before relativization. Falls back to suffix matching when the set is omitted (direct callers). Finding F2 (P2) from Review-3 of the S5b import re-pointer. --- graphify/extract.py | 6 +++++- graphify/extractors/perl.py | 22 +++++++++++++++++++--- tests/test_languages.py | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index c12610ec0..2c157a013 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4842,8 +4842,12 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # Re-point dangling in-corpus Perl `imports` edges onto the real package node. # Runs AFTER the call pass above so `_has_package_import_evidence` still sees the # bare module-label ids it needs to bind bare calls to imported packages (#S5b). + # Scope by extractor provenance, not suffix, so extensionless `#!/usr/bin/perl` + # scripts (dispatched to extract_perl by shebang) are re-pointed too. Keyed on + # str(path) to match the nodes' source_file, set before the relativization below. try: - _resolve_perl_imports(all_nodes, all_edges) + _perl_sources = {str(p) for p in paths if _get_extractor(p) is extract_perl} + _resolve_perl_imports(all_nodes, all_edges, _perl_sources) except Exception as exc: import logging logging.getLogger(__name__).warning("Perl import resolution failed, skipping: %s", exc) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index f38c14b8d..30b42262b 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -294,7 +294,11 @@ def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: "input_tokens": 0, "output_tokens": 0} -def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: +def _resolve_perl_imports( + all_nodes: list[dict], + all_edges: list[dict], + perl_source_files: set[str] | None = None, +) -> None: """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. ``use Foo::Bar;`` / ``require Foo::Bar;`` emit an imports edge to a bare @@ -307,12 +311,24 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: their edge keeps its bare target — matching the dangling-stub behavior other languages leave on unresolved external imports. + ``perl_source_files`` (absolute-path strings, from ``extract()`` where + ``_get_extractor(path) is extract_perl``) scopes both the package-node scan and + the re-pointed edges to Perl provenance. Suffix-based scoping (``.pl``/``.pm``) + silently skipped extensionless ``#!/usr/bin/perl`` scripts, which are dispatched + to extract_perl by shebang and whose imports were never re-pointed + (underreporting). When ``None`` (direct callers), falls back to suffix matching. + Runs AFTER the shared cross-file call pass: ``_has_package_import_evidence`` (extract.py) reads imports targets as bare module-label ids to bind a bare call to an imported package's sub, so re-pointing before it would break that binding. Mutates ``all_edges`` in place; the bare target was never a node, so there is nothing to prune. """ + def _is_perl_source(src: str) -> bool: + if perl_source_files is not None: + return src in perl_source_files + return src.endswith((".pl", ".pm")) + # module-label id -> list of package node ids carrying that fully-qualified # label. A bare `use Foo` cannot disambiguate a package re-opened under the # same label across files (Foswiki-real: `package Assert;` in AssertOn.pm AND @@ -321,7 +337,7 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: pkg_ids_by_label_id: dict[str, list[str]] = {} for node in all_nodes: src = str(node.get("source_file") or "") - if not src.endswith((".pl", ".pm")): + if not _is_perl_source(src): continue label = node.get("label", "") nid = node.get("id", "") @@ -337,7 +353,7 @@ def _resolve_perl_imports(all_nodes: list[dict], all_edges: list[dict]) -> None: continue # Scope to Perl imports only, so a same-named module-label id in another # language's imports edge is never re-pointed onto a Perl package node. - if not str(edge.get("source_file") or "").endswith((".pl", ".pm")): + if not _is_perl_source(str(edge.get("source_file") or "")): continue candidates = pkg_ids_by_label_id.get(edge.get("target")) if candidates and len(candidates) == 1 and candidates[0] != edge["target"]: diff --git a/tests/test_languages.py b/tests/test_languages.py index 95cc3b9f2..94a4d4d44 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3367,3 +3367,21 @@ def test_perl_import_duplicate_package_label_stays_dangling(tmp_path): f"ambiguous use P::A; must stay dangling on bare id {bare_id}, got {[e['target'] for e in imports]}" assert not any(e["target"] in pkg_ids for e in imports), \ f"ambiguous use P::A; must NOT bind to either P::A file node, got {[e['target'] for e in imports]}" + + +def test_perl_import_repoints_from_shebang_perl_file(tmp_path): + """An extensionless `#!/usr/bin/perl` script is dispatched to extract_perl by + shebang, but its source_file has no .pl/.pm suffix. Scoping the re-pointer by + file suffix would silently skip such a file's imports (underreporting); scoping + by extractor provenance re-points them like any other Perl source.""" + from graphify.extract import extract + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + script = tmp_path / "runme" + script.write_text( + "#!/usr/bin/perl\npackage Main;\nuse Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", script], parallel=False) + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"use Acme::Helper in a shebang-perl file must re-point to {pkg_id}, got {[e['target'] for e in imports]}" From 0da9fd75d6eb53bd7d8edffa2ebbc8d48a517dee Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:02:11 +0000 Subject: [PATCH 10/21] fix(extract): walk block-form perl packages without leaking package state `package Foo { ... }` created the Foo node but never walked the block, so subs inside it were lost, and the current package leaked to following top-level subs (mis-attributed to Foo). Distinguish block-form from statement-form: push package state, walk the block, restore state. --- graphify/extractors/perl.py | 16 +++++++++-- tests/test_languages.py | 53 ++++++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 30b42262b..28d68b964 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -205,8 +205,20 @@ def walk_statements(node) -> None: pkg_nid = _make_id(stem, name) add_node(pkg_nid, name, line) add_edge(file_nid, pkg_nid, "contains", line) - current_pkg_nid = pkg_nid - current_pkg_name = name + pkg_block = next( + (c for c in child.children if c.type == "block"), None) + if pkg_block is not None: + # Block-form `package Foo { ... }` scopes Foo to the block + # only. Walk the block under Foo, then restore the prior + # package so following top-level statements are not + # mis-attributed to Foo (and the block's subs are not lost). + prev_nid, prev_name = current_pkg_nid, current_pkg_name + current_pkg_nid, current_pkg_name = pkg_nid, name + walk_statements(pkg_block) + current_pkg_nid, current_pkg_name = prev_nid, prev_name + else: + current_pkg_nid = pkg_nid + current_pkg_name = name elif child.type == "use_statement": handle_use(child, line) elif child.type == "subroutine_declaration_statement": diff --git a/tests/test_languages.py b/tests/test_languages.py index 94a4d4d44..9d85e96af 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3209,9 +3209,56 @@ def test_perl_indirect_object_new_no_call_edge(tmp_path): f"indirect-object `new Widget()` must not create a calls edge, got {calls}" -# Package-aware second-pass resolution (F1/F3). Same S4 dependency as the corpus -# tests above: the shared pass reads the raw_calls' package qualifiers, but no -# calls edge exists until the .pl/.pm dispatch is registered — RED until S4. +def test_perl_block_scoped_package(tmp_path): + """Block-form `package Foo { ... }` scopes Foo to the block only: subs inside + the block belong to Foo, and a sub AFTER the block belongs to the package that + was current before the block (no state leak).""" + from graphify.extract import extract_perl + src = tmp_path / "block.pm" + src.write_text( + "package Outer;\n" + "sub outer_sub { }\n" + "package Acme::Block {\n" + " sub inner { }\n" + "}\n" + "sub after_block { }\n" + "1;\n" + ) + r = extract_perl(src) + label = {n["id"]: n["label"] for n in r["nodes"]} + container_of = { + label.get(e["target"]): label.get(e["source"]) + for e in r["edges"] if e["relation"] == "contains" + } + assert "Acme::Block" in label.values(), "block-form package node must be emitted" + assert container_of.get("inner()") == "Acme::Block", ( + f"sub inside a block package must belong to it, got {container_of.get('inner()')!r}" + ) + assert container_of.get("after_block()") == "Outer", ( + f"block-form package must not leak: after_block belongs to Outer, " + f"got {container_of.get('after_block()')!r}" + ) + + +def test_perl_block_scoped_package_body_calls(tmp_path): + """A bare call inside a block-package sub resolves within that package — + proves the block body is actually walked (not silently skipped).""" + from graphify.extract import extract + (tmp_path / "b.pm").write_text( + "package B::Blk {\n" + " sub inner { helper(); }\n" + " sub helper { return 1; }\n" + "}\n" + "1;\n" + ) + r = extract([tmp_path / "b.pm"], parallel=False) + calls = _calls(r) + assert any("inner" in s and "helper" in t for s, t in calls), \ + f"expected inner->helper inside the block package, got {calls}" + + +# Package-aware second-pass resolution: the shared pass reads the raw_calls' +# package qualifiers to bind a call to the same-named sub in the right package. def _calls_with_target_pkg(r): """(caller_label, target_label, target_enclosing_package_label) per calls edge.""" From 5d0355f408418c1e415100ecece6e77f317860ae Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:02:25 +0000 Subject: [PATCH 11/21] fix(extract): model qualified perl sub declarations in their named package `sub Bar::baz {...}` was stored as label `Bar::baz()` under the current package, so a call `Bar::baz()` (callee baz, package Bar) could never resolve and the body's caller package was wrong. Split the qualified name: the container is the qualified package (created if it has no `package` statement of its own), the node label is `baz()`, and the sub body carries caller package Bar. --- graphify/extractors/perl.py | 22 ++++++++++++++++++---- tests/test_languages.py | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 28d68b964..d92dcee31 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -230,12 +230,26 @@ def walk_statements(node) -> None: elif c.type == "block": block = c if name: - container = current_pkg_nid or file_nid - sub_nid = _make_id(container, name) - add_node(sub_nid, f"{name}()", line) + if "::" in name: + # Qualified declaration `sub Pkg::sub {...}` defines the sub + # IN the named package, not the current one. Container = that + # package (created if it has no `package` statement of its + # own); the body's caller-package is the qualifier so its + # calls resolve against Pkg. + pkg_qual, _, sub_name = name.rpartition("::") + container = _make_id(stem, pkg_qual) + add_node(container, pkg_qual, line) + add_edge(file_nid, container, "contains", line) + sub_package = pkg_qual + else: + container = current_pkg_nid or file_nid + sub_name = name + sub_package = current_pkg_name + sub_nid = _make_id(container, sub_name) + add_node(sub_nid, f"{sub_name}()", line) add_edge(container, sub_nid, "contains", line) if block is not None: - sub_bodies.append((sub_nid, block, current_pkg_name)) + sub_bodies.append((sub_nid, block, sub_package)) elif child.type == "expression_statement": for c in child.children: if c.type == "require_expression": diff --git a/tests/test_languages.py b/tests/test_languages.py index 9d85e96af..25a9cae93 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3284,6 +3284,26 @@ def test_perl_qualified_call_binds_correct_package(tmp_path): assert all(p == "P::A" for (_s, _t, p) in emit_calls), \ f"qualified P::A::emit() must bind only to P::A's emit, got {emit_calls}" +def test_perl_qualified_sub_declaration_call_edge(tmp_path): + """A qualified declaration `sub Ext::Pkg::helper {...}` defines the sub IN + Ext::Pkg (not the current package). A call `Ext::Pkg::helper()` from another + package must bind to it: the node's label is `helper()` and its enclosing + package is Ext::Pkg (so callee/package-qualifier resolution matches).""" + from graphify.extract import extract + (tmp_path / "q.pm").write_text( + "package Main::Mod;\n" + "sub Ext::Pkg::helper { return 1; }\n" + "sub run { Ext::Pkg::helper(); }\n" + "1;\n" + ) + r = extract([tmp_path / "q.pm"], parallel=False) + binds = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "helper" in t] + assert binds, f"expected run->helper via qualified sub decl, got {_calls_with_target_pkg(r)}" + assert all(t == "helper()" and p == "Ext::Pkg" for (_s, t, p) in binds), ( + f"qualified sub must be labelled helper() and enclosed by Ext::Pkg, got {binds}" + ) + + def test_perl_bare_foreign_package_call_no_edge(tmp_path): """A bare `helper()` call resolves to a same-package sub, never to a same-named sub in a DIFFERENT, un-imported package (zero-edge, not a guess).""" From cc7f83ab1824534d49cedae8222acf4e4e520e8b Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:02:37 +0000 Subject: [PATCH 12/21] chore(extract): tidy perl extractor review notes for upstream - drop `.t` from the extract_perl docstring (only .pl/.pm are registered) - dedupe sprintf/wantarray in _PERL_BUILTINS - reword in-code development notes (TDD-stage and issue-shorthand markers, internal corpus references) as stable invariants - harden the method-call negative test to also reject a bare `render()` label, not only `render`/`::render`/`.render` --- graphify/extract.py | 7 +++---- graphify/extractors/perl.py | 7 +++---- tests/test_languages.py | 26 +++++++++++++------------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 2c157a013..9860bfa99 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4701,10 +4701,9 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # Package-aware pre-filter for languages that tag calls with their # package (Perl). Additive + guarded: a raw_call only reaches this branch # when it carries a package field, which no other language sets, so every - # other language's candidate set is untouched (the existing 314 tests - # prove the unqualified path is unchanged). Zero-edge over a wrong guess: - # an unresolvable qualifier or a foreign-package bare call is dropped, not - # bound to a same-named sub in the wrong package (#F1/#F3). + # other language's candidate set is untouched. Zero-edge over a wrong + # guess: an unresolvable qualifier or a foreign-package bare call is + # dropped, not bound to a same-named sub in the wrong package. callee_package = rc.get("callee_package") caller_package = rc.get("caller_package") if callee_package is not None or caller_package is not None: diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index d92dcee31..2a879c760 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -41,10 +41,9 @@ # list / hash ops "shift", "unshift", "push", "pop", "splice", "map", "grep", "sort", "reverse", "join", "split", "keys", "values", "each", "exists", "delete", - "wantarray", # string ops "index", "rindex", "substr", "length", "uc", "lc", "ucfirst", "lcfirst", - "chomp", "chop", "chr", "ord", "hex", "oct", "sprintf", "pack", "unpack", + "chomp", "chop", "chr", "ord", "hex", "oct", "pack", "unpack", "quotemeta", # math "abs", "int", "sqrt", "sin", "cos", "atan2", "exp", "log", "rand", "srand", @@ -55,7 +54,7 @@ def extract_perl(path: Path) -> dict: - """Extract packages, subs, imports and inheritance from a .pl/.pm/.t file.""" + """Extract packages, subs, imports and inheritance from a .pl/.pm file.""" try: import tree_sitter_perl as tsperl from tree_sitter import Language, Parser @@ -357,7 +356,7 @@ def _is_perl_source(src: str) -> bool: # module-label id -> list of package node ids carrying that fully-qualified # label. A bare `use Foo` cannot disambiguate a package re-opened under the - # same label across files (Foswiki-real: `package Assert;` in AssertOn.pm AND + # same label across files (e.g. `package Assert;` in both AssertOn.pm and # AssertOff.pm), so we re-point ONLY when exactly one package node matches; # >1 candidate stays dangling (zero-edge over a guessed cross-file binding). pkg_ids_by_label_id: dict[str, list[str]] = {} diff --git a/tests/test_languages.py b/tests/test_languages.py index 25a9cae93..82f537abc 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -2948,10 +2948,9 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): # ── Perl ────────────────────────────────────────────────────────────────────── -# RED (S2): the `extract_perl` extractor does not exist yet (S3) and the .pl/.pm -# dispatch is not registered yet (S4). The `extract_perl` import is kept INSIDE -# each test so importing this module still succeeds and the existing suite stays -# GREEN — only the Perl tests fail (ImportError until S3, empty graph until S4). +# `extract_perl` (and `extract`) are imported INSIDE each test rather than at +# module top so this test module still imports even if the extractor is absent — +# only the Perl tests would fail, not the whole suite's collection. def test_perl_no_error(): from graphify.extract import extract_perl @@ -3135,7 +3134,7 @@ def test_perl_raw_calls_carry_package_qualifiers(): # Perl call resolution runs in the top-level `extract()` second pass (raw_calls -> # name-matched calls), so these drive the public multi-file dispatch, like the -# ObjC cross-file tests. RED until BOTH S3 (extractor) and S4 (.pl/.pm dispatch). +# ObjC cross-file tests. def _perl_corpus(): from graphify.extract import extract @@ -3178,12 +3177,13 @@ def test_perl_method_call_no_edge(): r = _perl_corpus() calls = _calls(r) # Positive guard: the corpus must actually produce static/bare calls, so this - # test is not a vacuous pass on an empty graph (RED until S3+S4). + # test is not a vacuous pass on an empty graph. assert any("format_line" in tgt or "emit" in tgt for _src, tgt in calls), \ f"expected the static/bare calls to resolve before asserting the negatives, got {calls}" assert not any("update" in tgt for _src, tgt in calls), \ f"method call $self->update() must not produce a calls edge, got {calls}" - assert not any(tgt == "render" or tgt.endswith("::render") or tgt.endswith(".render") + assert not any(tgt in ("render", "render()") + or tgt.endswith("::render") or tgt.endswith(".render") for _src, tgt in calls), \ f"method call $widget->render() must not produce a calls edge, got {calls}" @@ -3315,7 +3315,7 @@ def test_perl_bare_foreign_package_call_no_edge(tmp_path): r = extract([tmp_path / "x.pm", tmp_path / "y.pm"], parallel=False) calls = _calls(r) # Positive guard: the same-package bare call must resolve, so the negative - # assertion below is not a vacuous pass on an empty (pre-S4) graph. + # assertion below is not a vacuous pass on an empty graph. assert any("run" in s and "other" in t for s, t in calls), \ f"expected run->other same-package resolution, got {calls}" assert not any("helper" in t for _s, t in calls), \ @@ -3359,9 +3359,9 @@ def test_perl_bare_call_two_imported_packages_ambiguous_no_edge(tmp_path): f"bare emit() imported from two packages is ambiguous and must drop, got {calls}" -# In-corpus import re-pointer (S5b Befund A): a `use`/`require` whose module is a -# package defined elsewhere in the corpus must re-point its imports edge from the -# bare module-label id onto the real package node; external modules stay dangling. +# In-corpus import re-pointer: a `use`/`require` whose module is a package defined +# elsewhere in the corpus must re-point its imports edge from the bare module-label +# id onto the real package node; external modules stay dangling. def test_perl_import_repoints_to_in_corpus_package(tmp_path): """`use Acme::Helper;` from another file must re-point its imports edge onto @@ -3410,8 +3410,8 @@ def test_perl_require_repoints_to_in_corpus_package(tmp_path): def test_perl_import_duplicate_package_label_stays_dangling(tmp_path): """A re-opened package declared under the SAME fully-qualified label in two - files (Foswiki-real: `package Assert;` in both AssertOn.pm and AssertOff.pm) - is ambiguous: a bare `use P::A;` cannot say which file it means. Binding it to + files (e.g. `package Assert;` in both AssertOn.pm and AssertOff.pm) is + ambiguous: a bare `use P::A;` cannot say which file it means. Binding it to one file arbitrarily is a guessed edge — worse than dangling — so the imports edge must stay on the bare module-label id (no package node) when >1 package node carries the label. Only a unique label re-points (zero-edge over a guess).""" From 5bfef602dffdb0b589d4754191b0677929a3abe1 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:47:24 +0000 Subject: [PATCH 13/21] fix(extract): model perl main package; cache-safe re-pointer; harden walk & labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model Perl's implicit `main` package explicitly (R1): a package-less sub now lives in a lazily-created `main` node instead of hanging off the file node, so a qualified `main::helper()` binds and a bare call from a package-less file is scoped to `main` — it can no longer be mis-bound to a same-named sub in an unrelated package without import evidence. Match the import re-pointer's provenance on resolved paths (R2): a cached fragment's `source_file` returns absolutized against the resolved root, so the exact-string membership test against the raw (relative) provenance paths missed and the re-pointer silently regressed to dangling on the second (cached) run. Compare on the resolved on-disk identity so fresh and cached runs re-point alike. Walk the AST iteratively (S1): `_string_parents`, `walk_statements` and `walk_calls` were recursive, so a crafted deep nest raised RecursionError and `_safe_extract` dropped the WHOLE file. Explicit stacks plus a coarse traversal budget keep the file node and partial graph instead of losing everything. Validate inheritance targets (S2): `@ISA` / `use parent` / `use base` content is raw string data; a value that is not a well-formed package name (control chars, markdown, over-long) is discarded rather than emitted as a node label into graph.json / the Obsidian export. Stamp `lang="perl"` on raw_calls (A1, extractor half) so the shared second pass can claim exactly Perl's calls by the extractor stamp, matching the cpp/csharp/java/objc consumers. Constraint: zero-edge over a guess — unresolvable/foreign bindings are dropped. --- graphify/extractors/perl.py | 358 ++++++++++++++++++++++++------------ tests/test_languages.py | 148 +++++++++++++++ 2 files changed, 387 insertions(+), 119 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 2a879c760..79c57dd60 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -8,11 +8,37 @@ """ from __future__ import annotations +import logging +import re from pathlib import Path from typing import Any from graphify.extractors.base import _file_stem, _make_id +_LOG = logging.getLogger(__name__) + +# A valid Perl package/class name: a bareword component (`Foo`, `_priv`) optionally +# joined by `::`. Inheritance targets come from arbitrary string_literal / qw() +# content (`@ISA`, `use parent`, `use base`), so a crafted or malformed string +# (control chars, markdown, newlines, an over-long blob) could otherwise flow raw +# into a node label and on into graph.json / the Obsidian export. Names that do +# not match are discarded (zero-edge over a bogus stub). +_PERL_PKG_NAME_RE = re.compile(r"^[A-Za-z_]\w*(?:::\w+)*$") +_MAX_PERL_PKG_NAME_LEN = 256 + +# Coarse guard so a pathologically large or deeply nested file cannot make the +# (now iterative) tree walks run away; on exhaustion the file keeps whatever was +# already extracted (file node + partial graph) instead of nothing. +_MAX_PERL_TRAVERSAL_NODES = 2_000_000 + + +def _is_valid_perl_package_name(name: str) -> bool: + return ( + bool(name) + and len(name) <= _MAX_PERL_PKG_NAME_LEN + and _PERL_PKG_NAME_RE.match(name) is not None + ) + # ``use strict`` & friends are compiler pragmas, not module dependencies — they # must not become imports edges (matches the pragma-exclusion in other langs). _PERL_PRAGMAS: frozenset[str] = frozenset({ @@ -135,26 +161,62 @@ def _string_parents(node) -> list[str]: are a different node type and are intentionally skipped. """ out: list[str] = [] - - def visit(n) -> None: + stack = [node] + while stack: + n = stack.pop() if n.type in ("string_literal", "interpolated_string_literal", "quoted_word_list"): for c in n.children: if c.type == "string_content": out.extend(_text(c).split()) - return - for c in n.children: - visit(c) - - visit(node) + continue # an inheritance string's own children are not parents + stack.extend(reversed(n.children)) return out def add_inherits(pkg_nid: str, parent_name: str, line: int) -> None: + # Inheritance targets are raw string content; discard anything that is not + # a well-formed package name so a malformed/crafted string never becomes a + # node label or an edge (zero-edge, matching the untyped-drop discipline). + if not _is_valid_perl_package_name(parent_name): + return parent_nid = _make_id(parent_name) add_stub_node(parent_nid, parent_name) add_edge(pkg_nid, parent_nid, "inherits", line, context="inherit") current_pkg_nid: str | None = None current_pkg_name: str | None = None + main_pkg_nid: str | None = None + + def _ensure_main_pkg() -> str: + """Perl's implicit default package. Code with no ``package`` statement lives + in ``main``; modeling it explicitly (instead of hanging package-less subs off + the file node) means a qualified ``main::helper()`` call binds, and a bare + call from a package-less file is scoped to ``main`` — so it cannot be + mis-bound to a same-named sub in an unrelated package without import + evidence. Created lazily so a file with no package-less subs gets no empty + ``main`` node.""" + nonlocal main_pkg_nid + if main_pkg_nid is None: + main_pkg_nid = _make_id(stem, "main") + add_node(main_pkg_nid, "main", 1) + add_edge(file_nid, main_pkg_nid, "contains", 1) + return main_pkg_nid + + budget = [_MAX_PERL_TRAVERSAL_NODES] + budget_warned = [False] + + def _spend() -> bool: + """Charge one node against the shared traversal budget; False once spent so + the walkers stop instead of running away. Emits one bounded warning.""" + budget[0] -= 1 + if budget[0] < 0: + if not budget_warned[0]: + budget_warned[0] = True + _LOG.warning( + "perl: traversal budget exhausted for %s; graph for this file is partial", + str_path, + ) + return False + return True def handle_use(node, line: int) -> None: # In a use_statement the `use` keyword is its own node type, so the @@ -194,121 +256,152 @@ def handle_isa(assign_node, line: int) -> None: for parent in _string_parents(assign_node): add_inherits(current_pkg_nid, parent, line) - def walk_statements(node) -> None: + def walk_statements(root_node) -> None: nonlocal current_pkg_nid, current_pkg_name - for child in node.children: - line = child.start_point[0] + 1 - if child.type == "package_statement": - name = _package_name(child) - if name: - pkg_nid = _make_id(stem, name) - add_node(pkg_nid, name, line) - add_edge(file_nid, pkg_nid, "contains", line) - pkg_block = next( - (c for c in child.children if c.type == "block"), None) - if pkg_block is not None: - # Block-form `package Foo { ... }` scopes Foo to the block - # only. Walk the block under Foo, then restore the prior - # package so following top-level statements are not - # mis-attributed to Foo (and the block's subs are not lost). - prev_nid, prev_name = current_pkg_nid, current_pkg_name - current_pkg_nid, current_pkg_name = pkg_nid, name - walk_statements(pkg_block) - current_pkg_nid, current_pkg_name = prev_nid, prev_name - else: - current_pkg_nid = pkg_nid - current_pkg_name = name - elif child.type == "use_statement": - handle_use(child, line) - elif child.type == "subroutine_declaration_statement": - name = None - block = None - for c in child.children: - if c.type == "bareword" and name is None: - name = _text(c) - elif c.type == "block": - block = c - if name: - if "::" in name: - # Qualified declaration `sub Pkg::sub {...}` defines the sub - # IN the named package, not the current one. Container = that - # package (created if it has no `package` statement of its - # own); the body's caller-package is the qualifier so its - # calls resolve against Pkg. - pkg_qual, _, sub_name = name.rpartition("::") - container = _make_id(stem, pkg_qual) - add_node(container, pkg_qual, line) - add_edge(file_nid, container, "contains", line) - sub_package = pkg_qual - else: - container = current_pkg_nid or file_nid - sub_name = name - sub_package = current_pkg_name - sub_nid = _make_id(container, sub_name) - add_node(sub_nid, f"{sub_name}()", line) - add_edge(container, sub_nid, "contains", line) - if block is not None: - sub_bodies.append((sub_nid, block, sub_package)) - elif child.type == "expression_statement": - for c in child.children: - if c.type == "require_expression": - handle_require(c, line) - elif c.type == "assignment_expression": - handle_isa(c, line) + # Manual call stack in place of recursion: a pathologically deep nest of + # block-form packages would otherwise blow the Python stack, and the + # resulting RecursionError makes `_safe_extract` drop the WHOLE file. Each + # frame is an iterator over one block's statements plus the scope to restore + # once that block is exhausted; a block-form `package Foo { ... }` pushes a + # child frame under Foo's scope, so its subs are attributed to Foo and the + # prior package is restored for statements that follow the block. + stack: list[tuple[Any, str | None, str | None]] = [ + (iter(root_node.children), current_pkg_nid, current_pkg_name) + ] + while stack: + if not _spend(): + break + child_iter, restore_nid, restore_name = stack[-1] + descended = False + for child in child_iter: + line = child.start_point[0] + 1 + if child.type == "package_statement": + name = _package_name(child) + if name: + pkg_nid = _make_id(stem, name) + add_node(pkg_nid, name, line) + add_edge(file_nid, pkg_nid, "contains", line) + pkg_block = next( + (c for c in child.children if c.type == "block"), None) + if pkg_block is not None: + # Descend into the block under Foo; the frame remembers + # the pre-block scope so it is restored when the block is + # fully consumed (statements after the block are not + # mis-attributed to Foo). + prev_nid, prev_name = current_pkg_nid, current_pkg_name + current_pkg_nid, current_pkg_name = pkg_nid, name + stack.append((iter(pkg_block.children), prev_nid, prev_name)) + descended = True + break + else: + current_pkg_nid = pkg_nid + current_pkg_name = name + elif child.type == "use_statement": + handle_use(child, line) + elif child.type == "subroutine_declaration_statement": + name = None + block = None + for c in child.children: + if c.type == "bareword" and name is None: + name = _text(c) + elif c.type == "block": + block = c + if name: + if "::" in name: + # Qualified declaration `sub Pkg::sub {...}` defines the + # sub IN the named package, not the current one. Container + # = that package (created if it has no `package` statement + # of its own); the body's caller-package is the qualifier + # so its calls resolve against Pkg. + pkg_qual, _, sub_name = name.rpartition("::") + container = _make_id(stem, pkg_qual) + add_node(container, pkg_qual, line) + add_edge(file_nid, container, "contains", line) + sub_package = pkg_qual + else: + # Package-less sub → Perl's `main` (not the file node), so + # `main::sub()` binds and bare same-file calls resolve. + container = current_pkg_nid or _ensure_main_pkg() + sub_name = name + sub_package = current_pkg_name or "main" + sub_nid = _make_id(container, sub_name) + add_node(sub_nid, f"{sub_name}()", line) + add_edge(container, sub_nid, "contains", line) + if block is not None: + sub_bodies.append((sub_nid, block, sub_package)) + elif child.type == "expression_statement": + for c in child.children: + if c.type == "require_expression": + handle_require(c, line) + elif c.type == "assignment_expression": + handle_isa(c, line) + if descended: + continue + stack.pop() + current_pkg_nid, current_pkg_name = restore_nid, restore_name walk_statements(root) raw_calls: list[dict] = [] - def walk_calls(node, caller_nid: str, caller_package: str | None) -> None: - if node.type == "function_call_expression": - # Indirect-object constructor `new CLASS(...)` parses as - # ambiguous_function_call_expression(function 'new', function_call_expression - # 'CLASS()'); `new CLASS` == CLASS->new, an untyped member dispatch. Mark - # it a member call (edge-less) so it is not wired to a sub named CLASS. - parent = node.parent - indirect_new = ( - parent is not None - and parent.type == "ambiguous_function_call_expression" - and bool(parent.children) - and parent.children[0].type == "function" - and _text(parent.children[0]) == "new" - ) - for c in node.children: - if c.type == "function": - # `Acme::Helper::emit` -> callee `emit`, callee_package - # `Acme::Helper`; a bare `emit` -> callee `emit`, no package. - # The qualifier + caller package let the shared second pass - # bind to the right same-named sub instead of any `emit()`. - parts = _text(c).split("::") - callee = parts[-1] - callee_package = "::".join(parts[:-1]) or None - if callee and callee not in _PERL_BUILTINS: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": callee, - "callee_package": callee_package, - "caller_package": caller_package, - "is_member_call": indirect_new, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - break - elif node.type == "method_call_expression": - for c in node.children: - if c.type == "method": - callee = _text(c).split("::")[-1] - if callee: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": callee, - "is_member_call": True, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - break - for child in node.children: - walk_calls(child, caller_nid, caller_package) + def walk_calls(root_node, caller_nid: str, caller_package: str | None) -> None: + # Iterative (see walk_statements): a deeply nested expression / data + # structure in a sub body would recurse once per level, and RecursionError + # here would drop the whole file via `_safe_extract`. + stack = [root_node] + while stack: + if not _spend(): + break + node = stack.pop() + if node.type == "function_call_expression": + # Indirect-object constructor `new CLASS(...)` parses as + # ambiguous_function_call_expression(function 'new', function_call_expression + # 'CLASS()'); `new CLASS` == CLASS->new, an untyped member dispatch. Mark + # it a member call (edge-less) so it is not wired to a sub named CLASS. + parent = node.parent + indirect_new = ( + parent is not None + and parent.type == "ambiguous_function_call_expression" + and bool(parent.children) + and parent.children[0].type == "function" + and _text(parent.children[0]) == "new" + ) + for c in node.children: + if c.type == "function": + # `Acme::Helper::emit` -> callee `emit`, callee_package + # `Acme::Helper`; a bare `emit` -> callee `emit`, no package. + # The qualifier + caller package let the shared second pass + # bind to the right same-named sub instead of any `emit()`. + parts = _text(c).split("::") + callee = parts[-1] + callee_package = "::".join(parts[:-1]) or None + if callee and callee not in _PERL_BUILTINS: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "callee_package": callee_package, + "caller_package": caller_package, + "is_member_call": indirect_new, + "lang": "perl", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + elif node.type == "method_call_expression": + for c in node.children: + if c.type == "method": + callee = _text(c).split("::")[-1] + if callee: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": True, + "lang": "perl", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + stack.extend(reversed(node.children)) for caller_nid, body, caller_package in sub_bodies: walk_calls(body, caller_nid, caller_package) @@ -349,10 +442,37 @@ def _resolve_perl_imports( Mutates ``all_edges`` in place; the bare target was never a node, so there is nothing to prune. """ + # Provenance matching must survive the cache round-trip. A fresh run stamps a + # node's `source_file` with the raw `str(path)` the extractor was handed, which + # is exactly what `perl_source_files` holds — so exact membership matches. But a + # cached fragment is stored relative and re-anchored on load against the RESOLVED + # cache root, so its `source_file` comes back as an absolute resolved path that + # no longer equals the raw provenance string — exact membership then misses and + # the re-pointer silently no-ops on the second (cached) run. Compare on the + # resolved form so both shapes collapse to the same on-disk identity. + resolved_perl_sources: set[str] | None = None + if perl_source_files is not None: + resolved_perl_sources = set(perl_source_files) + for s in perl_source_files: + try: + resolved_perl_sources.add(str(Path(s).resolve())) + except OSError: + pass + _resolve_memo: dict[str, bool] = {} + def _is_perl_source(src: str) -> bool: - if perl_source_files is not None: - return src in perl_source_files - return src.endswith((".pl", ".pm")) + if perl_source_files is None: + return src.endswith((".pl", ".pm")) + if src in resolved_perl_sources: + return True + hit = _resolve_memo.get(src) + if hit is None: + try: + hit = str(Path(src).resolve()) in resolved_perl_sources + except OSError: + hit = False + _resolve_memo[src] = hit + return hit # module-label id -> list of package node ids carrying that fully-qualified # label. A bare `use Foo` cannot disambiguate a package re-opened under the diff --git a/tests/test_languages.py b/tests/test_languages.py index 82f537abc..3ce4edd33 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3452,3 +3452,151 @@ def test_perl_import_repoints_from_shebang_perl_file(tmp_path): imports = [e for e in r["edges"] if e["relation"] == "imports"] assert any(e["target"] == pkg_id for e in imports), \ f"use Acme::Helper in a shebang-perl file must re-point to {pkg_id}, got {[e['target'] for e in imports]}" + + +# ── Perl S6c review-finding regressions ──────────────────────────────────────── + +def test_perl_raw_calls_stamped_lang_perl(): + """Every Perl raw_call carries `lang="perl"` so the shared second pass claims + exactly Perl's calls via the extractor stamp (like cpp/csharp/java/objc), + rather than sniffing for a `*_package` field (A1).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert r["raw_calls"], "expected raw_calls from the sample module" + assert all(rc.get("lang") == "perl" for rc in r["raw_calls"]), \ + f"every perl raw_call must be lang=perl, got {[rc.get('lang') for rc in r['raw_calls']]}" + + +def test_perl_main_qualified_call_binds_packageless_sub(tmp_path): + """A package-less sub lives in Perl's default `main` package; a qualified + `main::helper()` call must bind to it (R1). Before main was modeled the sub + hung off the file node with the filename as its package label, so `main::` + never resolved.""" + from graphify.extract import extract + (tmp_path / "s.pl").write_text( + "sub helper { return 1; }\nsub run { main::helper(); }\n" + ) + r = extract([tmp_path / "s.pl"], parallel=False) + calls = _calls(r) + assert any("run" in s and "helper" in t for s, t in calls), \ + f"main::helper() must bind to the package-less (main) helper, got {calls}" + + +def test_perl_packageless_bare_call_no_bind_to_foreign_package(tmp_path): + """A bare call from a package-less (main) file must NOT bind to a same-named + sub in an unrelated, un-imported package (R1). Modeling `main` scopes the call + to main; without it the call carried no package and was generically matched to + the lone same-named sub in a foreign package.""" + from graphify.extract import extract + (tmp_path / "b.pm").write_text("package B;\nsub helper { return 1; }\n1;\n") + (tmp_path / "s.pl").write_text( + "sub other { return 2; }\nsub run { helper(); other(); }\n" + ) + r = extract([tmp_path / "b.pm", tmp_path / "s.pl"], parallel=False) + calls = _calls(r) + # Positive guard: the same-file (main) bare call resolves, so the negative + # assertion below is not a vacuous pass on an empty graph. + assert any("run" in s and "other" in t for s, t in calls), \ + f"expected run->other within main, got {calls}" + assert not any("helper" in t for _s, t in calls), \ + f"bare helper() from a main file must not bind to B::helper, got {calls}" + + +def test_perl_main_package_node_emitted_for_packageless_sub(tmp_path): + """The lazily-created `main` package node contains the package-less sub (the + `contains` edge is main -> sub, not file -> sub).""" + from graphify.extract import extract_perl + src = tmp_path / "s.pl" + src.write_text("sub helper { return 1; }\n") + r = extract_perl(src) + label = {n["id"]: n.get("label", "") for n in r["nodes"]} + container_of = { + label.get(e["target"]): label.get(e["source"]) + for e in r["edges"] if e["relation"] == "contains" + } + assert "main" in label.values(), "a package-less sub must materialize a main package node" + assert container_of.get("helper()") == "main", \ + f"package-less helper must be contained by main, got {container_of.get('helper()')!r}" + + +def test_perl_import_repoint_survives_cache_roundtrip(tmp_path, monkeypatch): + """The import re-pointer must still fire on a cached (second) run (R2). A cached + fragment's source_file returns absolutized against the resolved root, so exact + string matching against the raw (relative) provenance paths missed and the + re-pointer silently regressed to dangling. Matching on the resolved path makes + both runs produce the same re-pointed edge.""" + from graphify.extract import extract + monkeypatch.chdir(tmp_path) + Path("helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + Path("main.pm").write_text( + "package Main;\nuse Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + paths = [Path("helper.pm"), Path("main.pm")] + + def _import_targets(r): + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + return pkg_id, {e["target"] for e in r["edges"] if e["relation"] == "imports"} + + r1 = extract(paths, cache_root=Path("."), parallel=False) + pkg1, targets1 = _import_targets(r1) + assert pkg1 in targets1, f"first run must re-point onto {pkg1}, got {targets1}" + assert (tmp_path / "graphify-out" / "cache").exists(), "AST cache must be written" + + r2 = extract(paths, cache_root=Path("."), parallel=False) + pkg2, targets2 = _import_targets(r2) + assert pkg2 in targets2, \ + f"cached run must ALSO re-point onto {pkg2} (not regress to dangling), got {targets2}" + assert targets1 == targets2, \ + f"import targets must be identical across fresh and cached runs: {targets1} vs {targets2}" + + +def test_perl_deeply_nested_expression_does_not_crash(tmp_path): + """A pathologically deep expression nest must not RecursionError (which would + make _safe_extract drop the whole file); the iterative walk keeps the file node + and the sub (S1).""" + from graphify.extract import extract_perl + depth = 12000 # > _RECURSION_LIMIT (10_000): the old recursive walk would blow up + body = "[" * depth + "1" + "]" * depth + src = tmp_path / "deep.pl" + src.write_text(f"sub f {{ my $x = {body}; }}\n") + r = extract_perl(src) + assert not r.get("error"), f"deep nest must not error out, got {r.get('error')!r}" + labels = {n["label"] for n in r["nodes"]} + assert "deep.pl" in labels, "file node must survive deep nesting" + assert "f()" in labels, "the sub must still be extracted despite deep nesting" + + +def test_perl_isa_rejects_non_package_string(tmp_path): + """@ISA holds arbitrary string content; a value that is not a well-formed + package name (markdown/brackets/control chars) must not become an inherits + stub node or edge (S2, zero-edge over a bogus label).""" + from graphify.extract import extract_perl + src = tmp_path / "bad.pm" + src.write_text( + 'package Child;\n' + 'our @ISA = ("weird\\nname[x](y)");\n' + '1;\n' + ) + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert not inherits, f"malformed @ISA string must not create an inherits edge, got {inherits}" + bad = [n for n in r["nodes"] + if any(ch in n.get("label", "") for ch in ("[", "(", "\n"))] + assert not bad, f"no stub node from a malformed inheritance string, got {bad}" + + +def test_perl_isa_valid_package_still_inherits(tmp_path): + """The S2 validation must not reject legitimate package names: a normal + `Foo::Bar` @ISA entry still produces an inherits edge + stub.""" + from graphify.extract import extract_perl + src = tmp_path / "ok.pm" + src.write_text( + 'package Child;\n' + 'our @ISA = ("Acme::Base");\n' + '1;\n' + ) + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert inherits, "a well-formed @ISA package must still inherit" + labels = {n.get("label") for n in r["nodes"]} + assert "Acme::Base" in labels, "the valid base class stub must be emitted" From d3222375f6640c73008f8f5a9cc33e7a4a096cf7 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:48:01 +0000 Subject: [PATCH 14/21] refactor(extract): register perl import re-pointer via the resolver registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the Perl `imports` re-pointer off a private-underscore import called inline in `extract()`'s body onto the documented extension point (A2): it is now a registered `LanguageResolver`, so a language plugs in without editing `extract()`. It stays the last registered pass — after the shared call pass (which reads the bare module-label `use` targets via `_has_package_import_evidence`) and the member-call resolvers — the same position as before. Extend the registry with two optional hooks the re-pointer needs (both default off, so the existing 3-arg resolvers are untouched): - `activate`: a predicate over `paths` that overrides the suffix gate, so an extensionless `#!/usr/bin/perl` script (no `.pl`/`.pm` suffix, dispatched by shebang) still activates the pass by extractor PROVENANCE; - `wants_paths`: passes `paths` to `resolve` so the provenance-scoped pass can recompute which files it owns. Constraint: no behavior change to the re-pointer itself — same activation, same ordering, same fault isolation (log-and-skip). --- graphify/extract.py | 43 +++++++++++++++++++-------- graphify/resolver_registry.py | 28 ++++++++++++++++-- tests/test_language_resolvers.py | 51 ++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 15 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 9860bfa99..9ab1bbe94 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2752,6 +2752,33 @@ def _key(label: str) -> str: ) +def _resolve_perl_imports_pass(per_file, all_nodes, all_edges, paths) -> None: + """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. + + Registered LAST so it runs after the shared cross-file call pass (which reads + the bare module-label ``use`` targets via ``_has_package_import_evidence``) and + after the member-call resolvers — the same position as its former inline call at + the tail of ``extract()``. Scoped by extractor PROVENANCE, not suffix: an + extensionless ``#!/usr/bin/perl`` script is dispatched to ``extract_perl`` by + shebang and must be re-pointed too, which is why it declares a custom + ``activate`` predicate (a shebang-only corpus has no ``.pl``/``.pm`` suffix) and + takes ``paths`` (``wants_paths``) to recompute that provenance set. + """ + perl_sources = {str(p) for p in paths if _get_extractor(p) is extract_perl} + _resolve_perl_imports(all_nodes, all_edges, perl_sources) + + +register_language_resolver( + LanguageResolver( + "perl_import_repoint", + frozenset({".pl", ".pm"}), + _resolve_perl_imports_pass, + activate=lambda paths: any(_get_extractor(p) is extract_perl for p in paths), + wants_paths=True, + ) +) + + # Inline markdown link: [text](target "optional title"). The negative lookbehind # excludes images (![alt](src)). The target stops at whitespace/closing paren so # an optional "title" after the URL is dropped; an optional <...> wrapper is too. @@ -4836,21 +4863,11 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # receiver-typed/qualified calls the shared pass skipped) with its own # single-definition god-node guard. Registered in graphify.resolver_registry so # a new language plugs in without editing this body (#1356 Swift, #1446 Python). + # The Perl import re-pointer (`perl_import_repoint`) is the last registered + # resolver, so it runs here — after the call pass and member-call resolvers, + # before the relativization below — the same position as its former inline call. run_language_resolvers(paths, per_file, all_nodes, all_edges) - # Re-point dangling in-corpus Perl `imports` edges onto the real package node. - # Runs AFTER the call pass above so `_has_package_import_evidence` still sees the - # bare module-label ids it needs to bind bare calls to imported packages (#S5b). - # Scope by extractor provenance, not suffix, so extensionless `#!/usr/bin/perl` - # scripts (dispatched to extract_perl by shebang) are re-pointed too. Keyed on - # str(path) to match the nodes' source_file, set before the relativization below. - try: - _perl_sources = {str(p) for p in paths if _get_extractor(p) is extract_perl} - _resolve_perl_imports(all_nodes, all_edges, _perl_sources) - except Exception as exc: - import logging - logging.getLogger(__name__).warning("Perl import resolution failed, skipping: %s", exc) - # Relativize source_file fields so paths are portable across machines (#555) for item in all_nodes + all_edges: sf = item.get("source_file") diff --git a/graphify/resolver_registry.py b/graphify/resolver_registry.py index b17478a78..4ca592ce3 100644 --- a/graphify/resolver_registry.py +++ b/graphify/resolver_registry.py @@ -33,11 +33,25 @@ class LanguageResolver: mutates ``all_nodes`` / ``all_edges`` in place, matching the existing member-call resolvers. ``suffixes`` gates activation: the pass runs only when the corpus contains at least one file with one of these extensions. + + Two optional hooks cover passes that suffix-gating cannot express (the Perl + import re-pointer needs both): + + - ``activate`` overrides the suffix gate with a predicate over ``paths``. Use it + when membership is decided by EXTRACTOR provenance rather than extension — + e.g. an extensionless ``#!/usr/bin/perl`` script has no ``.pl``/``.pm`` suffix + to gate on, yet must still activate the pass. When ``None`` the suffix gate is + used. + - ``wants_paths`` makes the driver call ``resolve`` with a fourth positional + argument, the ``paths`` sequence, so a provenance-scoped pass can recompute + which files it owns. Default ``False`` keeps the three-argument contract. """ name: str suffixes: frozenset resolve: Callable + activate: Callable | None = None + wants_paths: bool = False # Module-level registry, populated by callers via register(). Ordered: resolvers @@ -77,9 +91,19 @@ def run_language_resolvers( active = _REGISTRY if resolvers is None else resolvers suffixes_present = {p.suffix for p in paths} for resolver in active: - if not (resolver.suffixes & suffixes_present): + if resolver.activate is not None: + try: + if not resolver.activate(paths): + continue + except Exception as exc: + _LOG.warning("%s activation check failed, skipping: %s", resolver.name, exc) + continue + elif not (resolver.suffixes & suffixes_present): continue try: - resolver.resolve(per_file, all_nodes, all_edges) + if resolver.wants_paths: + resolver.resolve(per_file, all_nodes, all_edges, paths) + else: + resolver.resolve(per_file, all_nodes, all_edges) except Exception as exc: _LOG.warning("%s resolution failed, skipping: %s", resolver.name, exc) diff --git a/tests/test_language_resolvers.py b/tests/test_language_resolvers.py index 787c1d505..bff014576 100644 --- a/tests/test_language_resolvers.py +++ b/tests/test_language_resolvers.py @@ -72,3 +72,54 @@ def _add_edge(per_file, all_nodes, all_edges): edges: list[dict] = [] run_language_resolvers([Path("a.rb")], [], [], edges, resolvers=resolvers) assert edges == [{"source": "x", "target": "y", "relation": "calls"}] + + +def test_resolver_activate_predicate_overrides_suffix_gate() -> None: + # Provenance-gated pass: no matching suffix present, but the activate predicate + # opts it in anyway (e.g. an extensionless shebang script). + log: list[str] = [] + + def _resolve(per_file, all_nodes, all_edges): + log.append("ran") + + resolver = LanguageResolver( + "prov", frozenset({".rb"}), _resolve, + activate=lambda paths: any(p.name == "runme" for p in paths), + ) + run_language_resolvers([Path("runme")], [], [], [], resolvers=[resolver]) + assert log == ["ran"] + + +def test_resolver_activate_false_skips_even_with_matching_suffix() -> None: + log: list[str] = [] + + def _resolve(per_file, all_nodes, all_edges): + log.append("ran") + + resolver = LanguageResolver( + "prov", frozenset({".rb"}), _resolve, activate=lambda paths: False, + ) + run_language_resolvers([Path("a.rb")], [], [], [], resolvers=[resolver]) + assert log == [] # activate wins over the suffix gate + + +def test_resolver_wants_paths_receives_paths() -> None: + seen: list = [] + + def _resolve(per_file, all_nodes, all_edges, paths): + seen.append(list(paths)) + + resolver = LanguageResolver("wp", frozenset({".rb"}), _resolve, wants_paths=True) + run_language_resolvers([Path("a.rb")], [], [], [], resolvers=[resolver]) + assert seen == [[Path("a.rb")]] + + +def test_default_registry_contains_perl_import_repoint() -> None: + import graphify.extract # noqa: F401 (registers resolvers on import) + + perl = next( + (r for r in registered_resolvers() if r.name == "perl_import_repoint"), None + ) + assert perl is not None, "perl_import_repoint must be registered in the shared registry" + assert perl.wants_paths, "the re-pointer needs paths to recompute Perl provenance" + assert perl.activate is not None, "the re-pointer activates by extractor provenance, not suffix" From 325c5db86d0d649ea9f21315fa351802d1c4e839 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:48:15 +0000 Subject: [PATCH 15/21] fix(extract): gate perl second pass on lang stamp; accept both import-evidence forms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate the package-aware second-pass branch on the extractor-stamped `lang == "perl"` (A1, resolver half) instead of sniffing for a `*_package` field, matching the cpp/csharp/java/objc raw-call consumers — so the branch claims exactly Perl's calls and a future language that set a package field could not fall into it. Make `_has_package_import_evidence` accept both the bare module-label id and the real package-node id as import evidence (A3). The re-pointer rewrites `use` targets from the bare id onto the package node id; matching either shape means this check no longer depends on whether the re-pointer has run, so the pass ordering stops being semantically load-bearing. The current order (re-pointer after this pass) is kept regardless. Constraint: additive + zero-edge — no other language sets `lang="perl"`, so their candidate sets are untouched. --- graphify/extract.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 9ab1bbe94..e47f7b0ad 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4646,9 +4646,11 @@ def extract( # this map is built but never read for them. _label_by_nid = {n["id"]: n.get("label", "") for n in all_nodes} pkg_label_by_nid: dict[str, str] = {} + pkg_nid_by_sub_nid: dict[str, str] = {} for e in all_edges: if e.get("relation") == "contains": pkg_label_by_nid[e["target"]] = _label_by_nid.get(e["source"], "") + pkg_nid_by_sub_nid[e["target"]] = e["source"] for rc in all_raw_calls: callee = rc.get("callee", "") @@ -4723,17 +4725,29 @@ def _has_package_import_evidence(candidate_id: str) -> bool: # Bridge it: the candidate sub's enclosing package, re-idized the same # way the `use` target is, must be among the caller file's imports. pkg = pkg_label_by_nid.get(candidate_id, "") - return bool(pkg) and _make_id(pkg) in imported_symbols - - # Package-aware pre-filter for languages that tag calls with their - # package (Perl). Additive + guarded: a raw_call only reaches this branch - # when it carries a package field, which no other language sets, so every - # other language's candidate set is untouched. Zero-edge over a wrong - # guess: an unresolvable qualifier or a foreign-package bare call is - # dropped, not bound to a same-named sub in the wrong package. + if not pkg: + return False + if _make_id(pkg) in imported_symbols: + return True + # Accept the real package-node id form too. The import re-pointer rewrites + # a `use` target from the bare module-label id onto the package node id; + # matching either shape means this evidence check no longer depends on + # whether that re-pointer has run yet — the pass ordering stops being + # semantically load-bearing (A3). The current order (re-pointer after this + # pass) is kept regardless. + pkg_nid = pkg_nid_by_sub_nid.get(candidate_id) + return pkg_nid is not None and pkg_nid in imported_symbols + + # Package-aware pre-filter for Perl, which tags every call with its + # enclosing package. Gated on the extractor-stamped `lang` (matching the + # cpp/csharp/java/objc raw-call consumers) rather than field-presence, so + # the branch claims exactly Perl's raw_calls and another language that + # happened to set a `*_package` field could never fall into it. Zero-edge + # over a wrong guess: an unresolvable qualifier or a foreign-package bare + # call is dropped, not bound to a same-named sub in the wrong package. callee_package = rc.get("callee_package") caller_package = rc.get("caller_package") - if callee_package is not None or caller_package is not None: + if rc.get("lang") == "perl": if callee_package is not None: # Qualified call `Pkg::sub()`: bind only to a sub whose enclosing # package matches the qualifier. None or several -> drop. From 432b152ba531b7e2ba90f997ff4c9e17b29073da Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:25:35 +0000 Subject: [PATCH 16/21] fix(extract): ASCII-validate perl package labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python `\w` is Unicode-default, so accented (`Basé`), fullwidth (`Base`) and digit-start (`Acme::1x`) names slipped the package-name validator and flowed into node labels, graph.json and the Obsidian export. Spell the classes out as explicit ASCII ranges, anchor every `::` component to a letter/underscore start, and switch to re.fullmatch so a trailing newline can no longer pass on a bare `$` anchor. 256-char cap unchanged. Constraint: fixes must not change real-corpus numbers (Foswiki core/lib: base and branch both 2776 nodes / 4611 edges / 211 inherits — Foswiki is ASCII, so no legitimate name is newly rejected). --- graphify/extractors/perl.py | 11 +++++++++-- tests/test_languages.py | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 79c57dd60..91c257237 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -23,7 +23,14 @@ # (control chars, markdown, newlines, an over-long blob) could otherwise flow raw # into a node label and on into graph.json / the Obsidian export. Names that do # not match are discarded (zero-edge over a bogus stub). -_PERL_PKG_NAME_RE = re.compile(r"^[A-Za-z_]\w*(?:::\w+)*$") +# +# The classes are spelled out as explicit ASCII ranges (not ``\w``): Python's +# ``\w`` is Unicode-default, so accented (``Basé``), fullwidth (``Base``) and +# other non-ASCII barewords would slip through. Every ``::`` component must begin +# with a letter/underscore, which also rejects a digit-start component +# (``Acme::1x``); ``fullmatch`` (below) anchors the whole string, so a trailing +# newline cannot pass on a ``$`` alone. +_PERL_PKG_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)*") _MAX_PERL_PKG_NAME_LEN = 256 # Coarse guard so a pathologically large or deeply nested file cannot make the @@ -36,7 +43,7 @@ def _is_valid_perl_package_name(name: str) -> bool: return ( bool(name) and len(name) <= _MAX_PERL_PKG_NAME_LEN - and _PERL_PKG_NAME_RE.match(name) is not None + and _PERL_PKG_NAME_RE.fullmatch(name) is not None ) # ``use strict`` & friends are compiler pragmas, not module dependencies — they diff --git a/tests/test_languages.py b/tests/test_languages.py index 3ce4edd33..3137ee576 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3600,3 +3600,41 @@ def test_perl_isa_valid_package_still_inherits(tmp_path): assert inherits, "a well-formed @ISA package must still inherit" labels = {n.get("label") for n in r["nodes"]} assert "Acme::Base" in labels, "the valid base class stub must be emitted" + + +@pytest.mark.parametrize("bad", [ + "Acme::Basé", # accented letter — Unicode \w matched, ASCII must not + "Acme::Base", # fullwidth latin letter — Unicode \w matched, ASCII must not + "Acme::1x", # component starting with a digit after :: + "1Acme", # leading digit + "Acme::Bar\n", # trailing newline (a bare $ anchor would let this through) +]) +def test_perl_label_validator_rejects_non_ascii_component(bad): + """The package-name validator is ASCII-only and anchors every `::` component to + a letter/underscore start (F3). Python `\\w` is Unicode-default, so accented, + fullwidth and digit-start names would otherwise pass and land in graph.json / + the Obsidian export.""" + from graphify.extractors.perl import _is_valid_perl_package_name + assert not _is_valid_perl_package_name(bad), \ + f"{bad!r} must be rejected by the ASCII package-name validator" + + +@pytest.mark.parametrize("ok", ["Foo", "_priv", "Acme::Base", "A::B::C", "F0o::B4r"]) +def test_perl_label_validator_accepts_ascii_package(ok): + """Legitimate ASCII package names (including digits after the first char of a + component) still validate — the F3 tightening must not over-reject.""" + from graphify.extractors.perl import _is_valid_perl_package_name + assert _is_valid_perl_package_name(ok), f"{ok!r} must validate" + + +def test_perl_isa_digit_start_component_no_stub(tmp_path): + """End-to-end: a @ISA entry whose ::-component starts with a digit must not + produce an inherits edge or a stub node (F3 Unicode/digit-start bypass).""" + from graphify.extract import extract_perl + src = tmp_path / "bad.pm" + src.write_text('package Child;\nour @ISA = ("Acme::1Base");\n1;\n') + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert not inherits, f"digit-start component must not inherit, got {inherits}" + assert "Acme::1Base" not in {n.get("label") for n in r["nodes"]}, \ + "no stub node for a digit-start component" From 6b26ac58f937e6ff6aef976ebd39d79430d3ba80 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:26:23 +0000 Subject: [PATCH 17/21] fix(extract): budget every node of the perl tree walks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The traversal budget was charged once per stack frame in walk_statements, then an unbounded number of sibling children were drained for free — a broad, flat file bypassed the guard entirely. _string_parents (the inheritance-string walk) was not budgeted at all. Charge _spend() per visited sibling in walk_statements (stop cleanly on exhaustion, keeping the partial graph) and per popped node in _string_parents, so all three walkers share one bounded budget as intended. walk_calls already charged per popped node — left as is. --- graphify/extractors/perl.py | 9 ++++-- tests/test_languages.py | 55 +++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py index 91c257237..e86c89285 100644 --- a/graphify/extractors/perl.py +++ b/graphify/extractors/perl.py @@ -170,6 +170,8 @@ def _string_parents(node) -> list[str]: out: list[str] = [] stack = [node] while stack: + if not _spend(): + break n = stack.pop() if n.type in ("string_literal", "interpolated_string_literal", "quoted_word_list"): for c in n.children: @@ -276,11 +278,14 @@ def walk_statements(root_node) -> None: (iter(root_node.children), current_pkg_nid, current_pkg_name) ] while stack: - if not _spend(): - break child_iter, restore_nid, restore_name = stack[-1] descended = False for child in child_iter: + # Charge every visited sibling, not once per frame: a broad flat + # file drains an unbounded number of children under a single frame, + # so a per-frame charge left them effectively free. + if not _spend(): + return # budget exhausted: keep the partial graph, stop walking line = child.start_point[0] + 1 if child.type == "package_statement": name = _package_name(child) diff --git a/tests/test_languages.py b/tests/test_languages.py index 3137ee576..4e89b429c 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3638,3 +3638,58 @@ def test_perl_isa_digit_start_component_no_stub(tmp_path): assert not inherits, f"digit-start component must not inherit, got {inherits}" assert "Acme::1Base" not in {n.get("label") for n in r["nodes"]}, \ "no stub node for a digit-start component" + + +def test_perl_statement_walk_charges_budget_per_sibling(tmp_path, monkeypatch): + """A broad, flat file (many sibling statements under one frame) must charge the + traversal budget per sibling, not once per stack frame (F2). Under a tiny budget + the walk stops early and emits a PARTIAL graph; the old code drained every + sibling on one charge, so all subs surfaced regardless of budget. Asserting the + partial output isolates walk_statements (the shared budget is also spent by the + later call walk, so a warning alone would not discriminate).""" + from graphify.extractors import perl + monkeypatch.setattr(perl, "_MAX_PERL_TRAVERSAL_NODES", 3) + src = tmp_path / "flat.pl" + src.write_text("package Wide;\n" + "".join( + f"sub s{i} {{ 1; }}\n" for i in range(30))) + r = perl.extract_perl(src) + sub_nodes = [n for n in r["nodes"] if n.get("label", "").endswith("()")] + assert len(sub_nodes) < 30, ( + "walk_statements must charge per sibling and stop early under a tiny budget; " + f"emitting all {len(sub_nodes)} subs means siblings were traversed for free") + + +def test_perl_statement_walk_no_warning_within_budget(tmp_path, caplog): + """Guard against a vacuous pass: a small file under the default budget extracts + fully and emits no traversal warning.""" + import logging + from graphify.extractors import perl + src = tmp_path / "small.pl" + src.write_text("package Wide;\n" + "".join( + f"sub s{i} {{ return {i}; }}\n" for i in range(20))) + with caplog.at_level(logging.WARNING, logger="graphify.extractors.perl"): + r = perl.extract_perl(src) + assert not any("traversal budget" in rec.message for rec in caplog.records), \ + "a small file must not trip the budget" + assert sum(1 for n in r["nodes"] if n.get("label", "").endswith("()")) == 20, \ + "all 20 subs extract when the budget is ample" + + +def test_perl_string_parents_charges_budget(tmp_path, monkeypatch, caplog): + """`_string_parents` shares the traversal budget (F2): an @ISA array of many + separate string literals is a per-node walk that must trip the bounded warning, + even though the file has only a couple of top-level statements. The old code + left `_string_parents` entirely unbudgeted, so it walked the whole subtree for + free. (Each literal is its own tree node — unlike a single qw() word list.)""" + import logging + from graphify.extractors import perl + # Above the ~2 top-level statement charges but far below the string-literal node + # count, so exhaustion can only come from _string_parents. + monkeypatch.setattr(perl, "_MAX_PERL_TRAVERSAL_NODES", 20) + parents = ", ".join(f"'P{i}'" for i in range(400)) + src = tmp_path / "wide_isa.pm" + src.write_text(f"package Child;\nour @ISA = ({parents});\n1;\n") + with caplog.at_level(logging.WARNING, logger="graphify.extractors.perl"): + perl.extract_perl(src) + assert any("traversal budget exhausted" in rec.message for rec in caplog.records), \ + "a wide @ISA literal array must charge the shared budget via _string_parents" From ef4626a95158778a15408c3517da66fd616e415e Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:26:39 +0000 Subject: [PATCH 18/21] fix(cache): bump AST cache schema so stale-shape entries invalidate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AST cache namespaced only on package version (cache/ast/v{version}/), so an extractor OUTPUT-SHAPE change that ships within a release (no version bump) kept serving pre-change entries on a cache hit — e.g. Perl fragments written before the per-call `lang` stamp and lazy `main` package node. Add an integer _AST_SCHEMA_VERSION riding alongside the version in the dir name (v{version}-s{schema}/); bump it when the emitted shape changes without a release. Follows the existing integer schema-version pattern (reflect.py, diagnostics.py); the existing sibling-sweep cleanup removes the old namespace. Verified on Foswiki core/lib: a poisoned base-commit cache under v0.9.12/ is not reused after the bump — fresh extract into v0.9.12-s1/, identical numbers (2776 nodes / 4611 edges), zero sentinel nodes in the output. --- graphify/cache.py | 26 +++++++++++++++------ tests/test_cache.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/graphify/cache.py b/graphify/cache.py index 31c945a25..3261ef6f9 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -18,8 +18,9 @@ # AST cache entries are the output of graphify's own extractor code, so they # are only valid for the version that wrote them: keying purely on file # content means extractor fixes shipped in a new release keep serving stale -# pre-fix results. The AST cache is therefore namespaced by package version -# (cache/ast/v{version}/), with entries from other versions removed on first +# pre-fix results. The AST cache is therefore namespaced by package version AND an +# output-shape schema counter (cache/ast/v{version}-s{schema}/, see +# _AST_SCHEMA_VERSION below), with entries from other namespaces removed on first # use. The semantic cache is deliberately NOT versioned — its entries are # produced by the LLM from file contents, and invalidating them on every # release would re-bill extraction for unchanged files. @@ -30,6 +31,17 @@ except Exception: _EXTRACTOR_VERSION = "unknown" +# Package version alone under-invalidates: an extractor OUTPUT-SHAPE change (a new +# per-node/edge/raw_calls field, a differently-shaped node) can ship WITHIN a +# release without a version bump, and a version-only namespace would keep serving +# the pre-change entries. This integer rides alongside the version in the AST cache +# dir name (``v{version}-s{schema}``); bump it whenever the AST extractor's emitted +# shape changes without a release, so old entries relocate out of the served +# namespace and are swept. Follows the integer schema-version pattern used by +# reflect.py (_LEARNING_SCHEMA_VERSION) and diagnostics.py. +# schema 1: per-call ``lang`` stamp + lazy ``main`` package node (Perl S6c). +_AST_SCHEMA_VERSION = 1 + # Version dirs already swept this process — cleanup runs once per (base, version). _cleaned_ast_dirs: set[str] = set() @@ -337,16 +349,16 @@ def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path: kind is "ast" or "semantic". Separate subdirectories prevent semantic cache entries from overwriting AST cache entries for the same source_file (#582). - AST entries live in graphify-out/cache/ast/v{version}/ — namespaced by - graphify version because they depend on extractor code, not just file - contents. Semantic entries live unversioned in graphify-out/cache/semantic/ - (re-extraction costs LLM calls). + AST entries live in graphify-out/cache/ast/v{version}-s{schema}/ — namespaced + by graphify version AND output-shape schema (see _AST_SCHEMA_VERSION) because + they depend on extractor code, not just file contents. Semantic entries live + unversioned in graphify-out/cache/semantic/ (re-extraction costs LLM calls). """ _out = Path(_GRAPHIFY_OUT) base = _out if _out.is_absolute() else Path(root).resolve() / _out d = base / "cache" / kind if kind == "ast": - d = d / f"v{_EXTRACTOR_VERSION}" + d = d / f"v{_EXTRACTOR_VERSION}-s{_AST_SCHEMA_VERSION}" _cleanup_stale_ast_entries(d.parent, d) d.mkdir(parents=True, exist_ok=True) return d diff --git a/tests/test_cache.py b/tests/test_cache.py index 9bf2cdc82..ec981d7d9 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -540,3 +540,58 @@ def test_save_semantic_cache_merge_existing_unions(tmp_path): ids = {n["id"] for n in cached["nodes"]} assert ids == {"a", "b"}, "merge_existing must union both chunk slices" assert len(cached["edges"]) == 1 + + +def test_ast_cache_invalidated_on_schema_bump(tmp_path, monkeypatch): + """AST output SHAPE can change within a release (no version bump) — e.g. a new + per-call `lang` stamp or a lazily-materialized package node. Bumping + _AST_SCHEMA_VERSION must invalidate entries written under the prior schema even + when _EXTRACTOR_VERSION is unchanged (F1).""" + import graphify.cache as cache_mod + + f = tmp_path / "mod.pl" + f.write_text("package Foo;\nsub run { helper(); }\n") + + monkeypatch.setattr(cache_mod, "_AST_SCHEMA_VERSION", 1, raising=False) + save_cached(f, {"nodes": [{"id": "n1"}], "edges": [], "raw_calls": []}, + root=tmp_path, kind="ast") + assert load_cached(f, root=tmp_path, kind="ast") is not None + + monkeypatch.setattr(cache_mod, "_AST_SCHEMA_VERSION", 2, raising=False) + assert load_cached(f, root=tmp_path, kind="ast") is None, ( + "an AST entry from a prior schema version must not be served after a bump" + ) + + +def test_ast_schema_version_in_cache_dir(tmp_path): + """The AST cache directory carries the schema counter alongside the package + version, so a shape change relocates the namespace (F1).""" + import graphify.cache as cache_mod + + d = cache_dir(tmp_path, "ast") + assert d.name == f"v{cache_mod._EXTRACTOR_VERSION}-s{cache_mod._AST_SCHEMA_VERSION}", ( + f"AST cache dir must namespace by version AND schema, got {d.name!r}" + ) + + +def test_pre_schema_ast_entries_not_served(tmp_path): + """An entry written by a base-commit graphify (version-only namespace + `v{version}/`, before the schema counter existed) is by definition from an + older extractor SHAPE and must not be served — the invalidation the schema + bump provides for changes that ship without a version bump (F1).""" + import json + from graphify.cache import file_hash, _GRAPHIFY_OUT, _EXTRACTOR_VERSION + + f = tmp_path / "mod.pl" + f.write_text("package Foo;\nsub run { helper(); }\n") + h = file_hash(f, tmp_path) + + # pre-schema layout: cache/ast/v{version}/{hash}.json (no `-s{schema}` suffix) + old_dir = tmp_path / _GRAPHIFY_OUT / "cache" / "ast" / f"v{_EXTRACTOR_VERSION}" + old_dir.mkdir(parents=True) + (old_dir / f"{h}.json").write_text( + json.dumps({"nodes": [{"id": "stale"}], "edges": [], "raw_calls": []})) + + assert load_cached(f, root=tmp_path, kind="ast") is None, ( + "a version-only (pre-schema) AST entry must not be served" + ) From e4b3df6563e5ae49fab34f717d6d42a0a564347e Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:26:50 +0000 Subject: [PATCH 19/21] docs(resolver): generalize the LanguageResolver hook docstring The shared registry API contract named the Perl import re-pointer and a `#!/usr/bin/perl` shebang as the motivating case for the activate/wants_paths hooks. This module deliberately knows nothing about any specific language; describe the hooks generically (provenance-scoped passes gated by which extractor produced a node, not by extension). The Perl-specific rationale already lives at its registration site in extract.py. --- graphify/resolver_registry.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/graphify/resolver_registry.py b/graphify/resolver_registry.py index 4ca592ce3..cb480c5d5 100644 --- a/graphify/resolver_registry.py +++ b/graphify/resolver_registry.py @@ -34,14 +34,15 @@ class LanguageResolver: member-call resolvers. ``suffixes`` gates activation: the pass runs only when the corpus contains at least one file with one of these extensions. - Two optional hooks cover passes that suffix-gating cannot express (the Perl - import re-pointer needs both): + Two optional hooks cover passes that suffix-gating cannot express — a + provenance-scoped pass (one whose file membership is decided by which + extractor produced a node, not by extension) may need both: - ``activate`` overrides the suffix gate with a predicate over ``paths``. Use it - when membership is decided by EXTRACTOR provenance rather than extension — - e.g. an extensionless ``#!/usr/bin/perl`` script has no ``.pl``/``.pm`` suffix - to gate on, yet must still activate the pass. When ``None`` the suffix gate is - used. + when membership is decided by EXTRACTOR provenance rather than extension — a + file dispatched to an extractor by content (e.g. a shebang) may carry no + matching suffix to gate on, yet must still activate the pass. When ``None`` + the suffix gate is used. - ``wants_paths`` makes the driver call ``resolve`` with a fourth positional argument, the ``paths`` sequence, so a provenance-scoped pass can recompute which files it owns. Default ``False`` keeps the three-argument contract. From 593072098361733f534c0520dddb7e32c4dbfee1 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:34:39 +0000 Subject: [PATCH 20/21] docs: add perl to language table and changelog --- CHANGELOG.md | 2 ++ README.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 057e32f3b..21309c586 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: a Java field/parameter/return-type reference to a class whose simple name is shared by two modules no longer dangles on a sourceless phantom node (#1744, thanks @aviciot). Both same-named classes already survive as distinct path-scoped nodes, but the cross-module `references` edge was left pointing at a bare no-source stub because `_resolve_java_type_references` re-pointed `implements`/`inherits`/`imports` but not `references` — so a query about the referenced class could miss it. The Java resolver now disambiguates `references` by the importing file's `import` statement (falling back to same-package), mirroring the C# resolver, and drops the orphaned phantom. +- Add: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#PR). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. + ## 0.9.11 (2026-07-08) - Fix: file enumeration no longer silently drops a directory subtree. `detect()`'s `os.walk` had no `onerror` handler, so an `os.scandir` failure (a permission error, or a directory created/deleted mid-walk by concurrent writes) was swallowed and that whole subtree vanished from the scan with no log, yielding a silently partial `graph.json`. The walk now records every skipped directory (surfaced in the result's `walk_errors`) and warns to stderr, while still enumerating the rest. Relatedly, `to_json`'s anti-shrink guard (#479) now fails safe: a non-empty but unreadable existing `graph.json` refuses the overwrite (pass `force=True` to override) instead of silently clobbering a good graph; an empty file still proceeds. diff --git a/README.md b/README.md index bd1dff423..48b39932c 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (36 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | +| Code (37 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .pl .pm .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | From f8c504475183421242f1684c6aeb1533d460af79 Mon Sep 17 00:00:00 2001 From: ad-astra-bot <263242209+ad-astra-bot@users.noreply.github.com> Date: Sat, 11 Jul 2026 10:19:56 +0000 Subject: [PATCH 21/21] docs: fill in PR number in changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21309c586..b8184339d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: a Java field/parameter/return-type reference to a class whose simple name is shared by two modules no longer dangles on a sourceless phantom node (#1744, thanks @aviciot). Both same-named classes already survive as distinct path-scoped nodes, but the cross-module `references` edge was left pointing at a bare no-source stub because `_resolve_java_type_references` re-pointed `implements`/`inherits`/`imports` but not `references` — so a query about the referenced class could miss it. The Java resolver now disambiguates `references` by the importing file's `import` statement (falling back to same-package), mirroring the C# resolver, and drops the orphaned phantom. -- Add: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#PR). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. +- Add: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#1788). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. ## 0.9.11 (2026-07-08)