diff --git a/CHANGELOG.md b/CHANGELOG.md index 057e32f3b..b8184339d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: a Java field/parameter/return-type reference to a class whose simple name is shared by two modules no longer dangles on a sourceless phantom node (#1744, thanks @aviciot). Both same-named classes already survive as distinct path-scoped nodes, but the cross-module `references` edge was left pointing at a bare no-source stub because `_resolve_java_type_references` re-pointed `implements`/`inherits`/`imports` but not `references` — so a query about the referenced class could miss it. The Java resolver now disambiguates `references` by the importing file's `import` statement (falling back to same-package), mirroring the C# resolver, and drops the orphaned phantom. +- Add: Perl support — `.pl`/`.pm` files (and `#!/usr/bin/perl` shebang scripts) extracted via tree-sitter-perl (#1788). Packages (including block form `package Foo { ... }` and mid-file package switches), subs, `use`/`require` imports with in-corpus re-pointing, and `@ISA`/`use parent`/`use base` inheritance are captured; calls resolve INFERRED through the shared package-aware second pass under a zero-edge ambiguity policy (a bare or same-named call that could bind to two packages emits no edge rather than a guess, mirroring the other cross-file resolvers' god-node guard). Method calls (`$obj->meth()`) and AUTOLOAD/symbolic refs are intentionally out of scope — without receiver types they are unresolvable and name-matching would wire spurious edges to same-named subs. + ## 0.9.11 (2026-07-08) - Fix: file enumeration no longer silently drops a directory subtree. `detect()`'s `os.walk` had no `onerror` handler, so an `os.scandir` failure (a permission error, or a directory created/deleted mid-walk by concurrent writes) was swallowed and that whole subtree vanished from the scan with no log, yielding a silently partial `graph.json`. The walk now records every skipped directory (surfaced in the result's `walk_errors`) and warns to stderr, while still enumerating the rest. Relatedly, `to_json`'s anti-shrink guard (#479) now fails safe: a non-empty but unreadable existing `graph.json` refuses the overwrite (pass `force=True` to override) instead of silently clobbering a good graph; an empty file still proceeds. diff --git a/README.md b/README.md index bd1dff423..48b39932c 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (36 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | +| Code (37 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .pl .pm .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | diff --git a/graphify/cache.py b/graphify/cache.py index 31c945a25..3261ef6f9 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -18,8 +18,9 @@ # AST cache entries are the output of graphify's own extractor code, so they # are only valid for the version that wrote them: keying purely on file # content means extractor fixes shipped in a new release keep serving stale -# pre-fix results. The AST cache is therefore namespaced by package version -# (cache/ast/v{version}/), with entries from other versions removed on first +# pre-fix results. The AST cache is therefore namespaced by package version AND an +# output-shape schema counter (cache/ast/v{version}-s{schema}/, see +# _AST_SCHEMA_VERSION below), with entries from other namespaces removed on first # use. The semantic cache is deliberately NOT versioned — its entries are # produced by the LLM from file contents, and invalidating them on every # release would re-bill extraction for unchanged files. @@ -30,6 +31,17 @@ except Exception: _EXTRACTOR_VERSION = "unknown" +# Package version alone under-invalidates: an extractor OUTPUT-SHAPE change (a new +# per-node/edge/raw_calls field, a differently-shaped node) can ship WITHIN a +# release without a version bump, and a version-only namespace would keep serving +# the pre-change entries. This integer rides alongside the version in the AST cache +# dir name (``v{version}-s{schema}``); bump it whenever the AST extractor's emitted +# shape changes without a release, so old entries relocate out of the served +# namespace and are swept. Follows the integer schema-version pattern used by +# reflect.py (_LEARNING_SCHEMA_VERSION) and diagnostics.py. +# schema 1: per-call ``lang`` stamp + lazy ``main`` package node (Perl S6c). +_AST_SCHEMA_VERSION = 1 + # Version dirs already swept this process — cleanup runs once per (base, version). _cleaned_ast_dirs: set[str] = set() @@ -337,16 +349,16 @@ def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path: kind is "ast" or "semantic". Separate subdirectories prevent semantic cache entries from overwriting AST cache entries for the same source_file (#582). - AST entries live in graphify-out/cache/ast/v{version}/ — namespaced by - graphify version because they depend on extractor code, not just file - contents. Semantic entries live unversioned in graphify-out/cache/semantic/ - (re-extraction costs LLM calls). + AST entries live in graphify-out/cache/ast/v{version}-s{schema}/ — namespaced + by graphify version AND output-shape schema (see _AST_SCHEMA_VERSION) because + they depend on extractor code, not just file contents. Semantic entries live + unversioned in graphify-out/cache/semantic/ (re-extraction costs LLM calls). """ _out = Path(_GRAPHIFY_OUT) base = _out if _out.is_absolute() else Path(root).resolve() / _out d = base / "cache" / kind if kind == "ast": - d = d / f"v{_EXTRACTOR_VERSION}" + d = d / f"v{_EXTRACTOR_VERSION}-s{_AST_SCHEMA_VERSION}" _cleanup_stale_ast_entries(d.parent, d) d.mkdir(parents=True, exist_ok=True) return d diff --git a/graphify/detect.py b/graphify/detect.py index 8c0f64a8f..2b6dd02c7 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -27,7 +27,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.pl', '.pm', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index dbef91ea0..e47f7b0ad 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -44,6 +44,7 @@ from graphify.extractors.json_config import extract_json # noqa: F401 from graphify.extractors.markdown import extract_markdown # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 +from graphify.extractors.perl import _resolve_perl_imports, extract_perl # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 @@ -1785,6 +1786,7 @@ def _lang_is_case_insensitive(source_file: object) -> bool: ".lua": "lua", ".luau": "lua", ".zig": "zig", ".ex": "elixir", ".exs": "elixir", + ".pl": "perl", ".pm": "perl", ".jl": "julia", ".dart": "dart", ".sh": "shell", ".bash": "shell", @@ -2750,6 +2752,33 @@ def _key(label: str) -> str: ) +def _resolve_perl_imports_pass(per_file, all_nodes, all_edges, paths) -> None: + """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. + + Registered LAST so it runs after the shared cross-file call pass (which reads + the bare module-label ``use`` targets via ``_has_package_import_evidence``) and + after the member-call resolvers — the same position as its former inline call at + the tail of ``extract()``. Scoped by extractor PROVENANCE, not suffix: an + extensionless ``#!/usr/bin/perl`` script is dispatched to ``extract_perl`` by + shebang and must be re-pointed too, which is why it declares a custom + ``activate`` predicate (a shebang-only corpus has no ``.pl``/``.pm`` suffix) and + takes ``paths`` (``wants_paths``) to recompute that provenance set. + """ + perl_sources = {str(p) for p in paths if _get_extractor(p) is extract_perl} + _resolve_perl_imports(all_nodes, all_edges, perl_sources) + + +register_language_resolver( + LanguageResolver( + "perl_import_repoint", + frozenset({".pl", ".pm"}), + _resolve_perl_imports_pass, + activate=lambda paths: any(_get_extractor(p) is extract_perl for p in paths), + wants_paths=True, + ) +) + + # Inline markdown link: [text](target "optional title"). The negative lookbehind # excludes images (![alt](src)). The target stops at whitespace/closing paren so # an optional "title" after the URL is dropped; an optional <...> wrapper is too. @@ -3777,6 +3806,8 @@ def add_existing_edge(edge: dict) -> None: ".psd1": extract_powershell_manifest, ".ex": extract_elixir, ".exs": extract_elixir, + ".pl": extract_perl, + ".pm": extract_perl, ".m": extract_objc, ".mm": extract_objc, ".jl": extract_julia, @@ -3853,7 +3884,7 @@ def add_existing_edge(edge: dict) -> None: # routes them to the CODE path via _shebang_interpreter; _get_extractor must # honor the same signal or these files are classified as code and then silently # dropped by extraction. Only interpreters with a real extractor are mapped — -# detect's wider set (perl, fish, tcsh, Rscript) stays unmapped and skipped. +# detect's wider set (fish, tcsh, Rscript) stays unmapped and skipped. _SHEBANG_DISPATCH: dict[str, Any] = { "python": extract_python, "python2": extract_python, @@ -3869,6 +3900,7 @@ def add_existing_edge(edge: dict) -> None: "lua": extract_lua, "php": extract_php, "julia": extract_julia, + "perl": extract_perl, } @@ -4605,6 +4637,21 @@ def extract( # file is real ONLY if the caller imported it. So a cross-file call from one # of these files with no import evidence is gated below (#1659). _JS_TS_CALL_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") + + # Enclosing-package label per node id, for the package-aware call resolution + # below. Only consulted for raw_calls that carry package qualifiers (Perl): + # a sub's direct container is its package node, whose label IS the package + # name (e.g. "Acme::Widget"), so the `contains` edge target->source gives + # sub_id -> package label. Other languages never set the package fields, so + # this map is built but never read for them. + _label_by_nid = {n["id"]: n.get("label", "") for n in all_nodes} + pkg_label_by_nid: dict[str, str] = {} + pkg_nid_by_sub_nid: dict[str, str] = {} + for e in all_edges: + if e.get("relation") == "contains": + pkg_label_by_nid[e["target"]] = _label_by_nid.get(e["source"], "") + pkg_nid_by_sub_nid[e["target"]] = e["source"] + for rc in all_raw_calls: callee = rc.get("callee", "") if not callee: @@ -4671,6 +4718,62 @@ def _has_import_evidence(candidate_id: str) -> bool: or (candidate_file_nid is not None and candidate_file_nid in imported_modules) ) + def _has_package_import_evidence(candidate_id: str) -> bool: + # Perl-only: `use P::A;` emits an imports edge to the MODULE label id + # (`_make_id('P::A')`), never to the sub id — so a bare call to an + # imported package's sub has no direct symbol/module evidence above. + # Bridge it: the candidate sub's enclosing package, re-idized the same + # way the `use` target is, must be among the caller file's imports. + pkg = pkg_label_by_nid.get(candidate_id, "") + if not pkg: + return False + if _make_id(pkg) in imported_symbols: + return True + # Accept the real package-node id form too. The import re-pointer rewrites + # a `use` target from the bare module-label id onto the package node id; + # matching either shape means this evidence check no longer depends on + # whether that re-pointer has run yet — the pass ordering stops being + # semantically load-bearing (A3). The current order (re-pointer after this + # pass) is kept regardless. + pkg_nid = pkg_nid_by_sub_nid.get(candidate_id) + return pkg_nid is not None and pkg_nid in imported_symbols + + # Package-aware pre-filter for Perl, which tags every call with its + # enclosing package. Gated on the extractor-stamped `lang` (matching the + # cpp/csharp/java/objc raw-call consumers) rather than field-presence, so + # the branch claims exactly Perl's raw_calls and another language that + # happened to set a `*_package` field could never fall into it. Zero-edge + # over a wrong guess: an unresolvable qualifier or a foreign-package bare + # call is dropped, not bound to a same-named sub in the wrong package. + callee_package = rc.get("callee_package") + caller_package = rc.get("caller_package") + if rc.get("lang") == "perl": + if callee_package is not None: + # Qualified call `Pkg::sub()`: bind only to a sub whose enclosing + # package matches the qualifier. None or several -> drop. + candidates = [ + c for c in candidates + if pkg_label_by_nid.get(c) == callee_package + ] + else: + # Bare call: prefer the caller's own package. If the sub is + # defined there, that's the target. Otherwise it can only be an + # imported sub — require unique import evidence, else drop (never + # bind a same-named sub from an unrelated package). + same_pkg = [ + c for c in candidates + if pkg_label_by_nid.get(c) == caller_package + ] + if same_pkg: + candidates = same_pkg + else: + candidates = [ + c for c in candidates + if _has_import_evidence(c) or _has_package_import_evidence(c) + ] + if len(candidates) != 1: + continue + if len(candidates) == 1: tgt = candidates[0] has_import_evidence = _has_import_evidence(tgt) @@ -4774,6 +4877,9 @@ def _has_import_evidence(candidate_id: str) -> bool: # receiver-typed/qualified calls the shared pass skipped) with its own # single-definition god-node guard. Registered in graphify.resolver_registry so # a new language plugs in without editing this body (#1356 Swift, #1446 Python). + # The Perl import re-pointer (`perl_import_repoint`) is the last registered + # resolver, so it runs here — after the call pass and member-call resolvers, + # before the relativization below — the same position as its former inline call. run_language_resolvers(paths, per_file, all_nodes, all_edges) # Relativize source_file fields so paths are portable across machines (#555) diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada517094..601ed547d 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -24,6 +24,7 @@ from graphify.extractors.objc import extract_objc from graphify.extractors.pascal import extract_pascal from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form +from graphify.extractors.perl import extract_perl from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest from graphify.extractors.razor import extract_razor from graphify.extractors.rust import extract_rust @@ -52,6 +53,7 @@ "markdown": extract_markdown, "objc": extract_objc, "pascal": extract_pascal, + "perl": extract_perl, "powershell": extract_powershell, "powershell_manifest": extract_powershell_manifest, "razor": extract_razor, diff --git a/graphify/extractors/perl.py b/graphify/extractors/perl.py new file mode 100644 index 000000000..e86c89285 --- /dev/null +++ b/graphify/extractors/perl.py @@ -0,0 +1,517 @@ +"""Perl extractor: packages, subs, imports (use/require), inheritance (@ISA/parent/base). + +Deliberately untyped, consistent with the other language extractors. Calls are +handed to the shared cross-file second pass as ``raw_calls`` (INFERRED by +name-matching); method calls (``$obj->meth()``) are marked ``is_member_call`` so +that pass drops them — without receiver types they are unresolvable and naive +name-matching would wire spurious edges to same-named subs. +""" +from __future__ import annotations + +import logging +import re +from pathlib import Path +from typing import Any + +from graphify.extractors.base import _file_stem, _make_id + +_LOG = logging.getLogger(__name__) + +# A valid Perl package/class name: a bareword component (`Foo`, `_priv`) optionally +# joined by `::`. Inheritance targets come from arbitrary string_literal / qw() +# content (`@ISA`, `use parent`, `use base`), so a crafted or malformed string +# (control chars, markdown, newlines, an over-long blob) could otherwise flow raw +# into a node label and on into graph.json / the Obsidian export. Names that do +# not match are discarded (zero-edge over a bogus stub). +# +# The classes are spelled out as explicit ASCII ranges (not ``\w``): Python's +# ``\w`` is Unicode-default, so accented (``Basé``), fullwidth (``Ｂase``) and +# other non-ASCII barewords would slip through. Every ``::`` component must begin +# with a letter/underscore, which also rejects a digit-start component +# (``Acme::1x``); ``fullmatch`` (below) anchors the whole string, so a trailing +# newline cannot pass on a ``$`` alone. +_PERL_PKG_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)*") +_MAX_PERL_PKG_NAME_LEN = 256 + +# Coarse guard so a pathologically large or deeply nested file cannot make the +# (now iterative) tree walks run away; on exhaustion the file keeps whatever was +# already extracted (file node + partial graph) instead of nothing. +_MAX_PERL_TRAVERSAL_NODES = 2_000_000 + + +def _is_valid_perl_package_name(name: str) -> bool: + return ( + bool(name) + and len(name) <= _MAX_PERL_PKG_NAME_LEN + and _PERL_PKG_NAME_RE.fullmatch(name) is not None + ) + +# ``use strict`` & friends are compiler pragmas, not module dependencies — they +# must not become imports edges (matches the pragma-exclusion in other langs). +_PERL_PRAGMAS: frozenset[str] = frozenset({ + "strict", "warnings", "utf8", "constant", "vars", "lib", "feature", + "integer", "bytes", "overload", "mro", "autodie", "diagnostics", + "sort", "subs", "attributes", "fields", "encoding", "if", "less", +}) + +# ``use parent`` / ``use base`` declare inheritance, not an import. +_PERL_INHERIT_PRAGMAS: frozenset[str] = frozenset({"parent", "base"}) + +# Perl builtins that surface as function_call_expression callees; excluding them +# keeps them from accumulating spurious calls edges as god-nodes. Kept to the +# canonical perlfunc core so a user sub that happens to share a name with a +# builtin still resolves against real definitions in the corpus. +_PERL_BUILTINS: frozenset[str] = frozenset({ + # I/O & formatting + "print", "printf", "say", "sprintf", "open", "close", "read", "write", + "binmode", "eof", "seek", "tell", "sysread", "syswrite", "readline", + # process / system + "system", "exec", "fork", "wait", "waitpid", "kill", "sleep", "time", + "exit", "die", "warn", + # filesystem + "mkdir", "rmdir", "unlink", "rename", "chdir", "chmod", "chown", "stat", + "lstat", "opendir", "readdir", "closedir", "glob", + # list / hash ops + "shift", "unshift", "push", "pop", "splice", "map", "grep", "sort", + "reverse", "join", "split", "keys", "values", "each", "exists", "delete", + # string ops + "index", "rindex", "substr", "length", "uc", "lc", "ucfirst", "lcfirst", + "chomp", "chop", "chr", "ord", "hex", "oct", "pack", "unpack", + "quotemeta", + # math + "abs", "int", "sqrt", "sin", "cos", "atan2", "exp", "log", "rand", "srand", + # scalars / refs / types + "defined", "ref", "scalar", "bless", "return", "eval", "local", "my", + "our", "sub", "do", "require", "use", "no", "wantarray", +}) + + +def extract_perl(path: Path) -> dict: + """Extract packages, subs, imports and inheritance from a .pl/.pm file.""" + try: + import tree_sitter_perl as tsperl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree_sitter_perl not installed"} + + try: + language = Language(tsperl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = _file_stem(path) + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + # Dedup edges on their identity tuple so a repeated construct (e.g. two + # `use Foo;` in one file) yields a single edge instead of parallel dups. + seen_edges: set[tuple[str, str, str, str | None]] = set() + # sub_nid, body block, and the sub's enclosing package name (for the shared + # second pass's package-aware call resolution). + sub_bodies: list[tuple[str, Any, str | None]] = [] + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + + def add_stub_node(nid: str, label: str) -> None: + """External inheritance target (base class defined elsewhere / in the RTL). + + Emitted with an empty ``source_file`` so it does not falsely claim this + child file as the parent's source and so the corpus-level cross-file + rewire can collapse it onto the real definition; ``origin_file`` keeps + distinct same-named stubs apart in the colliding-id pass. Mirrors the + external-stub shape used by go/julia/objc. + """ + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": "", "source_location": "", + "origin_file": str_path}) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", context: str | None = None) -> None: + key = (src, tgt, relation, context) + if key in seen_edges: + return + seen_edges.add(key) + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0} + if context: + edge["context"] = context + edges.append(edge) + + file_nid = _make_id(str(path)) + add_node(file_nid, path.name, 1) + + def _package_name(node) -> str | None: + """`package Foo::Bar;` -> the second `package`-typed child is the name.""" + names = [c for c in node.children if c.type == "package"] + return _text(names[1]) if len(names) >= 2 else None + + def _string_parents(node) -> list[str]: + """Every string / qw-word parent named under an inheritance construct. + + Reads string_literal (`'Foo'`) and quoted_word_list (`qw(Foo Bar)`) + content anywhere below ``node``; autoquoted barewords like ``-norequire`` + are a different node type and are intentionally skipped. + """ + out: list[str] = [] + stack = [node] + while stack: + if not _spend(): + break + n = stack.pop() + if n.type in ("string_literal", "interpolated_string_literal", "quoted_word_list"): + for c in n.children: + if c.type == "string_content": + out.extend(_text(c).split()) + continue # an inheritance string's own children are not parents + stack.extend(reversed(n.children)) + return out + + def add_inherits(pkg_nid: str, parent_name: str, line: int) -> None: + # Inheritance targets are raw string content; discard anything that is not + # a well-formed package name so a malformed/crafted string never becomes a + # node label or an edge (zero-edge, matching the untyped-drop discipline). + if not _is_valid_perl_package_name(parent_name): + return + parent_nid = _make_id(parent_name) + add_stub_node(parent_nid, parent_name) + add_edge(pkg_nid, parent_nid, "inherits", line, context="inherit") + + current_pkg_nid: str | None = None + current_pkg_name: str | None = None + main_pkg_nid: str | None = None + + def _ensure_main_pkg() -> str: + """Perl's implicit default package. Code with no ``package`` statement lives + in ``main``; modeling it explicitly (instead of hanging package-less subs off + the file node) means a qualified ``main::helper()`` call binds, and a bare + call from a package-less file is scoped to ``main`` — so it cannot be + mis-bound to a same-named sub in an unrelated package without import + evidence. Created lazily so a file with no package-less subs gets no empty + ``main`` node.""" + nonlocal main_pkg_nid + if main_pkg_nid is None: + main_pkg_nid = _make_id(stem, "main") + add_node(main_pkg_nid, "main", 1) + add_edge(file_nid, main_pkg_nid, "contains", 1) + return main_pkg_nid + + budget = [_MAX_PERL_TRAVERSAL_NODES] + budget_warned = [False] + + def _spend() -> bool: + """Charge one node against the shared traversal budget; False once spent so + the walkers stop instead of running away. Emits one bounded warning.""" + budget[0] -= 1 + if budget[0] < 0: + if not budget_warned[0]: + budget_warned[0] = True + _LOG.warning( + "perl: traversal budget exhausted for %s; graph for this file is partial", + str_path, + ) + return False + return True + + def handle_use(node, line: int) -> None: + # In a use_statement the `use` keyword is its own node type, so the + # module/pragma name is the first (and only) `package`-typed child — + # unlike package_statement, where `package` is also the keyword's type. + pkgs = [c for c in node.children if c.type == "package"] + if not pkgs: + return + module = _text(pkgs[0]) + if module in _PERL_INHERIT_PRAGMAS: + if current_pkg_nid: + for parent in _string_parents(node): + add_inherits(current_pkg_nid, parent, line) + return + if module in _PERL_PRAGMAS: + return + add_edge(file_nid, _make_id(module), "imports", line, context="import") + + def handle_require(req_node, line: int) -> None: + for c in req_node.children: + if c.type == "bareword": + add_edge(file_nid, _make_id(_text(c)), "imports", line, context="import") + return + + def handle_isa(assign_node, line: int) -> None: + """`our @ISA = (...)` -> inherits edges to each named parent.""" + if not current_pkg_nid: + return + is_isa = False + for child in assign_node.children: + if child.type == "variable_declaration": + for sub in child.children: + if sub.type == "array" and _text(sub) == "@ISA": + is_isa = True + if not is_isa: + return + for parent in _string_parents(assign_node): + add_inherits(current_pkg_nid, parent, line) + + def walk_statements(root_node) -> None: + nonlocal current_pkg_nid, current_pkg_name + # Manual call stack in place of recursion: a pathologically deep nest of + # block-form packages would otherwise blow the Python stack, and the + # resulting RecursionError makes `_safe_extract` drop the WHOLE file. Each + # frame is an iterator over one block's statements plus the scope to restore + # once that block is exhausted; a block-form `package Foo { ... }` pushes a + # child frame under Foo's scope, so its subs are attributed to Foo and the + # prior package is restored for statements that follow the block. + stack: list[tuple[Any, str | None, str | None]] = [ + (iter(root_node.children), current_pkg_nid, current_pkg_name) + ] + while stack: + child_iter, restore_nid, restore_name = stack[-1] + descended = False + for child in child_iter: + # Charge every visited sibling, not once per frame: a broad flat + # file drains an unbounded number of children under a single frame, + # so a per-frame charge left them effectively free. + if not _spend(): + return # budget exhausted: keep the partial graph, stop walking + line = child.start_point[0] + 1 + if child.type == "package_statement": + name = _package_name(child) + if name: + pkg_nid = _make_id(stem, name) + add_node(pkg_nid, name, line) + add_edge(file_nid, pkg_nid, "contains", line) + pkg_block = next( + (c for c in child.children if c.type == "block"), None) + if pkg_block is not None: + # Descend into the block under Foo; the frame remembers + # the pre-block scope so it is restored when the block is + # fully consumed (statements after the block are not + # mis-attributed to Foo). + prev_nid, prev_name = current_pkg_nid, current_pkg_name + current_pkg_nid, current_pkg_name = pkg_nid, name + stack.append((iter(pkg_block.children), prev_nid, prev_name)) + descended = True + break + else: + current_pkg_nid = pkg_nid + current_pkg_name = name + elif child.type == "use_statement": + handle_use(child, line) + elif child.type == "subroutine_declaration_statement": + name = None + block = None + for c in child.children: + if c.type == "bareword" and name is None: + name = _text(c) + elif c.type == "block": + block = c + if name: + if "::" in name: + # Qualified declaration `sub Pkg::sub {...}` defines the + # sub IN the named package, not the current one. Container + # = that package (created if it has no `package` statement + # of its own); the body's caller-package is the qualifier + # so its calls resolve against Pkg. + pkg_qual, _, sub_name = name.rpartition("::") + container = _make_id(stem, pkg_qual) + add_node(container, pkg_qual, line) + add_edge(file_nid, container, "contains", line) + sub_package = pkg_qual + else: + # Package-less sub → Perl's `main` (not the file node), so + # `main::sub()` binds and bare same-file calls resolve. + container = current_pkg_nid or _ensure_main_pkg() + sub_name = name + sub_package = current_pkg_name or "main" + sub_nid = _make_id(container, sub_name) + add_node(sub_nid, f"{sub_name}()", line) + add_edge(container, sub_nid, "contains", line) + if block is not None: + sub_bodies.append((sub_nid, block, sub_package)) + elif child.type == "expression_statement": + for c in child.children: + if c.type == "require_expression": + handle_require(c, line) + elif c.type == "assignment_expression": + handle_isa(c, line) + if descended: + continue + stack.pop() + current_pkg_nid, current_pkg_name = restore_nid, restore_name + + walk_statements(root) + + raw_calls: list[dict] = [] + + def walk_calls(root_node, caller_nid: str, caller_package: str | None) -> None: + # Iterative (see walk_statements): a deeply nested expression / data + # structure in a sub body would recurse once per level, and RecursionError + # here would drop the whole file via `_safe_extract`. + stack = [root_node] + while stack: + if not _spend(): + break + node = stack.pop() + if node.type == "function_call_expression": + # Indirect-object constructor `new CLASS(...)` parses as + # ambiguous_function_call_expression(function 'new', function_call_expression + # 'CLASS()'); `new CLASS` == CLASS->new, an untyped member dispatch. Mark + # it a member call (edge-less) so it is not wired to a sub named CLASS. + parent = node.parent + indirect_new = ( + parent is not None + and parent.type == "ambiguous_function_call_expression" + and bool(parent.children) + and parent.children[0].type == "function" + and _text(parent.children[0]) == "new" + ) + for c in node.children: + if c.type == "function": + # `Acme::Helper::emit` -> callee `emit`, callee_package + # `Acme::Helper`; a bare `emit` -> callee `emit`, no package. + # The qualifier + caller package let the shared second pass + # bind to the right same-named sub instead of any `emit()`. + parts = _text(c).split("::") + callee = parts[-1] + callee_package = "::".join(parts[:-1]) or None + if callee and callee not in _PERL_BUILTINS: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "callee_package": callee_package, + "caller_package": caller_package, + "is_member_call": indirect_new, + "lang": "perl", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + elif node.type == "method_call_expression": + for c in node.children: + if c.type == "method": + callee = _text(c).split("::")[-1] + if callee: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": True, + "lang": "perl", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + break + stack.extend(reversed(node.children)) + + for caller_nid, body, caller_package in sub_bodies: + walk_calls(body, caller_nid, caller_package) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] == "imports")] + return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, + "input_tokens": 0, "output_tokens": 0} + + +def _resolve_perl_imports( + all_nodes: list[dict], + all_edges: list[dict], + perl_source_files: set[str] | None = None, +) -> None: + """Re-point dangling in-corpus Perl ``imports`` edges onto the real package node. + + ``use Foo::Bar;`` / ``require Foo::Bar;`` emit an imports edge to a bare + module-label id (``_make_id('Foo::Bar')``) that matches no node: when Foo::Bar + is defined in the corpus its package node's id is ``_make_id(stem, 'Foo::Bar')`` + (and a ``package Foo::Bar;`` may live in a file whose stem is unrelated — a + multi-package file — so we key on the package LABEL, not the file stem). Bridge + module-label -> real package id so in-corpus imports connect instead of + dangling. External modules (POSIX, Carp, …) have no in-corpus package node, so + their edge keeps its bare target — matching the dangling-stub behavior other + languages leave on unresolved external imports. + + ``perl_source_files`` (absolute-path strings, from ``extract()`` where + ``_get_extractor(path) is extract_perl``) scopes both the package-node scan and + the re-pointed edges to Perl provenance. Suffix-based scoping (``.pl``/``.pm``) + silently skipped extensionless ``#!/usr/bin/perl`` scripts, which are dispatched + to extract_perl by shebang and whose imports were never re-pointed + (underreporting). When ``None`` (direct callers), falls back to suffix matching. + + Runs AFTER the shared cross-file call pass: ``_has_package_import_evidence`` + (extract.py) reads imports targets as bare module-label ids to bind a bare call + to an imported package's sub, so re-pointing before it would break that binding. + Mutates ``all_edges`` in place; the bare target was never a node, so there is + nothing to prune. + """ + # Provenance matching must survive the cache round-trip. A fresh run stamps a + # node's `source_file` with the raw `str(path)` the extractor was handed, which + # is exactly what `perl_source_files` holds — so exact membership matches. But a + # cached fragment is stored relative and re-anchored on load against the RESOLVED + # cache root, so its `source_file` comes back as an absolute resolved path that + # no longer equals the raw provenance string — exact membership then misses and + # the re-pointer silently no-ops on the second (cached) run. Compare on the + # resolved form so both shapes collapse to the same on-disk identity. + resolved_perl_sources: set[str] | None = None + if perl_source_files is not None: + resolved_perl_sources = set(perl_source_files) + for s in perl_source_files: + try: + resolved_perl_sources.add(str(Path(s).resolve())) + except OSError: + pass + _resolve_memo: dict[str, bool] = {} + + def _is_perl_source(src: str) -> bool: + if perl_source_files is None: + return src.endswith((".pl", ".pm")) + if src in resolved_perl_sources: + return True + hit = _resolve_memo.get(src) + if hit is None: + try: + hit = str(Path(src).resolve()) in resolved_perl_sources + except OSError: + hit = False + _resolve_memo[src] = hit + return hit + + # module-label id -> list of package node ids carrying that fully-qualified + # label. A bare `use Foo` cannot disambiguate a package re-opened under the + # same label across files (e.g. `package Assert;` in both AssertOn.pm and + # AssertOff.pm), so we re-point ONLY when exactly one package node matches; + # >1 candidate stays dangling (zero-edge over a guessed cross-file binding). + pkg_ids_by_label_id: dict[str, list[str]] = {} + for node in all_nodes: + src = str(node.get("source_file") or "") + if not _is_perl_source(src): + continue + label = node.get("label", "") + nid = node.get("id", "") + # package nodes only: skip the file node (label == basename) and subs + # (label ends in `()`); neither keys an import target. + if not label or not nid or label.endswith(")") or label == Path(src).name: + continue + pkg_ids_by_label_id.setdefault(_make_id(label), []).append(nid) + if not pkg_ids_by_label_id: + return + for edge in all_edges: + if edge.get("relation") != "imports": + continue + # Scope to Perl imports only, so a same-named module-label id in another + # language's imports edge is never re-pointed onto a Perl package node. + if not _is_perl_source(str(edge.get("source_file") or "")): + continue + candidates = pkg_ids_by_label_id.get(edge.get("target")) + if candidates and len(candidates) == 1 and candidates[0] != edge["target"]: + edge["target"] = candidates[0] diff --git a/graphify/resolver_registry.py b/graphify/resolver_registry.py index b17478a78..cb480c5d5 100644 --- a/graphify/resolver_registry.py +++ b/graphify/resolver_registry.py @@ -33,11 +33,26 @@ class LanguageResolver: mutates ``all_nodes`` / ``all_edges`` in place, matching the existing member-call resolvers. ``suffixes`` gates activation: the pass runs only when the corpus contains at least one file with one of these extensions. + + Two optional hooks cover passes that suffix-gating cannot express — a + provenance-scoped pass (one whose file membership is decided by which + extractor produced a node, not by extension) may need both: + + - ``activate`` overrides the suffix gate with a predicate over ``paths``. Use it + when membership is decided by EXTRACTOR provenance rather than extension — a + file dispatched to an extractor by content (e.g. a shebang) may carry no + matching suffix to gate on, yet must still activate the pass. When ``None`` + the suffix gate is used. + - ``wants_paths`` makes the driver call ``resolve`` with a fourth positional + argument, the ``paths`` sequence, so a provenance-scoped pass can recompute + which files it owns. Default ``False`` keeps the three-argument contract. """ name: str suffixes: frozenset resolve: Callable + activate: Callable | None = None + wants_paths: bool = False # Module-level registry, populated by callers via register(). Ordered: resolvers @@ -77,9 +92,19 @@ def run_language_resolvers( active = _REGISTRY if resolvers is None else resolvers suffixes_present = {p.suffix for p in paths} for resolver in active: - if not (resolver.suffixes & suffixes_present): + if resolver.activate is not None: + try: + if not resolver.activate(paths): + continue + except Exception as exc: + _LOG.warning("%s activation check failed, skipping: %s", resolver.name, exc) + continue + elif not (resolver.suffixes & suffixes_present): continue try: - resolver.resolve(per_file, all_nodes, all_edges) + if resolver.wants_paths: + resolver.resolve(per_file, all_nodes, all_edges, paths) + else: + resolver.resolve(per_file, all_nodes, all_edges) except Exception as exc: _LOG.warning("%s resolution failed, skipping: %s", resolver.name, exc) diff --git a/pyproject.toml b/pyproject.toml index 510e77469..3efaad19f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "tree-sitter-fortran>=0.6,<0.8", "tree-sitter-bash>=0.23,<0.27", "tree-sitter-json>=0.23,<0.26", + "tree-sitter-perl>=1.2.0,<2.0", ] [project.urls] diff --git a/tests/fixtures/sample.pl b/tests/fixtures/sample.pl new file mode 100644 index 000000000..102f75bf6 --- /dev/null +++ b/tests/fixtures/sample.pl @@ -0,0 +1,27 @@ +#!/usr/bin/perl +use strict; +use warnings; +use Acme::Widget; + +package Acme::Helper; + +sub emit { + my ($text) = @_; + print "$text\n"; + return length $text; +} + +sub format { + my ($text) = @_; + return uc $text; +} + +package main; + +sub run_report { + my $widget = Acme::Widget->new(); + $widget->render(); + Acme::Helper::emit("done"); +} + +run_report(); diff --git a/tests/fixtures/sample_module.pm b/tests/fixtures/sample_module.pm new file mode 100644 index 000000000..02e0eaa20 --- /dev/null +++ b/tests/fixtures/sample_module.pm @@ -0,0 +1,46 @@ +package Acme::Widget; + +use strict; +use warnings; +use Scalar::Util qw(blessed); +use Carp; +require Acme::Helper; + +use parent -norequire, 'Acme::Role'; +use base 'Acme::Mixin'; + +our @ISA = ('Acme::Base'); + +sub new { + my ($class, %args) = @_; + my $self = { %args }; + return bless $self, $class; +} + +sub render { + my $self = shift; + my $line = format_line(); + Acme::Helper::emit($line); + $self->update(); + return $line; +} + +sub format_line { + return "rendered"; +} + +sub update { + my $self = shift; + $self->{dirty} = 1; + return $self; +} + +package Acme::Widget::Inner; + +use POSIX qw(floor); + +sub tick { + return floor(1.5); +} + +1; diff --git a/tests/test_cache.py b/tests/test_cache.py index 9bf2cdc82..ec981d7d9 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -540,3 +540,58 @@ def test_save_semantic_cache_merge_existing_unions(tmp_path): ids = {n["id"] for n in cached["nodes"]} assert ids == {"a", "b"}, "merge_existing must union both chunk slices" assert len(cached["edges"]) == 1 + + +def test_ast_cache_invalidated_on_schema_bump(tmp_path, monkeypatch): + """AST output SHAPE can change within a release (no version bump) — e.g. a new + per-call `lang` stamp or a lazily-materialized package node. Bumping + _AST_SCHEMA_VERSION must invalidate entries written under the prior schema even + when _EXTRACTOR_VERSION is unchanged (F1).""" + import graphify.cache as cache_mod + + f = tmp_path / "mod.pl" + f.write_text("package Foo;\nsub run { helper(); }\n") + + monkeypatch.setattr(cache_mod, "_AST_SCHEMA_VERSION", 1, raising=False) + save_cached(f, {"nodes": [{"id": "n1"}], "edges": [], "raw_calls": []}, + root=tmp_path, kind="ast") + assert load_cached(f, root=tmp_path, kind="ast") is not None + + monkeypatch.setattr(cache_mod, "_AST_SCHEMA_VERSION", 2, raising=False) + assert load_cached(f, root=tmp_path, kind="ast") is None, ( + "an AST entry from a prior schema version must not be served after a bump" + ) + + +def test_ast_schema_version_in_cache_dir(tmp_path): + """The AST cache directory carries the schema counter alongside the package + version, so a shape change relocates the namespace (F1).""" + import graphify.cache as cache_mod + + d = cache_dir(tmp_path, "ast") + assert d.name == f"v{cache_mod._EXTRACTOR_VERSION}-s{cache_mod._AST_SCHEMA_VERSION}", ( + f"AST cache dir must namespace by version AND schema, got {d.name!r}" + ) + + +def test_pre_schema_ast_entries_not_served(tmp_path): + """An entry written by a base-commit graphify (version-only namespace + `v{version}/`, before the schema counter existed) is by definition from an + older extractor SHAPE and must not be served — the invalidation the schema + bump provides for changes that ship without a version bump (F1).""" + import json + from graphify.cache import file_hash, _GRAPHIFY_OUT, _EXTRACTOR_VERSION + + f = tmp_path / "mod.pl" + f.write_text("package Foo;\nsub run { helper(); }\n") + h = file_hash(f, tmp_path) + + # pre-schema layout: cache/ast/v{version}/{hash}.json (no `-s{schema}` suffix) + old_dir = tmp_path / _GRAPHIFY_OUT / "cache" / "ast" / f"v{_EXTRACTOR_VERSION}" + old_dir.mkdir(parents=True) + (old_dir / f"{h}.json").write_text( + json.dumps({"nodes": [{"id": "stale"}], "edges": [], "raw_calls": []})) + + assert load_cached(f, root=tmp_path, kind="ast") is None, ( + "a version-only (pre-schema) AST entry must not be served" + ) diff --git a/tests/test_extract.py b/tests/test_extract.py index 6247ec59c..eeffd8934 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1618,6 +1618,11 @@ def test_extensionless_shebang_via_dispatch(tmp_path): split.write_text("#!/usr/bin/env -S bash -eu\necho hi\n") assert _get_extractor(split) is extract_bash + from graphify.extract import extract_perl + perltool = tmp_path / "legacy" + perltool.write_text("#!/usr/bin/perl\nprint 1;\n") + assert _get_extractor(perltool) is extract_perl + def test_extensionless_without_usable_shebang_stays_unsupported(tmp_path): from graphify.extract import _get_extractor @@ -1628,9 +1633,9 @@ def test_extensionless_without_usable_shebang_stays_unsupported(tmp_path): # Interpreter known to detect but with no AST extractor: stays skipped # rather than being mis-parsed by a wrong grammar. - perl = tmp_path / "legacy" - perl.write_text("#!/usr/bin/env perl\nprint 1;\n") - assert _get_extractor(perl) is None + tcsh = tmp_path / "legacy" + tcsh.write_text("#!/usr/bin/env tcsh\necho 1\n") + assert _get_extractor(tcsh) is None def test_extract_extensionless_bash_cli_end_to_end(tmp_path): diff --git a/tests/test_language_resolvers.py b/tests/test_language_resolvers.py index 787c1d505..bff014576 100644 --- a/tests/test_language_resolvers.py +++ b/tests/test_language_resolvers.py @@ -72,3 +72,54 @@ def _add_edge(per_file, all_nodes, all_edges): edges: list[dict] = [] run_language_resolvers([Path("a.rb")], [], [], edges, resolvers=resolvers) assert edges == [{"source": "x", "target": "y", "relation": "calls"}] + + +def test_resolver_activate_predicate_overrides_suffix_gate() -> None: + # Provenance-gated pass: no matching suffix present, but the activate predicate + # opts it in anyway (e.g. an extensionless shebang script). + log: list[str] = [] + + def _resolve(per_file, all_nodes, all_edges): + log.append("ran") + + resolver = LanguageResolver( + "prov", frozenset({".rb"}), _resolve, + activate=lambda paths: any(p.name == "runme" for p in paths), + ) + run_language_resolvers([Path("runme")], [], [], [], resolvers=[resolver]) + assert log == ["ran"] + + +def test_resolver_activate_false_skips_even_with_matching_suffix() -> None: + log: list[str] = [] + + def _resolve(per_file, all_nodes, all_edges): + log.append("ran") + + resolver = LanguageResolver( + "prov", frozenset({".rb"}), _resolve, activate=lambda paths: False, + ) + run_language_resolvers([Path("a.rb")], [], [], [], resolvers=[resolver]) + assert log == [] # activate wins over the suffix gate + + +def test_resolver_wants_paths_receives_paths() -> None: + seen: list = [] + + def _resolve(per_file, all_nodes, all_edges, paths): + seen.append(list(paths)) + + resolver = LanguageResolver("wp", frozenset({".rb"}), _resolve, wants_paths=True) + run_language_resolvers([Path("a.rb")], [], [], [], resolvers=[resolver]) + assert seen == [[Path("a.rb")]] + + +def test_default_registry_contains_perl_import_repoint() -> None: + import graphify.extract # noqa: F401 (registers resolvers on import) + + perl = next( + (r for r in registered_resolvers() if r.name == "perl_import_repoint"), None + ) + assert perl is not None, "perl_import_repoint must be registered in the shared registry" + assert perl.wants_paths, "the re-pointer needs paths to recompute Perl provenance" + assert perl.activate is not None, "the re-pointer activates by extractor provenance, not suffix" diff --git a/tests/test_languages.py b/tests/test_languages.py index f610fbc40..4e89b429c 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -2945,3 +2945,751 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): r = _corpus("cpp_samedir/Alpha.h", "cpp_samedir/Beta.h") dups = _nodes_with_label(r, "Dup") assert len(dups) == 2, f"same-dir distinct Dups must stay distinct, got {[n['id'] for n in dups]}" + + +# ── Perl ────────────────────────────────────────────────────────────────────── +# `extract_perl` (and `extract`) are imported INSIDE each test rather than at +# module top so this test module still imports even if the extractor is absent — +# only the Perl tests would fail, not the whole suite's collection. + +def test_perl_no_error(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert "error" not in r + +def test_perl_finds_packages(): + """Both the leading `package Acme::Widget` and the mid-file + `package Acme::Widget::Inner` switch must surface as nodes.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + labels = _labels(r) + assert any("Acme::Widget" in l for l in labels) + assert any("Acme::Widget::Inner" in l for l in labels) + +def test_perl_finds_subs(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + labels = _labels(r) + for name in ("new", "render", "format_line", "update", "tick"): + assert any(name in l for l in labels), f"missing sub {name!r}" + +def test_perl_finds_imports(): + """`use Module` / `require Module` become imports edges; the qw-list form + (`use Scalar::Util qw(blessed)`) and bareword `require` both count.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert "imports" in _relations(r) + import_edges = _edges_with_relation(r, "imports", "imports_from") + targets = " ".join(str(e.get("target", "")) for e in import_edges).lower() + assert "scalar" in targets or "util" in targets, "Scalar::Util import missing" + assert "carp" in targets, "Carp import missing" + assert "helper" in targets, "require Acme::Helper import missing" + +def test_perl_import_edges_have_import_context(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + assert import_edges + assert all(e.get("context") == "import" for e in import_edges) + +def test_perl_import_edges_are_extracted(): + """`use`/`require` are literal in source -> EXTRACTED confidence.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + assert import_edges + assert all(e.get("confidence") == "EXTRACTED" for e in import_edges) + +def test_perl_pragmas_not_imported(): + """`use strict` / `use warnings` are pragmas, not module dependencies, and + must NOT create imports edges (matches the pragma-exclusion in other langs).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + import_edges = _edges_with_relation(r, "imports", "imports_from") + targets = " ".join(str(e.get("target", "")) for e in import_edges).lower() + assert "strict" not in targets, "`use strict` pragma must not be an import" + assert "warnings" not in targets, "`use warnings` pragma must not be an import" + +def test_perl_isa_inherits_edge(): + """`our @ISA = ('Acme::Base')` must emit an inherits edge to Acme::Base.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + found = any( + "Acme::Widget" in node_by_id.get(e["source"], "") + and "Acme::Base" in node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + ) + assert found, "Acme::Widget should have inherits edge to Acme::Base (our @ISA)" + +def test_perl_use_parent_inherits_edge(): + """`use parent -norequire, 'Acme::Role'` must emit an inherits edge; the + `-norequire` flag is not a parent.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + parents = { + node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + and "Acme::Widget" in node_by_id.get(e["source"], "") + } + assert any("Acme::Role" in p for p in parents), "use parent target Acme::Role missing" + assert not any("norequire" in p for p in parents), "-norequire flag must not be a parent" + +def test_perl_use_base_inherits_edge(): + """`use base 'Acme::Mixin'` must emit an inherits edge to Acme::Mixin.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_by_id = {n["id"]: n["label"] for n in r["nodes"]} + parents = { + node_by_id.get(e["target"], "") + for e in r["edges"] if e["relation"] == "inherits" + and "Acme::Widget" in node_by_id.get(e["source"], "") + } + assert any("Acme::Mixin" in p for p in parents), "use base target Acme::Mixin missing" + +def test_perl_inherits_edges_are_extracted(): + """@ISA / use parent / use base are literal in source -> EXTRACTED.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert inherits + assert all(e.get("confidence") == "EXTRACTED" for e in inherits) + +def test_perl_no_dangling_edges(): + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"dangling edge source: {e}" + +def test_perl_inherit_stub_has_no_source_file(): + """Inheritance parents defined outside this file (RTL / another module) must + not claim the child file as their source; they are external stubs with an + empty source_file (mirrors go/julia/objc).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + stubs = [n for n in r["nodes"] + if n["label"] in ("Acme::Base", "Acme::Role", "Acme::Mixin")] + assert stubs, "expected external inheritance stub nodes" + for n in stubs: + assert n.get("source_file") == "", ( + f"external parent stub {n['label']!r} must have empty source_file, " + f"got {n.get('source_file')!r}" + ) + +def test_perl_builtins_not_called(tmp_path): + """Perl core builtins (open/close/system/exec/index/…) surface as call + expressions but are not user subs; they must not become raw-call edges.""" + from graphify.extract import extract_perl + src = tmp_path / "builtins.pm" + src.write_text( + "package B;\n" + "sub run {\n" + " open(my $fh, '<', 'f');\n" + " system('ls');\n" + " my $i = index('abc', 'b');\n" + " my $n = substr('abc', 1);\n" + " close($fh);\n" + "}\n" + "1;\n" + ) + r = extract_perl(src) + callees = {rc["callee"] for rc in r["raw_calls"]} + for b in ("open", "system", "index", "substr", "close"): + assert b not in callees, f"builtin {b!r} must not be a raw call, got {callees}" + +def test_perl_duplicate_use_dedup(tmp_path): + """Two identical `use Foo;` statements must dedup to a single imports edge.""" + from graphify.extract import extract_perl + src = tmp_path / "dup.pm" + src.write_text( + "package D;\n" + "use Foo;\n" + "use Foo;\n" + "1;\n" + ) + r = extract_perl(src) + foo_imports = [e for e in r["edges"] + if e["relation"] == "imports" and "foo" in str(e["target"]).lower()] + assert len(foo_imports) == 1, ( + f"duplicate `use Foo;` must dedup to one edge, got {len(foo_imports)}: {foo_imports}" + ) + +def test_perl_raw_calls_carry_package_qualifiers(): + """The extractor half of package-aware resolution: a bare call carries its + caller's package (and no callee_package); a qualified `Pkg::sub()` carries + the qualifier as callee_package. The shared second pass reads these to bind + the right same-named sub.""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + by_callee = {rc["callee"]: rc for rc in r["raw_calls"] if not rc.get("is_member_call")} + bare = by_callee["format_line"] + assert bare["callee_package"] is None, bare + assert bare["caller_package"] == "Acme::Widget", bare + qualified = by_callee["emit"] + assert qualified["callee_package"] == "Acme::Helper", qualified + assert qualified["caller_package"] == "Acme::Widget", qualified + + +# Perl call resolution runs in the top-level `extract()` second pass (raw_calls -> +# name-matched calls), so these drive the public multi-file dispatch, like the +# ObjC cross-file tests. + +def _perl_corpus(): + from graphify.extract import extract + return extract([FIXTURES / "sample.pl", FIXTURES / "sample_module.pm"], parallel=False) + +def test_perl_bare_sub_call_edge(): + """`format_line()` bare call inside `render` (same file/package) must resolve + to a calls edge render -> format_line.""" + r = _perl_corpus() + calls = _calls(r) + assert any("render" in src and "format_line" in tgt for src, tgt in calls), \ + f"expected render->format_line calls edge, got {calls}" + +def test_perl_static_qualified_call_edge(): + """`Acme::Helper::emit(...)` static, package-qualified call must resolve to a + calls edge targeting the emit sub.""" + r = _perl_corpus() + calls = _calls(r) + assert any("emit" in tgt for _src, tgt in calls), \ + f"expected a calls edge into Acme::Helper::emit, got {calls}" + +def test_perl_call_edges_are_inferred(): + """Second-pass name-matched calls carry INFERRED confidence.""" + r = _perl_corpus() + call_edges = [e for e in r["edges"] if e["relation"] == "calls"] + assert call_edges + assert all(e.get("confidence") == "INFERRED" for e in call_edges), \ + f"perl calls should be INFERRED (second-pass), got {[e.get('confidence') for e in call_edges]}" + +def test_perl_call_edges_have_call_context(): + r = _perl_corpus() + call_edges = _edges_with_relation(r, "calls") + assert call_edges + assert all(e.get("context") == "call" for e in call_edges) + +def test_perl_method_call_no_edge(): + """`$self->update()` / `$widget->render()` are untyped method dispatch and must + NOT create calls edges, even though `update` and `render` are defined subs + (naive name-matching would wrongly connect them).""" + r = _perl_corpus() + calls = _calls(r) + # Positive guard: the corpus must actually produce static/bare calls, so this + # test is not a vacuous pass on an empty graph. + assert any("format_line" in tgt or "emit" in tgt for _src, tgt in calls), \ + f"expected the static/bare calls to resolve before asserting the negatives, got {calls}" + assert not any("update" in tgt for _src, tgt in calls), \ + f"method call $self->update() must not produce a calls edge, got {calls}" + assert not any(tgt in ("render", "render()") + or tgt.endswith("::render") or tgt.endswith(".render") + for _src, tgt in calls), \ + f"method call $widget->render() must not produce a calls edge, got {calls}" + + +def test_perl_indirect_object_new_no_call_edge(tmp_path): + """`my $x = new Widget();` is indirect-object constructor syntax (== Widget->new), + an untyped member dispatch; it must NOT bind to a same-named sub `Widget` as if + `Widget()` were a direct function call.""" + from graphify.extract import extract + (tmp_path / "m.pm").write_text( + "package M;\n" + "sub Widget { return 1; }\n" + "sub helper { return 2; }\n" + "sub run { helper(); my $x = new Widget(); }\n" + "1;\n" + ) + r = extract([tmp_path / "m.pm"], parallel=False) + calls = _calls(r) + # Positive guard: the plain bare call resolves, so the negative isn't vacuous. + assert any("run" in s and "helper" in t for s, t in calls), \ + f"expected run->helper bare-call resolution, got {calls}" + assert not any("Widget" in t for _s, t in calls), \ + f"indirect-object `new Widget()` must not create a calls edge, got {calls}" + + +def test_perl_block_scoped_package(tmp_path): + """Block-form `package Foo { ... }` scopes Foo to the block only: subs inside + the block belong to Foo, and a sub AFTER the block belongs to the package that + was current before the block (no state leak).""" + from graphify.extract import extract_perl + src = tmp_path / "block.pm" + src.write_text( + "package Outer;\n" + "sub outer_sub { }\n" + "package Acme::Block {\n" + " sub inner { }\n" + "}\n" + "sub after_block { }\n" + "1;\n" + ) + r = extract_perl(src) + label = {n["id"]: n["label"] for n in r["nodes"]} + container_of = { + label.get(e["target"]): label.get(e["source"]) + for e in r["edges"] if e["relation"] == "contains" + } + assert "Acme::Block" in label.values(), "block-form package node must be emitted" + assert container_of.get("inner()") == "Acme::Block", ( + f"sub inside a block package must belong to it, got {container_of.get('inner()')!r}" + ) + assert container_of.get("after_block()") == "Outer", ( + f"block-form package must not leak: after_block belongs to Outer, " + f"got {container_of.get('after_block()')!r}" + ) + + +def test_perl_block_scoped_package_body_calls(tmp_path): + """A bare call inside a block-package sub resolves within that package — + proves the block body is actually walked (not silently skipped).""" + from graphify.extract import extract + (tmp_path / "b.pm").write_text( + "package B::Blk {\n" + " sub inner { helper(); }\n" + " sub helper { return 1; }\n" + "}\n" + "1;\n" + ) + r = extract([tmp_path / "b.pm"], parallel=False) + calls = _calls(r) + assert any("inner" in s and "helper" in t for s, t in calls), \ + f"expected inner->helper inside the block package, got {calls}" + + +# Package-aware second-pass resolution: the shared pass reads the raw_calls' +# package qualifiers to bind a call to the same-named sub in the right package. + +def _calls_with_target_pkg(r): + """(caller_label, target_label, target_enclosing_package_label) per calls edge.""" + label = {n["id"]: n.get("label", "") for n in r["nodes"]} + contained_by = {e["target"]: e["source"] for e in r["edges"] if e["relation"] == "contains"} + return [ + (label.get(e["source"], e["source"]), + label.get(e["target"], e["target"]), + label.get(contained_by.get(e["target"]), "")) + for e in r["edges"] if e["relation"] == "calls" + ] + +def test_perl_qualified_call_binds_correct_package(tmp_path): + """Two packages define `emit`; the package-qualified call `P::A::emit()` must + bind only to P::A's emit, never the same-named sub in P::B.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "b.pm").write_text("package P::B;\nsub emit { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text("package P::C;\nsub run { P::A::emit(); }\n1;\n") + r = extract([tmp_path / "a.pm", tmp_path / "b.pm", tmp_path / "c.pm"], parallel=False) + emit_calls = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "emit" in t] + assert emit_calls, f"expected a calls edge into emit, got none: {_calls_with_target_pkg(r)}" + assert all(p == "P::A" for (_s, _t, p) in emit_calls), \ + f"qualified P::A::emit() must bind only to P::A's emit, got {emit_calls}" + +def test_perl_qualified_sub_declaration_call_edge(tmp_path): + """A qualified declaration `sub Ext::Pkg::helper {...}` defines the sub IN + Ext::Pkg (not the current package). A call `Ext::Pkg::helper()` from another + package must bind to it: the node's label is `helper()` and its enclosing + package is Ext::Pkg (so callee/package-qualifier resolution matches).""" + from graphify.extract import extract + (tmp_path / "q.pm").write_text( + "package Main::Mod;\n" + "sub Ext::Pkg::helper { return 1; }\n" + "sub run { Ext::Pkg::helper(); }\n" + "1;\n" + ) + r = extract([tmp_path / "q.pm"], parallel=False) + binds = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "helper" in t] + assert binds, f"expected run->helper via qualified sub decl, got {_calls_with_target_pkg(r)}" + assert all(t == "helper()" and p == "Ext::Pkg" for (_s, t, p) in binds), ( + f"qualified sub must be labelled helper() and enclosed by Ext::Pkg, got {binds}" + ) + + +def test_perl_bare_foreign_package_call_no_edge(tmp_path): + """A bare `helper()` call resolves to a same-package sub, never to a + same-named sub in a DIFFERENT, un-imported package (zero-edge, not a guess).""" + from graphify.extract import extract + (tmp_path / "x.pm").write_text("package X;\nsub helper { return 1; }\n1;\n") + (tmp_path / "y.pm").write_text( + "package Y;\nsub run { helper(); other(); }\nsub other { return 2; }\n1;\n" + ) + r = extract([tmp_path / "x.pm", tmp_path / "y.pm"], parallel=False) + calls = _calls(r) + # Positive guard: the same-package bare call must resolve, so the negative + # assertion below is not a vacuous pass on an empty graph. + assert any("run" in s and "other" in t for s, t in calls), \ + f"expected run->other same-package resolution, got {calls}" + assert not any("helper" in t for _s, t in calls), \ + f"bare helper() must not bind to X::helper without import evidence, got {calls}" + + +def test_perl_bare_call_binds_imported_package(tmp_path): + """A bare `emit()` call resolves to a sub in a FOREIGN package when that + package is pulled in with `use P::A;` — the `use` import is the evidence that + disambiguates the otherwise-foreign sub. `use P::A;` emits an imports edge to + the module label (`_make_id('P::A')`), not to the sub id, so plain + symbol-import evidence misses it; package-import evidence must bridge the gap.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nsub run { emit(); }\n1;\n" + ) + r = extract([tmp_path / "a.pm", tmp_path / "c.pm"], parallel=False) + emit_calls = [(s, t, p) for (s, t, p) in _calls_with_target_pkg(r) if "emit" in t] + assert emit_calls, \ + f"bare emit() with `use P::A;` must bind to P::A::emit, got none: {_calls_with_target_pkg(r)}" + assert all(p == "P::A" for (_s, _t, p) in emit_calls), \ + f"emit() must bind only to the imported P::A::emit, got {emit_calls}" + + +def test_perl_bare_call_two_imported_packages_ambiguous_no_edge(tmp_path): + """Two imported packages both define `emit`; a bare `emit()` is ambiguous — + the `use` import evidence points at two candidates, so the edge is dropped + (zero-edge over a guess), same discipline as the qualified-drop path.""" + from graphify.extract import extract + (tmp_path / "a.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "b.pm").write_text("package P::B;\nsub emit { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nuse P::B;\nsub run { emit(); }\n1;\n" + ) + r = extract( + [tmp_path / "a.pm", tmp_path / "b.pm", tmp_path / "c.pm"], parallel=False + ) + calls = _calls(r) + assert not any("emit" in t for _s, t in calls), \ + f"bare emit() imported from two packages is ambiguous and must drop, got {calls}" + + +# In-corpus import re-pointer: a `use`/`require` whose module is a package defined +# elsewhere in the corpus must re-point its imports edge from the bare module-label +# id onto the real package node; external modules stay dangling. + +def test_perl_import_repoints_to_in_corpus_package(tmp_path): + """`use Acme::Helper;` from another file must re-point its imports edge onto + the real Acme::Helper package node (id `_make_id(stem, 'Acme::Helper')`), + not dangle on the bare module-label id `_make_id('Acme::Helper')`. An external + `use POSIX;` has no in-corpus package, so its edge keeps the bare (node-less) + target — mirroring how other languages leave external imports dangling.""" + from graphify.extract import extract + from graphify.extractors.base import _make_id + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + (tmp_path / "main.pm").write_text( + "package Main;\nuse Acme::Helper;\nuse POSIX qw(floor);\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", tmp_path / "main.pm"], parallel=False) + node_ids = {n["id"] for n in r["nodes"]} + pkg_nodes = [n for n in r["nodes"] if n.get("label") == "Acme::Helper"] + assert len(pkg_nodes) == 1, f"expected one Acme::Helper package node, got {pkg_nodes}" + pkg_id = pkg_nodes[0]["id"] + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"use Acme::Helper must re-point to package node {pkg_id}, got {[e['target'] for e in imports]}" + bare_helper = _make_id("Acme::Helper") + assert bare_helper != pkg_id + assert not any(e["target"] == bare_helper for e in imports), \ + "the in-corpus import must no longer dangle on the bare module-label id" + posix_id = _make_id("POSIX") + assert any(e["target"] == posix_id for e in imports), "external POSIX import edge missing" + assert posix_id not in node_ids, \ + "external POSIX must stay a dangling label-id stub, not become a node" + + +def test_perl_require_repoints_to_in_corpus_package(tmp_path): + """`require Acme::Helper;` (bareword require form) re-points the same way as + `use` when the module is an in-corpus package.""" + from graphify.extract import extract + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + (tmp_path / "main.pm").write_text( + "package Main;\nrequire Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", tmp_path / "main.pm"], parallel=False) + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"require Acme::Helper must re-point to package node {pkg_id}, got {[e['target'] for e in imports]}" + + +def test_perl_import_duplicate_package_label_stays_dangling(tmp_path): + """A re-opened package declared under the SAME fully-qualified label in two + files (e.g. `package Assert;` in both AssertOn.pm and AssertOff.pm) is + ambiguous: a bare `use P::A;` cannot say which file it means. Binding it to + one file arbitrarily is a guessed edge — worse than dangling — so the imports + edge must stay on the bare module-label id (no package node) when >1 package + node carries the label. Only a unique label re-points (zero-edge over a guess).""" + from graphify.extract import extract + from graphify.extractors.base import _make_id + (tmp_path / "a1.pm").write_text("package P::A;\nsub emit { return 1; }\n1;\n") + (tmp_path / "a2.pm").write_text("package P::A;\nsub other { return 2; }\n1;\n") + (tmp_path / "c.pm").write_text( + "package C;\nuse P::A;\nsub run { return 1; }\n1;\n" + ) + r = extract( + [tmp_path / "a1.pm", tmp_path / "a2.pm", tmp_path / "c.pm"], parallel=False + ) + pkg_nodes = [n for n in r["nodes"] if n.get("label") == "P::A"] + assert len(pkg_nodes) == 2, f"expected two P::A package nodes, got {pkg_nodes}" + pkg_ids = {n["id"] for n in pkg_nodes} + imports = [e for e in r["edges"] if e["relation"] == "imports"] + bare_id = _make_id("P::A") + assert any(e["target"] == bare_id for e in imports), \ + f"ambiguous use P::A; must stay dangling on bare id {bare_id}, got {[e['target'] for e in imports]}" + assert not any(e["target"] in pkg_ids for e in imports), \ + f"ambiguous use P::A; must NOT bind to either P::A file node, got {[e['target'] for e in imports]}" + + +def test_perl_import_repoints_from_shebang_perl_file(tmp_path): + """An extensionless `#!/usr/bin/perl` script is dispatched to extract_perl by + shebang, but its source_file has no .pl/.pm suffix. Scoping the re-pointer by + file suffix would silently skip such a file's imports (underreporting); scoping + by extractor provenance re-points them like any other Perl source.""" + from graphify.extract import extract + (tmp_path / "helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + script = tmp_path / "runme" + script.write_text( + "#!/usr/bin/perl\npackage Main;\nuse Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + r = extract([tmp_path / "helper.pm", script], parallel=False) + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + imports = [e for e in r["edges"] if e["relation"] == "imports"] + assert any(e["target"] == pkg_id for e in imports), \ + f"use Acme::Helper in a shebang-perl file must re-point to {pkg_id}, got {[e['target'] for e in imports]}" + + +# ── Perl S6c review-finding regressions ──────────────────────────────────────── + +def test_perl_raw_calls_stamped_lang_perl(): + """Every Perl raw_call carries `lang="perl"` so the shared second pass claims + exactly Perl's calls via the extractor stamp (like cpp/csharp/java/objc), + rather than sniffing for a `*_package` field (A1).""" + from graphify.extract import extract_perl + r = extract_perl(FIXTURES / "sample_module.pm") + assert r["raw_calls"], "expected raw_calls from the sample module" + assert all(rc.get("lang") == "perl" for rc in r["raw_calls"]), \ + f"every perl raw_call must be lang=perl, got {[rc.get('lang') for rc in r['raw_calls']]}" + + +def test_perl_main_qualified_call_binds_packageless_sub(tmp_path): + """A package-less sub lives in Perl's default `main` package; a qualified + `main::helper()` call must bind to it (R1). Before main was modeled the sub + hung off the file node with the filename as its package label, so `main::` + never resolved.""" + from graphify.extract import extract + (tmp_path / "s.pl").write_text( + "sub helper { return 1; }\nsub run { main::helper(); }\n" + ) + r = extract([tmp_path / "s.pl"], parallel=False) + calls = _calls(r) + assert any("run" in s and "helper" in t for s, t in calls), \ + f"main::helper() must bind to the package-less (main) helper, got {calls}" + + +def test_perl_packageless_bare_call_no_bind_to_foreign_package(tmp_path): + """A bare call from a package-less (main) file must NOT bind to a same-named + sub in an unrelated, un-imported package (R1). Modeling `main` scopes the call + to main; without it the call carried no package and was generically matched to + the lone same-named sub in a foreign package.""" + from graphify.extract import extract + (tmp_path / "b.pm").write_text("package B;\nsub helper { return 1; }\n1;\n") + (tmp_path / "s.pl").write_text( + "sub other { return 2; }\nsub run { helper(); other(); }\n" + ) + r = extract([tmp_path / "b.pm", tmp_path / "s.pl"], parallel=False) + calls = _calls(r) + # Positive guard: the same-file (main) bare call resolves, so the negative + # assertion below is not a vacuous pass on an empty graph. + assert any("run" in s and "other" in t for s, t in calls), \ + f"expected run->other within main, got {calls}" + assert not any("helper" in t for _s, t in calls), \ + f"bare helper() from a main file must not bind to B::helper, got {calls}" + + +def test_perl_main_package_node_emitted_for_packageless_sub(tmp_path): + """The lazily-created `main` package node contains the package-less sub (the + `contains` edge is main -> sub, not file -> sub).""" + from graphify.extract import extract_perl + src = tmp_path / "s.pl" + src.write_text("sub helper { return 1; }\n") + r = extract_perl(src) + label = {n["id"]: n.get("label", "") for n in r["nodes"]} + container_of = { + label.get(e["target"]): label.get(e["source"]) + for e in r["edges"] if e["relation"] == "contains" + } + assert "main" in label.values(), "a package-less sub must materialize a main package node" + assert container_of.get("helper()") == "main", \ + f"package-less helper must be contained by main, got {container_of.get('helper()')!r}" + + +def test_perl_import_repoint_survives_cache_roundtrip(tmp_path, monkeypatch): + """The import re-pointer must still fire on a cached (second) run (R2). A cached + fragment's source_file returns absolutized against the resolved root, so exact + string matching against the raw (relative) provenance paths missed and the + re-pointer silently regressed to dangling. Matching on the resolved path makes + both runs produce the same re-pointed edge.""" + from graphify.extract import extract + monkeypatch.chdir(tmp_path) + Path("helper.pm").write_text("package Acme::Helper;\nsub emit { return 1; }\n1;\n") + Path("main.pm").write_text( + "package Main;\nuse Acme::Helper;\nsub run { return 1; }\n1;\n" + ) + paths = [Path("helper.pm"), Path("main.pm")] + + def _import_targets(r): + pkg_id = next(n["id"] for n in r["nodes"] if n.get("label") == "Acme::Helper") + return pkg_id, {e["target"] for e in r["edges"] if e["relation"] == "imports"} + + r1 = extract(paths, cache_root=Path("."), parallel=False) + pkg1, targets1 = _import_targets(r1) + assert pkg1 in targets1, f"first run must re-point onto {pkg1}, got {targets1}" + assert (tmp_path / "graphify-out" / "cache").exists(), "AST cache must be written" + + r2 = extract(paths, cache_root=Path("."), parallel=False) + pkg2, targets2 = _import_targets(r2) + assert pkg2 in targets2, \ + f"cached run must ALSO re-point onto {pkg2} (not regress to dangling), got {targets2}" + assert targets1 == targets2, \ + f"import targets must be identical across fresh and cached runs: {targets1} vs {targets2}" + + +def test_perl_deeply_nested_expression_does_not_crash(tmp_path): + """A pathologically deep expression nest must not RecursionError (which would + make _safe_extract drop the whole file); the iterative walk keeps the file node + and the sub (S1).""" + from graphify.extract import extract_perl + depth = 12000 # > _RECURSION_LIMIT (10_000): the old recursive walk would blow up + body = "[" * depth + "1" + "]" * depth + src = tmp_path / "deep.pl" + src.write_text(f"sub f {{ my $x = {body}; }}\n") + r = extract_perl(src) + assert not r.get("error"), f"deep nest must not error out, got {r.get('error')!r}" + labels = {n["label"] for n in r["nodes"]} + assert "deep.pl" in labels, "file node must survive deep nesting" + assert "f()" in labels, "the sub must still be extracted despite deep nesting" + + +def test_perl_isa_rejects_non_package_string(tmp_path): + """@ISA holds arbitrary string content; a value that is not a well-formed + package name (markdown/brackets/control chars) must not become an inherits + stub node or edge (S2, zero-edge over a bogus label).""" + from graphify.extract import extract_perl + src = tmp_path / "bad.pm" + src.write_text( + 'package Child;\n' + 'our @ISA = ("weird\\nname[x](y)");\n' + '1;\n' + ) + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert not inherits, f"malformed @ISA string must not create an inherits edge, got {inherits}" + bad = [n for n in r["nodes"] + if any(ch in n.get("label", "") for ch in ("[", "(", "\n"))] + assert not bad, f"no stub node from a malformed inheritance string, got {bad}" + + +def test_perl_isa_valid_package_still_inherits(tmp_path): + """The S2 validation must not reject legitimate package names: a normal + `Foo::Bar` @ISA entry still produces an inherits edge + stub.""" + from graphify.extract import extract_perl + src = tmp_path / "ok.pm" + src.write_text( + 'package Child;\n' + 'our @ISA = ("Acme::Base");\n' + '1;\n' + ) + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert inherits, "a well-formed @ISA package must still inherit" + labels = {n.get("label") for n in r["nodes"]} + assert "Acme::Base" in labels, "the valid base class stub must be emitted" + + +@pytest.mark.parametrize("bad", [ + "Acme::Basé", # accented letter — Unicode \w matched, ASCII must not + "Acme::Ｂase", # fullwidth latin letter — Unicode \w matched, ASCII must not + "Acme::1x", # component starting with a digit after :: + "1Acme", # leading digit + "Acme::Bar\n", # trailing newline (a bare $ anchor would let this through) +]) +def test_perl_label_validator_rejects_non_ascii_component(bad): + """The package-name validator is ASCII-only and anchors every `::` component to + a letter/underscore start (F3). Python `\\w` is Unicode-default, so accented, + fullwidth and digit-start names would otherwise pass and land in graph.json / + the Obsidian export.""" + from graphify.extractors.perl import _is_valid_perl_package_name + assert not _is_valid_perl_package_name(bad), \ + f"{bad!r} must be rejected by the ASCII package-name validator" + + +@pytest.mark.parametrize("ok", ["Foo", "_priv", "Acme::Base", "A::B::C", "F0o::B4r"]) +def test_perl_label_validator_accepts_ascii_package(ok): + """Legitimate ASCII package names (including digits after the first char of a + component) still validate — the F3 tightening must not over-reject.""" + from graphify.extractors.perl import _is_valid_perl_package_name + assert _is_valid_perl_package_name(ok), f"{ok!r} must validate" + + +def test_perl_isa_digit_start_component_no_stub(tmp_path): + """End-to-end: a @ISA entry whose ::-component starts with a digit must not + produce an inherits edge or a stub node (F3 Unicode/digit-start bypass).""" + from graphify.extract import extract_perl + src = tmp_path / "bad.pm" + src.write_text('package Child;\nour @ISA = ("Acme::1Base");\n1;\n') + r = extract_perl(src) + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert not inherits, f"digit-start component must not inherit, got {inherits}" + assert "Acme::1Base" not in {n.get("label") for n in r["nodes"]}, \ + "no stub node for a digit-start component" + + +def test_perl_statement_walk_charges_budget_per_sibling(tmp_path, monkeypatch): + """A broad, flat file (many sibling statements under one frame) must charge the + traversal budget per sibling, not once per stack frame (F2). Under a tiny budget + the walk stops early and emits a PARTIAL graph; the old code drained every + sibling on one charge, so all subs surfaced regardless of budget. Asserting the + partial output isolates walk_statements (the shared budget is also spent by the + later call walk, so a warning alone would not discriminate).""" + from graphify.extractors import perl + monkeypatch.setattr(perl, "_MAX_PERL_TRAVERSAL_NODES", 3) + src = tmp_path / "flat.pl" + src.write_text("package Wide;\n" + "".join( + f"sub s{i} {{ 1; }}\n" for i in range(30))) + r = perl.extract_perl(src) + sub_nodes = [n for n in r["nodes"] if n.get("label", "").endswith("()")] + assert len(sub_nodes) < 30, ( + "walk_statements must charge per sibling and stop early under a tiny budget; " + f"emitting all {len(sub_nodes)} subs means siblings were traversed for free") + + +def test_perl_statement_walk_no_warning_within_budget(tmp_path, caplog): + """Guard against a vacuous pass: a small file under the default budget extracts + fully and emits no traversal warning.""" + import logging + from graphify.extractors import perl + src = tmp_path / "small.pl" + src.write_text("package Wide;\n" + "".join( + f"sub s{i} {{ return {i}; }}\n" for i in range(20))) + with caplog.at_level(logging.WARNING, logger="graphify.extractors.perl"): + r = perl.extract_perl(src) + assert not any("traversal budget" in rec.message for rec in caplog.records), \ + "a small file must not trip the budget" + assert sum(1 for n in r["nodes"] if n.get("label", "").endswith("()")) == 20, \ + "all 20 subs extract when the budget is ample" + + +def test_perl_string_parents_charges_budget(tmp_path, monkeypatch, caplog): + """`_string_parents` shares the traversal budget (F2): an @ISA array of many + separate string literals is a per-node walk that must trip the bounded warning, + even though the file has only a couple of top-level statements. The old code + left `_string_parents` entirely unbudgeted, so it walked the whole subtree for + free. (Each literal is its own tree node — unlike a single qw() word list.)""" + import logging + from graphify.extractors import perl + # Above the ~2 top-level statement charges but far below the string-literal node + # count, so exhaustion can only come from _string_parents. + monkeypatch.setattr(perl, "_MAX_PERL_TRAVERSAL_NODES", 20) + parents = ", ".join(f"'P{i}'" for i in range(400)) + src = tmp_path / "wide_isa.pm" + src.write_text(f"package Child;\nour @ISA = ({parents});\n1;\n") + with caplog.at_level(logging.WARNING, logger="graphify.extractors.perl"): + perl.extract_perl(src) + assert any("traversal budget exhausted" in rec.message for rec in caplog.records), \ + "a wide @ISA literal array must charge the shared budget via _string_parents" diff --git a/uv.lock b/uv.lock index 088ebbbdc..c1b8de73e 100644 --- a/uv.lock +++ b/uv.lock @@ -1090,7 +1090,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.9.6" +version = "0.9.12" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -1114,6 +1114,7 @@ dependencies = [ { name = "tree-sitter-kotlin" }, { name = "tree-sitter-lua" }, { name = "tree-sitter-objc" }, + { name = "tree-sitter-perl" }, { name = "tree-sitter-php" }, { name = "tree-sitter-powershell" }, { name = "tree-sitter-python" }, @@ -1314,6 +1315,7 @@ requires-dist = [ { name = "tree-sitter-objc", specifier = ">=3.0,<4.0" }, { name = "tree-sitter-pascal", marker = "extra == 'all'" }, { name = "tree-sitter-pascal", marker = "extra == 'pascal'" }, + { name = "tree-sitter-perl", specifier = ">=1.2.0,<2.0" }, { name = "tree-sitter-php", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-powershell", specifier = ">=0.26,<0.28" }, { name = "tree-sitter-python", specifier = ">=0.23,<0.26" }, @@ -4754,6 +4756,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/ad/e5381365ed5b247cad2e047682b081ccbb79f84bb5c1b48642ef3385f1d3/tree_sitter_pascal-0.11.0-cp38-abi3-win_arm64.whl", hash = "sha256:9d22b974a47765f1c98629338d82d1df89c35c270a73f08dda8fbd49538e32d4", size = 124914, upload-time = "2026-07-01T03:53:15.191Z" }, ] +[[package]] +name = "tree-sitter-perl" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/44/c211fda4f42c01791391928b23192b50d160b14e676f2c1fcce1850d76d9/tree_sitter_perl-1.2.1.tar.gz", hash = "sha256:7b5ce3fe9cd7d23b1619ed49d5539b106e301b68dd7139bcae6b9d4ad1e3a1d3", size = 1749866, upload-time = "2026-06-30T18:53:23.646Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/29/4cdb0da188ce7864507317e70db6c9dc045f41d6250f0865aef03294eeb2/tree_sitter_perl-1.2.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5e77a13168ae5699ee9942aaa928920b9a2d8ce051934550ba2a4b547833a8ba", size = 347325, upload-time = "2026-06-30T18:53:14.467Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b3/7489ac765c9d5e2a8ef75f43caaa5f1df3cbc93bbc5c0f49559bc4c2e280/tree_sitter_perl-1.2.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a18f09b6b76d9bb52da49f3bb4ed7fe4db0857fbf21d1aad78021abaddb47d72", size = 408980, upload-time = "2026-06-30T18:53:16.211Z" }, + { url = "https://files.pythonhosted.org/packages/a3/50/7340015653da77bc5d12e45678c9339cabf389aba7404d3f5e0514e04acc/tree_sitter_perl-1.2.1-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6cd149c5f5fb5f82d337d5764d8626eecb03b00604c8c0c7ffc4e165e26b273", size = 385181, upload-time = "2026-06-30T18:53:17.741Z" }, + { url = "https://files.pythonhosted.org/packages/8b/77/76486f28c0d837d1424d3a314736ca2b6916a56b5adf073d6e038fb03522/tree_sitter_perl-1.2.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:66361e2d61467843aeb9ef1233a461e2880ed294a368d81e4b898b33c76913fe", size = 370909, upload-time = "2026-06-30T18:53:19.223Z" }, + { url = "https://files.pythonhosted.org/packages/7f/cd/6fbecae40fb5f78bcb95432fc41d6eb0b6a246e43bb63237544c92aa271a/tree_sitter_perl-1.2.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0cfd166f0f11de834b96d99432a86b54444ccbd270d9e19eed06c2de9d6f932e", size = 373213, upload-time = "2026-06-30T18:53:20.769Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/a17bd77d350cc7344baf6f3f568f297a1d1a88d2afbf3024a8e86618e96f/tree_sitter_perl-1.2.1-cp39-abi3-win_amd64.whl", hash = "sha256:168704111ed36461b1d1da1a36b16332e2c8e2693b5daec57bd8abada1e02f58", size = 320105, upload-time = "2026-06-30T18:53:22.14Z" }, +] + [[package]] name = "tree-sitter-php" version = "0.24.1"