From 557d4df26a80da5fd5135b29fd3c0d01b1ee21f5 Mon Sep 17 00:00:00 2001 From: MasterKinjalk Date: Mon, 15 Jun 2026 20:52:46 -0500 Subject: [PATCH] fix(extract): skip Python primitive/library type-annotation nodes Type annotations (return_type, parameter_type, field, generic_arg, base class) materialize language primitives and common library types -- str, int, Tensor, ndarray, Path, Module, np, torch -- as graph nodes. They carry no domain meaning and become spurious high-degree hubs that pollute community clustering and clutter the report's node list. _LANGUAGE_BUILTIN_GLOBALS already filtered these at call sites but not at type-reference sites, which route through ensure_named_node. What this does - Add _PYTHON_TYPE_REF_PRIMITIVES (builtin globals + stdlib/typing/NumPy/ PyTorch annotation types) and gate it per language via _TYPE_REF_PRIMITIVES_BY_LANG / _type_ref_primitives(ts_module). - _extract_generic skips node creation for these names only when the language is Python; the id is still returned so reference edges resolve (dangling-to-external, already tolerated by build.py). - Add a regression test per affected extractor (Python/Rust/Julia/Go): primitive dropped, a real same-named user type kept, edge still resolves. Why it is scoped to Python (review #1332) The first cut applied one flat primitive set in every extractor. That over-filters: the same bare names are ordinary user-defined types in other languages -- a Rust `Response`, a Julia `Module`, a Go `Set` -- and a name-only membership test silently dropped them. Same-file definitions were never at risk (ensure_named_node returns the stem-qualified id for in-file symbols; only the bare cross-file/external id is gated), but the foreign-name application was still wrong, so it is removed. Why Go is not guarded, and the hand-written walkers are reverted - Go already filters its predeclared types at the collector (_GO_PREDECLARED_TYPES in _go_collect_type_refs), so it never materializes int/string/error as nodes; adding the flat guard would only over-filter Go user types. Rust likewise skips `primitive_type` nodes and Python scalars are dropped by _PYTHON_ANNOTATION_NOISE (#1147). Predeclared-type filtering belongs at the per-language collector, not a global membership test. - The Julia/Fortran/Rust/PowerShell/Obj-C walkers therefore revert to a plain `if nid not in seen_ids`. Their extractor tests intentionally assert primitive type refs resolve as nodes (PowerShell `string`/`void`, Julia `Float64`, Obj-C `NSString`), so filtering them there would both regress those tests and contradict established behaviour. Net effect: the noise-hub fix lands for Python (the motivating case) without mis-dropping real types in any other language. Rebased onto v8. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_015pQL3KWPSPtYQZj22wK95m --- graphify/extract.py | 50 +++++++++- tests/test_type_ref_primitives.py | 156 ++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 tests/test_type_ref_primitives.py diff --git a/graphify/extract.py b/graphify/extract.py index 008c150b0..ba1bab058 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -43,6 +43,47 @@ }) +# Python primitive / library type names that leak in as nodes purely via type +# annotations (return_type, parameter_type, field, generic_arg, base class) +# rather than as real domain concepts. Skipping node creation for these keeps the +# graph clean; the leftover reference edges are the dangling-to-external kind +# build.py already tolerates. Reuses _LANGUAGE_BUILTIN_GLOBALS (the shared +# call-site builtin filter) and layers stdlib / NumPy / PyTorch annotation types. +# +# Deliberately PYTHON-ONLY: these bare names are ordinary user-defined types in +# other languages (a Rust `Response`, a Julia `Module`, a Go `Set`), so applying +# one flat list to every extractor mis-drops real types (review #1332). Languages +# that suppress their own primitives already do so at the collector, scoped to the +# language — Go via _GO_PREDECLARED_TYPES, Rust by skipping `primitive_type` +# nodes, Python scalars via _PYTHON_ANNOTATION_NOISE (#1147) — so this set only +# layers the library/annotation type names on top, for Python. +_PYTHON_TYPE_REF_PRIMITIVES: frozenset[str] = _LANGUAGE_BUILTIN_GLOBALS | frozenset({ + # Python typing / stdlib annotation types + "Any", "Optional", "Union", "List", "Dict", "Set", "Tuple", "Sequence", + "Iterable", "Iterator", "Mapping", "Callable", "Type", "Literal", "Final", + "Annotated", "Path", "PathLike", "datetime", "timedelta", "Decimal", + "frozenset", "bytearray", "complex", "object", "None", "NoneType", + # NumPy + "ndarray", "np", "dtype", "NDArray", "ArrayLike", + # PyTorch + "Tensor", "Module", "torch", "device", "DataLoader", "Dataset", + "nn", "Parameter", +}) + + +# Type-reference primitive filter keyed by tree-sitter module. Only languages with +# an entry suppress annotation-only type names in ensure_named_node; every other +# extractor keeps them (or filters its own predeclared types at the collector). +_TYPE_REF_PRIMITIVES_BY_LANG: dict[str, frozenset[str]] = { + "tree_sitter_python": _PYTHON_TYPE_REF_PRIMITIVES, +} + + +def _type_ref_primitives(ts_module: str) -> frozenset[str]: + """Primitive/library type-ref names to skip as nodes for the given language.""" + return _TYPE_REF_PRIMITIVES_BY_LANG.get(ts_module, frozenset()) + + def _raise_recursion_limit() -> None: if sys.getrecursionlimit() < _RECURSION_LIMIT: sys.setrecursionlimit(_RECURSION_LIMIT) @@ -2459,12 +2500,19 @@ def add_edge(src: str, tgt: str, relation: str, line: int, edge["context"] = context edges.append(edge) + # Annotation-only primitive/library type names to skip as nodes, scoped to + # this language (empty for everything but Python — see _type_ref_primitives). + type_ref_primitives = _type_ref_primitives(config.ts_module) + def ensure_named_node(name: str, line: int) -> str: nid = _make_id(stem, name) if nid in seen_ids: return nid nid = _make_id(name) - if nid not in seen_ids: + # Skip node creation for primitive / library type names — they pollute the + # graph as noise hubs. Return the id so reference edges still resolve + # (dangling-to-external, tolerated by build.py); just don't materialize the node. + if nid not in seen_ids and name not in type_ref_primitives: add_node(nid, name, line) return nid diff --git a/tests/test_type_ref_primitives.py b/tests/test_type_ref_primitives.py new file mode 100644 index 000000000..898a9735a --- /dev/null +++ b/tests/test_type_ref_primitives.py @@ -0,0 +1,156 @@ +"""Regression tests for type-reference primitive filtering (PR #1332 review). + +Behaviour contract: + * Python primitive / library type names that appear only in annotations + (return_type / parameter_type / field / generic_arg / base) are NOT + materialized as graph nodes — they are noise hubs, not domain concepts. + * A real, user-defined type whose NAME collides with a primitive/library type + in some *other* language must still be materialized, and its edges resolve. + * The filter is scoped per language: a name that is a primitive in Python + (`Tensor`, `Module`) is a legitimate user-defined type name in Rust/Julia/Go, + so a single flat global list applied to every extractor mis-drops real types + (the over-filtering #1332 review flagged). Languages that already filter + their own predeclared types at the collector (Go `_GO_PREDECLARED_TYPES`, + Rust `primitive_type`, Python `_PYTHON_ANNOTATION_NOISE`) keep doing so. +""" +from __future__ import annotations + +from graphify.extract import extract_python, extract_go, extract_rust, extract_julia + + +def _labels(r) -> list[str]: + return [n["label"] for n in r["nodes"]] + + +def _norm(label: str) -> str: + return label.strip("()").lstrip(".") + + +def _edge_pairs(r, relation, context=None) -> set[tuple[str, str]]: + """(normalized source label, normalized target label) for matching edges. + + The target label resolves from the node list, so a *suppressed* target + degrades to its raw id — an assertion of ``(src, "Type")`` therefore only + holds when the target node was actually materialized. + """ + id_to_label = {n["id"]: _norm(n["label"]) for n in r["nodes"]} + pairs = set() + for e in r["edges"]: + if e.get("relation") != relation: + continue + if context is not None and e.get("context") != context: + continue + pairs.add((id_to_label.get(e["source"], e["source"]), + id_to_label.get(e["target"], e["target"]))) + return pairs + + +# ── Python: primitives dropped, real same-named user types kept ─────────────── + +def test_python_primitive_type_refs_not_materialized(tmp_path): + p = tmp_path / "net.py" + p.write_text( + "from torch import Tensor, nn\n" + "from numpy import ndarray\n" + "class Net(nn.Module):\n" + " def forward(self, x: Tensor) -> Tensor:\n" + " return x\n" + " def to_np(self) -> ndarray:\n" + " return x\n" + "def count() -> int:\n" + " return 0\n", + encoding="utf-8", + ) + labels = _labels(extract_python(p)) + assert "Net" in labels + assert any("forward" in l for l in labels) + assert any("count" in l for l in labels) + for noise in ("Tensor", "ndarray", "int", "Module", "nn"): + assert noise not in labels, f"{noise!r} leaked as a node" + + +def test_python_user_type_named_like_primitive_is_kept(tmp_path): + """A user class literally named like a Python/PyTorch type (`Dataset`) must + stay a node and keep its inheritance edge.""" + p = tmp_path / "model.py" + p.write_text( + "class Dataset:\n" + " pass\n" + "class ImageData(Dataset):\n" + " pass\n" + "def make() -> Dataset:\n" + " return Dataset()\n", + encoding="utf-8", + ) + r = extract_python(p) + assert "Dataset" in _labels(r), "real user type Dataset was dropped" + assert ("ImageData", "Dataset") in _edge_pairs(r, "inherits") + + +# ── Cross-language over-filtering: foreign primitive names must NOT filter ───── + +def test_rust_type_named_like_js_global_not_overfiltered(tmp_path): + """`Response` is a JS built-in but an ordinary Rust type. A flat global + primitive list mis-dropped it from Rust type references (#1332).""" + p = tmp_path / "handler.rs" + p.write_text("fn handle() -> Response {\n todo!()\n}\n", encoding="utf-8") + r = extract_rust(p) + assert "Response" in _labels(r), "Rust `Response` over-filtered as a node" + assert ("handle", "Response") in _edge_pairs(r, "references", "return_type") + + +def test_rust_keeps_user_type_and_still_filters_native_primitive(tmp_path): + p = tmp_path / "lib.rs" + p.write_text( + "struct Module {\n id: u32,\n}\nfn build() -> Module {\n todo!()\n}\n", + encoding="utf-8", + ) + r = extract_rust(p) + labels = _labels(r) + assert "Module" in labels + assert ("build", "Module") in _edge_pairs(r, "references", "return_type") + # u32 is a genuine Rust primitive, filtered by the Rust collector, not a node. + assert "u32" not in labels + + +def test_julia_struct_named_like_foreign_primitive_is_kept(tmp_path): + """A Julia struct named `Dataset`/`Module` must not be dropped by a + Python/PyTorch-flavoured primitive list.""" + p = tmp_path / "types.jl" + p.write_text( + "struct Dataset\n n::Int\nend\n\nstruct Wrapper\n d::Dataset\nend\n", + encoding="utf-8", + ) + labels = _labels(extract_julia(p)) + assert "Dataset" in labels, "Julia user struct Dataset was over-filtered" + assert "Wrapper" in labels + + +# ── Go: already correct via _GO_PREDECLARED_TYPES (comment #1) ───────────────── + +def test_go_primitive_type_refs_not_materialized(tmp_path): + p = tmp_path / "svc.go" + p.write_text( + "package svc\n" + "type Service struct{}\n" + "func (s *Service) Save() error { return nil }\n" + "func (s *Service) Count() int { return 0 }\n" + "func (s *Service) Name() string { return \"\" }\n", + encoding="utf-8", + ) + labels = _labels(extract_go(p)) + assert "Service" in labels + for noise in ("error", "int", "string"): + assert noise not in labels, f"Go primitive {noise!r} leaked as a node" + + +def test_go_user_type_named_like_primitive_is_kept(tmp_path): + """`Set` collides with Python's typing.Set, but is an ordinary Go type.""" + p = tmp_path / "container.go" + p.write_text( + "package container\ntype Set struct{}\nfunc New() Set { return Set{} }\n", + encoding="utf-8", + ) + r = extract_go(p) + assert "Set" in _labels(r), "Go user type Set was dropped" + assert ("New", "Set") in _edge_pairs(r, "references", "return_type")