From 0c63d0139c490b86cb53060f1c89c7fb1efefe08 Mon Sep 17 00:00:00 2001 From: SinghAman21 Date: Thu, 16 Jul 2026 23:25:29 +0530 Subject: [PATCH] fix(extract): anchor source_file on the scan root, not the --out dir (#1941) `graphify extract --out ` reduced every node's `source_file` to a bare filename, so a graph.json could no longer be resolved back to files on disk by joining source_file onto the scan root. --out passes the output dir as cache_root to relocate the cache, but that value also anchored relativization. Every scanned file then failed relative_to(root), fell through to the #1899 out-of-root fallback, tripped its `updepth > 3` walk-up guard -- written for a stray ProjectReference, not a whole corpus -- and collapsed to a basename. On Windows an --out on another drive hit the cross-drive branch and basenamed unconditionally, which is what the reporter saw: 0 of ~120k source_files kept a separator. The directory survived only in the node id slug, lossily (`.`, `/`, `\`, `-`, spaces all map to `_`), and no other field carried it -- origin_file is stripped (#1516) and the export has no file table -- so 0% of nodes resolved. extract() now takes an explicit `root` anchor for source_file/ids/symbol resolution, which the CLI pins to the scan root independent of where the cache lives. This completes the cache/anchor decoupling #1774 started and matches build(root=target), which already anchored on the scan root -- extract was the lone component keying off --out. cache_root keeps its fallback-anchor role, so callers that pass the scan root as cache_root (watch, the no---out CLI path, tests) are unchanged, as are cache location and out-of-root portability (#1899). --- CHANGELOG.md | 1 + graphify/cli.py | 12 ++++++++---- graphify/extract.py | 15 ++++++++++++++- tests/test_extract.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84d865a6a..7c1fa0cd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.17 (unreleased) +- Fix: `graphify extract --out ` no longer reduces every node's `source_file` to a bare filename, so a `graph.json` stays resolvable against its scan root (#1941, thanks @JensD-git). `--out` passes the output dir as `cache_root` to relocate the cache, but that value also anchored relativization — so every scanned file failed `relative_to(root)`, fell through to the #1899 out-of-root fallback, tripped its `updepth > 3` walk-up guard (meant for a stray `ProjectReference`), and collapsed to a basename; on Windows an `--out` on another drive hit the cross-drive branch and basenamed unconditionally. With the directory gone from the only field that carried it (the `id` slug is lossy — `.`, `/`, `\`, `-` and spaces all map to `_`), no downstream tool could resolve a node back to its file. `extract()` now takes an explicit `root` anchor that the CLI pins to the scan root, independent of where the cache lives (completing the #1774 cache/anchor decoupling and matching `build(root=target)`, which already anchored on the scan root). Cache location, out-of-root portability (#1899), and the no---out/watch paths are unchanged; existing graphs re-resolve on the next extract. - Fix: a missing `manifest.json` no longer degrades `graphify extract --code-only` into a full scan that discards the committed semantic layer (#1925). On a fresh clone (or when the manifest is deliberately untracked because its mtimes churn), the incremental gate required both `manifest.json` and `graph.json`; with only the graph present it fell to a full scan, and under `--code-only` that dropped every doc/paper/image node — silently replacing a curated graph with an AST-only skeleton. An existing `graph.json` is now a sufficient incremental baseline: `detect_incremental` already treats an absent manifest as "everything new / nothing deleted", so `build_merge` + `_stale_graph_sources` preserve files that are merely out of this run's scope while still evicting genuinely deleted sources. - Fix: hyperedge-only documents are now stamped in the manifest instead of being re-extracted on every run (#1920). `_stamped_manifest_files` (#1897) decided a semantic doc "produced output" by inspecting only `nodes` and `edges`, never `hyperedges`, so a chunk whose only output for a doc was a hyperedge (3+ nodes sharing a concept) left that doc unstamped and perpetually re-queued. Stamping now counts hyperedge output too, mirroring the per-`source_file` keying the semantic cache already uses. - Fix: the PHP extractor now disambiguates same-named classes across namespaces (#1923). A bare class reference (`extends Page`) collapsed onto the only internal class named `Page`, so `App\Models\Page` and an imported `Filament\Pages\Page` fused into one node, manufacturing a false `inherits`/`imports` edge and a bogus cross-community bridge. A new namespace/`use`-aware resolution pass (mirroring the Java resolver, running before the unique-name rewire) re-points supertype/import references to the real definition, or parks provably-external ones on a fully-qualified stub the bare-name rewire cannot collapse. Plain, non-namespaced PHP is unchanged. diff --git a/graphify/cli.py b/graphify/cli.py index d817401ac..69b3d378d 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -125,9 +125,11 @@ def _stale_graph_sources( (--include sources, symlinked external corpora) are never walked by detect, so their absence from the corpus is not staleness evidence. Relative entries are re-anchored against both the scan root and the - graph's own output root (``--out`` extracts store source_files relative - to the OUT root, e.g. ``../project/x.py``, #555/#1899); only anchors - that land inside the scan root count. + graph's own output root; only anchors that land inside the scan root + count. Since #1941 extracts always store source_file relative to the SCAN + root, so the scan-root anchor is the live one; the out-root anchor stays + for graphs written by <=0.9.16, which stored them relative to the OUT root + (e.g. ``../project/x.py``, #555/#1899). ``seen_files`` must be the FULL detect output including unclassified files, so nodes from walked-but-unsupported sources (e.g. introspected Cargo.toml manifests) are not misread as stale. @@ -2527,7 +2529,9 @@ def _parse_float(name: str, raw: str) -> float: # Anchor the cache at the output root, not the scanned project: # with --out, a /graphify-out/cache/ would leak a # graphify-out/ dir into a project that asked for external output. - ast_kwargs: dict = {"cache_root": out_root} + # `root` stays the scanned project so source_file/ids relativize + # against it; conflating the two basenamed every node (#1941). + ast_kwargs: dict = {"cache_root": out_root, "root": target} if cli_max_workers is not None: ast_kwargs["max_workers"] = cli_max_workers print(f"[graphify extract] AST extraction on {len(code_files)} code files...") diff --git a/graphify/extract.py b/graphify/extract.py index 2ff5411cf..844bb6875 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4293,6 +4293,7 @@ def extract( paths: list[Path], cache_root: Path | None = None, *, + root: Path | None = None, parallel: bool = True, max_workers: int | None = None, ) -> dict: @@ -4305,15 +4306,21 @@ def extract( Args: paths: files to extract from + root: explicit anchor for source_file relativization, node ids, and + symbol resolution. Pass the SCAN root whenever the cache lives + somewhere else (`--out`); without it the anchor falls back to + cache_root and every scanned file reads as out-of-root (#1941). cache_root: explicit root for graphify-out/cache/ (overrides the inferred common path prefix). Pass Path('.') when running on a subdirectory so the cache stays at ./graphify-out/cache/. + Anchors ids/source_file only as a fallback when `root` is unset. parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files, use ProcessPoolExecutor for multi-core extraction. max_workers: max subprocess count. Defaults to cpu_count (or the value of GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work). """ paths = [Path(p) for p in paths] + anchor_root = Path(root) if root is not None else None _check_tree_sitter_version() _raise_recursion_limit() # Workspace package manifests/globs can change during watch or repeated extraction. @@ -4337,7 +4344,13 @@ def extract( root = Path(*paths[0].parts[:common_len]) if common_len else Path(".") except Exception: root = Path(".") - if cache_root is not None: + # An explicit anchor wins. cache_root is only a fallback anchor: it happens to + # equal the scan root for the no---out CLI path and for watch, but with --out it + # is the OUTPUT dir, and letting it anchor made every scanned file "out-of-root" + # -> _portable_out_of_root_sf() -> bare basename for the whole corpus (#1941). + if anchor_root is not None: + root = anchor_root + elif cache_root is not None: root = cache_root root = root.resolve() diff --git a/tests/test_extract.py b/tests/test_extract.py index 32c8e8e13..98b73edb9 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -987,6 +987,48 @@ def test_degenerate_symbol_name_does_not_leak_absolute_id(tmp_path): assert "$()" not in labels, "the degenerate `$` symbol must be dropped (#1899)" +def test_out_of_tree_cache_root_keeps_source_file_relative_to_scan_root(tmp_path): + """#1941: `--out ` must not basename every in-root node. + + The CLI passes cache_root= to relocate the cache, but that value also + anchored relativization, so every scanned file failed `relative_to(root)`, fell + into `_portable_out_of_root_sf`, tripped the `updepth > 3` walk-up guard meant + for stray out-of-root ProjectReferences, and collapsed to a bare basename. + An explicit `root=` anchors ids/source_file on the SCAN root regardless of + where the cache lives. + """ + scan_root = tmp_path / "corpus" + nested = scan_root / "src" / "Data" / "Database" / "RepositoryTests" + nested.mkdir(parents=True) + (nested / "order_repository_tests.py").write_text( + "class OrderRepositoryTests:\n def test_get(self):\n return 1\n", + encoding="utf-8", + ) + # >3 levels off the shared ancestor: the exact shape that triggered basenaming. + out_dir = tmp_path / "a" / "b" / "c" / "d" / "out" + out_dir.mkdir(parents=True) + + result = extract( + [nested / "order_repository_tests.py"], + cache_root=out_dir, + root=scan_root, + ) + source_files = { + n["source_file"] for n in result["nodes"] if n.get("source_file") + } + assert source_files, "expected nodes carrying a source_file" + assert source_files == { + "src/Data/Database/RepositoryTests/order_repository_tests.py" + }, f"source_file must stay relative to the scan root, got {source_files}" + # The point of the field: it resolves back to a real file against the root. + for sf in source_files: + assert (scan_root / sf).is_file(), f"{sf} does not resolve under {scan_root}" + # #1899 must not regress: no absolute path / username leak. + for n in result["nodes"]: + assert str(tmp_path) not in (n.get("source_file") or "") + assert str(tmp_path) not in n["id"] + + def test_python_module_qualified_call_resolves_extracted(tmp_path): """`module.func()` where `module` is imported resolves to the callable that module contains, with an EXTRACTED `calls` edge (#1883). A lowercase module