From 5606ba17d4ea9443f36feffa5ae5059c6e990baf Mon Sep 17 00:00:00 2001 From: stijnswapped Date: Tue, 2 Jun 2026 14:48:10 +0200 Subject: [PATCH 1/2] fix: prune stale AST nodes on full re-extraction (#1116) `graphify update` and the post-checkout hook call `_rebuild_code` with no `changed_paths`, taking the full re-extraction branch. A symbol removed from a file that still exists was never pruned: its node is absent from the fresh `new_ast_ids` but its `source_file` still exists on disk, so it was not added to `evict_sources` and survived every incremental rebuild (along with its inbound call/use edge) until a full clean rebuild. A naive "evict every re-extracted file's nodes by source_file" fix would silently delete semantic/LLM nodes, which are not re-extracted in the AST-only update pass yet carry a source_file pointing at a surviving code file. Fix by node identity instead of by file: tag AST nodes with `_origin="ast"` at extraction time, then on a full rebuild drop any AST-marked node missing from the fresh AST output. Semantic nodes lack the marker and are preserved (still evicted by the existing deleted-file reconciliation). The marker round-trips through graph.json, so a later `graphify update` reads it back. Incremental rebuilds (changed_paths set) are unaffected. Adds a regression test covering both the removed-symbol eviction (node + edge) and the guard that a pre-seeded semantic node on a surviving file is kept. --- graphify/extract.py | 8 ++++++ graphify/watch.py | 10 +++++++ tests/test_watch.py | 67 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/graphify/extract.py b/graphify/extract.py index 7b392d6c5..42db819a5 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -11075,6 +11075,14 @@ def extract( except ValueError: pass + # Tag AST provenance so the incremental watch rebuild can distinguish + # AST-extracted nodes from semantic/LLM nodes (which are added by a separate + # pass and never flow through here). On a full re-extraction the watcher + # drops any AST-marked node missing from the fresh output — even when its + # source file still exists — without touching genuine semantic nodes (#1116). + for n in all_nodes: + n["_origin"] = "ast" + return { "nodes": all_nodes, "edges": all_edges, diff --git a/graphify/watch.py b/graphify/watch.py index 8a2c75279..cde4c3f70 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -543,9 +543,19 @@ def _rebuild_code( evict_sources.add(sf) evict_sources.add(norm) deleted_paths.add(norm) + # On a full re-extraction every code file is re-extracted, so + # new_ast_ids is the complete current AST set. Any AST-marked node + # missing from it is stale and must be dropped even if its source + # file still exists (a symbol removed from a surviving file, #1116). + # Gate on full_rebuild: in incremental mode an AST node from an + # unchanged file is legitimately absent from new_ast_ids. Semantic + # nodes lack the "_origin" marker, so they are never dropped here — + # only by the deleted-file eviction in evict_sources above. + full_rebuild = changed_paths is None preserved_nodes = [ n for n in existing.get("nodes", []) if n["id"] not in new_ast_ids + and not (full_rebuild and n.get("_origin") == "ast") and (not evict_sources or n.get("source_file") not in evict_sources) ] all_ids = new_ast_ids | {n["id"] for n in preserved_nodes} diff --git a/tests/test_watch.py b/tests/test_watch.py index 80dff3148..07772e1a6 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -172,6 +172,73 @@ def test_rebuild_code_evicts_nodes_from_deleted_files(tmp_path): assert "login()" in node_labels_after, "nodes from surviving file must be kept" +def test_rebuild_code_evicts_removed_symbol_from_surviving_file(tmp_path): + """#1116: graphify update (_rebuild_code with no changed_paths) must prune a + symbol removed from a file that still exists — and its inbound call edge — + without dropping genuine semantic nodes that share the surviving file.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + + (corpus / "a.py").write_text( + "def foo(): pass\ndef bar(): pass\n", encoding="utf-8" + ) + (corpus / "b.py").write_text( + "from a import foo\n\ndef caller():\n foo()\n", encoding="utf-8" + ) + + assert _rebuild_code(corpus, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + + def labels(d): + return {n["label"] for n in d.get("nodes", [])} + + def id_for(d, label): + return next(n["id"] for n in d.get("nodes", []) if n["label"] == label) + + def edges(d): + return d.get("links", d.get("edges", [])) + + before = labels(data) + assert {"foo()", "bar()", "caller()"} <= before + foo_id = id_for(data, "foo()") + caller_id = id_for(data, "caller()") + assert any( + {e.get("source"), e.get("target")} == {caller_id, foo_id} + for e in edges(data) + ), "cross-file caller->foo call edge must exist before removal" + + # Pre-seed a semantic node on the surviving a.py (no AST id, no _origin + # marker). A naive "evict every re-extracted file's nodes by source_file" + # fix would wrongly delete this; the identity-based fix must keep it. + data["nodes"].append({ + "id": "a_authconcept", + "label": "AuthConcept", + "file_type": "concept", + "source_file": "a.py", + }) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + # Remove foo() from a.py (keep bar); leave b.py untouched. + (corpus / "a.py").write_text("def bar(): pass\n", encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False, force=True) is True + after_data = json.loads(graph_path.read_text(encoding="utf-8")) + after = labels(after_data) + + assert "foo()" not in after, "removed symbol must be pruned from surviving file" + assert not any( + e.get("source") == foo_id or e.get("target") == foo_id + for e in edges(after_data) + ), "dangling edge to the removed symbol must be dropped" + assert "bar()" in after, "surviving symbol in the same file must be kept" + assert "caller()" in after, "unchanged file's nodes must be kept" + assert "AuthConcept" in after, "semantic node on a surviving file must not be evicted" + + @pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_lock_non_blocking_does_not_clobber_holder(tmp_path): """GH-858: a non-blocking caller that fails to acquire the lock must not From c84c8ce2dd96c7a96499d5a0c281f931e881c0b4 Mon Sep 17 00:00:00 2001 From: stijnswapped Date: Thu, 4 Jun 2026 08:52:28 +0200 Subject: [PATCH 2/2] address #1118 review: pre-upgrade lag test + export marker strip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add a backward-compat test seeding a marker-less AST node (pre-#1116 graph) for a symbol removed from a surviving file, asserting it is NOT pruned on the first post-upgrade update (the documented one-cycle lag, no data loss) and that it self-heals once stamped `_origin="ast"`. - CHANGELOG: record the #1116 fix plus a one-line backward-compat note about the first-update lag. - export.py: stop leaking internal `_`-prefixed markers (`_origin`, `_src`/`_tgt`) through the unfiltered prop passthrough — strip them in push_to_neo4j and to_graphml so they don't surface as node/edge props. --- CHANGELOG.md | 1 + graphify/export.py | 19 +++++++++++++-- tests/test_watch.py | 59 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec335bcf8..f6f5939da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: `uv tool install graphifyy` / `pip install graphifyy` no longer fails to build on Linux/macOS — `tree-sitter-dm` (BYOND DreamMaker) ships only a Windows wheel, so on other platforms it compiled from source and aborted the entire install when a C toolchain or `python3-dev` headers were missing. It is now an optional extra (`graphifyy[dm]`, also in `[all]`) instead of a core dependency, so the default install needs no compiler (#1104). - **Upgrade note:** DreamMaker `.dm`/`.dme` users must reinstall with `graphifyy[dm]` (or `[all]`) to keep AST extraction — on `uv tool upgrade` the now-optional grammar is removed. `.dmi`/`.dmm`/`.dmf` parsing is unaffected (no tree-sitter dependency). - Fix: community IDs are now assigned by a total order (`(-size, sorted node IDs)`) so an identical grouping always gets identical IDs across runs — previously the equal-sized small communities that dominate a sparse graph were numbered by the partitioner's (not seed-stable) enumeration order, making a per-node community diff report large spurious "churn" even though the actual grouping was reproducible (#1090 follow-up) +- Fix: `graphify update` (full re-extraction) now prunes a symbol removed from a file that still exists — and its dangling inbound edge — instead of keeping the stale node until the next clean rebuild. AST nodes are tagged `_origin="ast"` at extraction so the watcher drops AST nodes missing from the fresh output by identity, never blanket-by-source-file, so genuine semantic/LLM nodes on the same file are preserved (#1116). **Backward-compat note:** a `graph.json` built before this change has no `_origin` markers, so on the *first* update after upgrading a removed symbol is not pruned that one cycle (no data loss); it self-heals once the node is re-stamped on a full re-extraction. ## 0.8.27 (2026-05-31) diff --git a/graphify/export.py b/graphify/export.py index 6a6fec8f6..425cced6b 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -1274,7 +1274,10 @@ def _safe_label(label: str) -> str: with driver.session() as session: for node_id, data in G.nodes(data=True): - props = {k: v for k, v in data.items() if isinstance(v, (str, int, float, bool))} + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } props["id"] = node_id cid = node_community.get(node_id) if cid is not None: @@ -1289,7 +1292,10 @@ def _safe_label(label: str) -> str: for u, v, data in G.edges(data=True): rel = _safe_rel(data.get("relation", "RELATED_TO")) - props = {k: v for k, v in data.items() if isinstance(v, (str, int, float, bool))} + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } session.run( f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " f"MERGE (a)-[r:{rel}]->(b) SET r += $props", @@ -1317,6 +1323,15 @@ def to_graphml( node_community = _node_community_map(communities) for node_id in H.nodes(): H.nodes[node_id]["community"] = node_community.get(node_id, -1) + # Drop internal markers (e.g. the AST-provenance "_origin" tag, #1116, and + # the "_src"/"_tgt" direction markers) — they are persistence/runtime details, + # not graph data, and should not leak into the exported file. + for _, attrs in H.nodes(data=True): + for k in [k for k in attrs if k.startswith("_")]: + del attrs[k] + for _, _, attrs in H.edges(data=True): + for k in [k for k in attrs if k.startswith("_")]: + del attrs[k] nx.write_graphml(H, output_path) diff --git a/tests/test_watch.py b/tests/test_watch.py index 07772e1a6..0745ce872 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -239,6 +239,65 @@ def edges(d): assert "AuthConcept" in after, "semantic node on a surviving file must not be evicted" +def test_rebuild_code_preupgrade_marker_less_node_one_cycle_lag(tmp_path): + """#1118 backward-compat: a graph.json built before #1116 has no `_origin` + markers. On the first `graphify update` after upgrading, a symbol removed + from a surviving file is NOT pruned that cycle — its old node carries no + marker, so the new drop-rule skips it. This is a deliberate one-cycle lag + (no data loss); it self-heals once the node has been stamped `_origin="ast"` + (which a full re-extraction does for every surviving symbol).""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "a.py").write_text("def bar(): pass\n", encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + + def labels(d): + return {n["label"] for n in d.get("nodes", [])} + + # Simulate a pre-#1116 graph: strip every `_origin` marker, then inject a + # stale AST node for a symbol no longer present in a.py's source — also + # marker-less, exactly as a pre-upgrade graph would carry it. + for n in data["nodes"]: + n.pop("_origin", None) + data["nodes"].append({ + "id": "a_foo", + "label": "foo()", + "file_type": "function", + "source_file": "a.py", + }) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + # First update after "upgrade" (full rebuild, no changed_paths): the stale + # node has no marker, so the drop-rule skips it and it survives this cycle. + assert _rebuild_code(corpus, acquire_lock=False, force=True) is True + after = json.loads(graph_path.read_text(encoding="utf-8")) + assert "foo()" in labels(after), ( + "pre-upgrade marker-less stale node must survive the first update — " + "documented one-cycle backward-compat lag (#1118)" + ) + + # Once stamped (a full re-extraction stamps every surviving symbol), the + # drop-rule applies on the next update and the stale node self-heals away. + for n in after["nodes"]: + if n["label"] == "foo()": + n["_origin"] = "ast" + graph_path.write_text(json.dumps(after), encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False, force=True) is True + healed = json.loads(graph_path.read_text(encoding="utf-8")) + assert "foo()" not in labels(healed), ( + "once carrying _origin=ast, the stale node is pruned on the next " + "update (self-heal)" + ) + assert "bar()" in labels(healed), "surviving symbol must be kept throughout" + + @pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_lock_non_blocking_does_not_clobber_holder(tmp_path): """GH-858: a non-blocking caller that fails to acquire the lock must not