From 8e4c803f316ec3789d856b69de5e38d9ae72dc24 Mon Sep 17 00:00:00 2001 From: Ahmad Fathallah Date: Tue, 12 May 2026 02:23:51 +0300 Subject: [PATCH 1/2] reduce graph update churn and stabilize community IDs Make `graphify update` idempotent by skipping output rewrites when graph/report content is unchanged, add `update --no-cluster`, and preserve community IDs across runs via overlap-based remapping with deterministic partition inputs. Co-authored-by: Cursor --- graphify/__main__.py | 29 ++++-- graphify/cluster.py | 71 ++++++++++++++- graphify/watch.py | 184 +++++++++++++++++++++++++++++++++------ tests/test_cli_export.py | 14 +++ tests/test_cluster.py | 26 +++++- tests/test_watch.py | 33 +++++++ 6 files changed, 320 insertions(+), 37 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index cdb40ea33..3aa5484a4 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -1151,6 +1151,7 @@ def main() -> None: print(" update re-extract code files and update the graph (no LLM needed)") print(" --force overwrite graph.json even if the rebuild has fewer nodes") print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") + print(" --no-cluster skip clustering, write raw extraction only") print(" cluster-only rerun clustering on an existing graph.json and regenerate report") print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") print(" --graph path to graph.json (default /graphify-out/graph.json)") @@ -1717,12 +1718,26 @@ def main() -> None: elif cmd == "update": force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") - argv = list(sys.argv) - if "--force" in argv[2:]: - force = True - argv = [a for a in argv if a != "--force"] - if len(argv) > 2: - watch_path = Path(argv[2]) + no_cluster = False + args = sys.argv[2:] + watch_arg: str | None = None + for a in args: + if a == "--force": + force = True + continue + if a == "--no-cluster": + no_cluster = True + continue + if a.startswith("-"): + print(f"error: unknown update option: {a}", file=sys.stderr) + sys.exit(2) + if watch_arg is not None: + print("error: update accepts at most one path argument", file=sys.stderr) + sys.exit(2) + watch_arg = a + + if watch_arg is not None: + watch_path = Path(watch_arg) else: # Try to recover the scan root saved by the last full build saved = Path(_GRAPHIFY_OUT) / ".graphify_root" @@ -1738,7 +1753,7 @@ def main() -> None: # Interactive CLI: block on the per-repo lock rather than skip, so the # user sees their explicit `graphify update` complete instead of # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, block_on_lock=True) + ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) if ok: print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") if not ( diff --git a/graphify/cluster.py b/graphify/cluster.py index b5555a85b..b1f1df299 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -3,6 +3,7 @@ import contextlib import inspect import io +import json import sys import networkx as nx @@ -27,15 +28,30 @@ def _partition(G: nx.Graph) -> dict[str, int]: Output from graspologic is suppressed to prevent ANSI escape codes from corrupting terminal scroll buffers on Windows PowerShell 5.1. """ + stable = nx.Graph() + stable.add_nodes_from(sorted(G.nodes(), key=str)) + edge_rows = sorted( + G.edges(data=True), + key=lambda row: (str(row[0]), str(row[1]), json.dumps(row[2], sort_keys=True, ensure_ascii=False)), + ) + for src, tgt, attrs in edge_rows: + stable.add_edge(src, tgt, **attrs) + try: from graspologic.partition import leiden + lsig = inspect.signature(leiden).parameters + kwargs: dict = {} + if "random_seed" in lsig: + kwargs["random_seed"] = 42 + if "trials" in lsig: + kwargs["trials"] = 1 # Suppress graspologic output to prevent ANSI escape codes from # corrupting PowerShell 5.1 scroll buffer (issue #19) old_stderr = sys.stderr try: sys.stderr = io.StringIO() with _suppress_output(): - result = leiden(G) + result = leiden(stable, **kwargs) finally: sys.stderr = old_stderr return result @@ -48,7 +64,7 @@ def _partition(G: nx.Graph) -> dict[str, int]: kwargs: dict = {"seed": 42, "threshold": 1e-4} if "max_level" in inspect.signature(nx.community.louvain_communities).parameters: kwargs["max_level"] = 10 - communities = nx.community.louvain_communities(G, **kwargs) + communities = nx.community.louvain_communities(stable, **kwargs) return {node: cid for cid, nodes in enumerate(communities) for node in nodes} @@ -148,3 +164,54 @@ def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float: def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]: return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()} + + +def remap_communities_to_previous( + communities: dict[int, list[str]], + previous_node_community: dict[str, int], +) -> dict[int, list[str]]: + """Remap community IDs to maximize overlap with a previous assignment. + + Uses greedy one-to-one matching by intersection size, then assigns fresh IDs + to unmatched communities in deterministic order (size desc, lexical tie-break). + """ + if not communities: + return {} + + new_sets = {cid: set(nodes) for cid, nodes in communities.items()} + old_sets: dict[int, set[str]] = {} + for node, old_cid in previous_node_community.items(): + old_sets.setdefault(old_cid, set()).add(node) + + overlaps: list[tuple[int, int, int]] = [] + for old_cid, old_nodes in old_sets.items(): + for new_cid, new_nodes in new_sets.items(): + overlap = len(old_nodes & new_nodes) + if overlap > 0: + overlaps.append((overlap, old_cid, new_cid)) + overlaps.sort(key=lambda x: (-x[0], x[1], x[2])) + + new_to_final: dict[int, int] = {} + used_old_ids: set[int] = set() + matched_new_ids: set[int] = set() + for _overlap, old_cid, new_cid in overlaps: + if old_cid in used_old_ids or new_cid in matched_new_ids: + continue + new_to_final[new_cid] = old_cid + used_old_ids.add(old_cid) + matched_new_ids.add(new_cid) + + unmatched = [cid for cid in communities if cid not in matched_new_ids] + unmatched.sort(key=lambda cid: (-len(communities[cid]), tuple(sorted(communities[cid])))) + next_id = 0 + for new_cid in unmatched: + while next_id in used_old_ids: + next_id += 1 + new_to_final[new_cid] = next_id + used_old_ids.add(next_id) + next_id += 1 + + remapped: dict[int, list[str]] = {} + for new_cid, nodes in communities.items(): + remapped[new_to_final[new_cid]] = sorted(nodes) + return dict(sorted(remapped.items(), key=lambda kv: kv[0])) diff --git a/graphify/watch.py b/graphify/watch.py index 0dbede148..c1cdee419 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -3,6 +3,7 @@ import contextlib import json import os +import re import sys import time from pathlib import Path @@ -115,16 +116,48 @@ def _relativize_source_files(payload: dict, root: Path) -> None: continue +def _node_community_map(graph_data: dict) -> dict[str, int]: + out: dict[str, int] = {} + for node in graph_data.get("nodes", []): + node_id = node.get("id") + cid = node.get("community") + if node_id is None or cid is None: + continue + out[str(node_id)] = int(cid) + return out + + +def _canonical_graph_for_compare(graph_data: dict) -> dict: + canonical = dict(graph_data) + canonical.pop("built_at_commit", None) + for key in ("nodes", "links", "edges", "hyperedges"): + if key in canonical and isinstance(canonical[key], list): + canonical[key] = sorted( + canonical[key], + key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False), + ) + return canonical + + +def _report_for_compare(report_text: str) -> str: + return re.sub(r"^- Built from commit: `[^`]+`\n?", "", report_text, flags=re.MULTILINE) + + +def _json_text(data: dict) -> str: + return json.dumps(data, indent=2, ensure_ascii=False) + "\n" + + def _rebuild_code( watch_path: Path, *, changed_paths: list[Path] | None = None, follow_symlinks: bool = False, force: bool = False, + no_cluster: bool = False, acquire_lock: bool = True, block_on_lock: bool = False, ) -> bool: - """Re-run AST extraction + build + cluster + report for code files. No LLM needed. + """Re-run AST extraction + build + optional cluster + report for code files. When ``force`` is True the node-count safety check in ``to_json`` is bypassed so the rebuilt graph overwrites graph.json even if it has fewer nodes. @@ -142,6 +175,9 @@ def _rebuild_code( ``block_on_lock=True`` to wait instead of skip (used by the interactive ``graphify update`` CLI). + ``no_cluster`` skips community detection and writes raw merged extraction + JSON to graphify-out/graph.json (mirrors ``extract --no-cluster``). + Returns True on success, False on error or skipped-due-to-lock. """ out = watch_path / _GRAPHIFY_OUT @@ -156,6 +192,7 @@ def _rebuild_code( changed_paths=changed_paths, follow_symlinks=follow_symlinks, force=force, + no_cluster=no_cluster, acquire_lock=False, ) @@ -166,7 +203,7 @@ def _rebuild_code( from graphify.extract import extract, _get_extractor from graphify.detect import detect from graphify.build import build_from_json - from graphify.cluster import cluster, score_all + from graphify.cluster import cluster, remap_communities_to_previous, score_all from graphify.analyze import god_nodes, surprising_connections, suggest_questions from graphify.report import generate from graphify.export import to_json, to_html @@ -225,9 +262,11 @@ def _rebuild_code( # source_file matches a path that was changed (re-extracted) or deleted — # otherwise the old nodes for those files would survive forever. existing_graph = out / "graph.json" + existing_graph_data: dict = {} if existing_graph.exists(): try: existing = json.loads(existing_graph.read_text(encoding="utf-8")) + existing_graph_data = existing new_ast_ids = {n["id"] for n in result["nodes"]} evict_sources: set[str] = set(deleted_paths) if changed_paths is not None: @@ -257,6 +296,58 @@ def _rebuild_code( pass # corrupt graph.json - proceed with AST-only _relativize_source_files(result, project_root) + out.mkdir(exist_ok=True) + (out / ".graphify_root").write_text(str(watch_root), encoding="utf-8") + + if no_cluster: + candidate_graph_data = dict(result) + candidate_graph_text = _json_text(candidate_graph_data) + existing_text = existing_graph.read_text(encoding="utf-8") if existing_graph.exists() else "" + same_graph = False + if existing_graph.exists(): + try: + existing_payload = json.loads(existing_text) + same_graph = ( + json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False) + == json.dumps(_canonical_graph_for_compare(candidate_graph_data), sort_keys=True, ensure_ascii=False) + ) + except Exception: + same_graph = False + if not same_graph: + if (not force) and existing_graph_data: + existing_n = len(existing_graph_data.get("nodes", [])) + new_n = len(candidate_graph_data.get("nodes", [])) + if new_n < existing_n: + print( + f"[graphify] WARNING: new graph has {new_n} nodes but existing " + f"graph.json has {existing_n}. Refusing to overwrite — you may be " + f"missing chunk files from a previous session. " + f"Pass force=True to override.", + file=sys.stderr, + ) + return False + existing_graph.write_text(candidate_graph_text, encoding="utf-8") + + try: + from graphify.detect import save_manifest + save_manifest(detected["files"]) + except Exception: + pass + + # clear stale needs_update flag if present + flag = out / "needs_update" + if flag.exists(): + flag.unlink() + + if same_graph: + print("[graphify watch] No code-graph changes detected (--no-cluster); outputs left untouched.") + else: + print( + "[graphify watch] Rebuilt (no clustering): " + f"{len(result.get('nodes', []))} nodes, {len(result.get('edges', []))} edges" + ) + print(f"[graphify watch] graph.json updated in {out}") + return True detection = { "files": {"code": [str(f) for f in code_files], "document": [], "paper": [], "image": []}, @@ -266,6 +357,9 @@ def _rebuild_code( G = build_from_json(result) communities = cluster(G) + previous_node_community = _node_community_map(existing_graph_data) + if previous_node_community: + communities = remap_communities_to_previous(communities, previous_node_community) cohesion = score_all(G, communities) gods = god_nodes(G) surprises = surprising_connections(G, communities) @@ -280,13 +374,52 @@ def _rebuild_code( if cid not in labels: labels[cid] = "Community " + str(cid) questions = suggest_questions(G, communities, labels) - - out.mkdir(exist_ok=True) - (out / ".graphify_root").write_text(str(watch_root), encoding="utf-8") - - json_written = to_json(G, communities, str(out / "graph.json"), force=force, built_at_commit=commit) + report = generate(G, communities, cohesion, labels, gods, surprises, detection, + {"input": 0, "output": 0}, report_root, suggested_questions=questions, + built_at_commit=commit) + report_path = out / "GRAPH_REPORT.md" + labels_json = json.dumps({str(k): v for k, v in sorted(labels.items())}, ensure_ascii=False, indent=2) + "\n" + graph_tmp = out / ".graph.tmp.json" + json_written = to_json(G, communities, str(graph_tmp), force=True, built_at_commit=commit) if not json_written: return False + candidate_graph_data = json.loads(graph_tmp.read_text(encoding="utf-8")) + same_graph = False + same_report = False + if existing_graph.exists(): + try: + existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) + same_graph = ( + json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False) + == json.dumps(_canonical_graph_for_compare(candidate_graph_data), sort_keys=True, ensure_ascii=False) + ) + except Exception: + same_graph = False + if report_path.exists(): + old_report = report_path.read_text(encoding="utf-8") + same_report = _report_for_compare(old_report) == _report_for_compare(report) + no_change = same_graph and same_report + if no_change: + graph_tmp.unlink(missing_ok=True) + print("[graphify watch] No code-graph changes detected; graph.json/GRAPH_REPORT.md left untouched.") + else: + if (not force) and existing_graph_data: + existing_n = len(existing_graph_data.get("nodes", [])) + new_n = len(candidate_graph_data.get("nodes", [])) + if new_n < existing_n: + graph_tmp.unlink(missing_ok=True) + print( + f"[graphify] WARNING: new graph has {new_n} nodes but existing " + f"graph.json has {existing_n}. Refusing to overwrite — you may be " + f"missing chunk files from a previous session. " + f"Pass force=True to override.", + file=sys.stderr, + ) + return False + graph_tmp.replace(existing_graph) + report_path.write_text(report, encoding="utf-8") + + labels_file.write_text(labels_json, encoding="utf-8") try: from graphify.detect import save_manifest @@ -294,27 +427,23 @@ def _rebuild_code( except Exception: pass - report = generate(G, communities, cohesion, labels, gods, surprises, detection, - {"input": 0, "output": 0}, report_root, suggested_questions=questions, - built_at_commit=commit) - (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") - # to_html raises ValueError for graphs > MAX_NODES_FOR_VIZ (5000). # Wrap so core outputs (graph.json + GRAPH_REPORT.md) always land. html_written = False - try: - to_html(G, communities, str(out / "graph.html"), community_labels=labels or None) - html_written = True - except ValueError as viz_err: - print(f"[graphify watch] Skipped graph.html: {viz_err}") - stale = out / "graph.html" - if stale.exists(): - stale.unlink() + if not no_change: + try: + to_html(G, communities, str(out / "graph.html"), community_labels=labels or None) + html_written = True + except ValueError as viz_err: + print(f"[graphify watch] Skipped graph.html: {viz_err}") + stale = out / "graph.html" + if stale.exists(): + stale.unlink() # Regenerate callflow HTML if the user previously generated one — # opt-in by existence so users who never ran callflow-html aren't affected. callflow_files = list(out.glob("*-callflow.html")) - if callflow_files: + if callflow_files and not no_change: try: from graphify.callflow_html import write_callflow_html for cf in callflow_files: @@ -333,12 +462,13 @@ def _rebuild_code( if flag.exists(): flag.unlink() - print(f"[graphify watch] Rebuilt: {G.number_of_nodes()} nodes, " - f"{G.number_of_edges()} edges, {len(communities)} communities") - products = "graph.json" + (", graph.html" if html_written else "") + " and GRAPH_REPORT.md" - if callflow_files: - products += f", {len(callflow_files)} callflow HTML" - print(f"[graphify watch] {products} updated in {out}") + if not no_change: + print(f"[graphify watch] Rebuilt: {G.number_of_nodes()} nodes, " + f"{G.number_of_edges()} edges, {len(communities)} communities") + products = "graph.json" + (", graph.html" if html_written else "") + " and GRAPH_REPORT.md" + if callflow_files: + products += f", {len(callflow_files)} callflow HTML" + print(f"[graphify watch] {products} updated in {out}") return True except Exception as exc: diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 8d1525083..35cfa6453 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -236,3 +236,17 @@ def test_explain_uses_graphify_out_env(tmp_path): def test_export_unknown_format_fails(tmp_path): r = _run(["export", "pdf"], tmp_path) assert r.returncode != 0 + + +def test_update_no_cluster_writes_raw_graph(tmp_path): + src = tmp_path / "sample.py" + src.write_text("def f():\n return 1\n", encoding="utf-8") + + r = _run(["update", ".", "--no-cluster"], tmp_path) + assert r.returncode == 0, r.stderr + + graph_path = tmp_path / "graphify-out" / "graph.json" + assert graph_path.exists() + data = json.loads(graph_path.read_text(encoding="utf-8")) + assert "nodes" in data and "edges" in data + assert all("community" not in node for node in data["nodes"]) diff --git a/tests/test_cluster.py b/tests/test_cluster.py index b5c16fad6..21fd2ca3a 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -3,7 +3,7 @@ import networkx as nx from pathlib import Path from graphify.build import build_from_json -from graphify.cluster import cluster, cohesion_score, score_all +from graphify.cluster import cluster, cohesion_score, remap_communities_to_previous, score_all FIXTURES = Path(__file__).parent / "fixtures" @@ -74,3 +74,27 @@ def test_cluster_does_not_write_to_stderr(capsys): # Allow logging output (starts with [graphify]) but no raw ANSI codes for line in captured.err.splitlines(): assert "\x1b" not in line, f"cluster() wrote ANSI to stderr: {line!r}" + + +def test_remap_communities_to_previous_reuses_old_ids(): + communities = { + 10: ["a", "b", "c"], + 11: ["d", "e"], + } + previous = {"a": 5, "b": 5, "c": 5, "d": 1, "e": 1} + remapped = remap_communities_to_previous(communities, previous) + assert set(remapped.keys()) == {1, 5} + assert remapped[5] == ["a", "b", "c"] + assert remapped[1] == ["d", "e"] + + +def test_remap_communities_to_previous_assigns_deterministic_new_ids(): + communities = { + 7: ["x", "y", "z"], + 8: ["m"], + } + previous = {"a": 3} + remapped = remap_communities_to_previous(communities, previous) + assert list(remapped.keys()) == [0, 1] + assert remapped[0] == ["x", "y", "z"] + assert remapped[1] == ["m"] diff --git a/tests/test_watch.py b/tests/test_watch.py index ac396aa6e..c5eff4272 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -94,3 +94,36 @@ def mock_import(name, *args, **kwargs): from graphify.watch import watch with pytest.raises(ImportError, match="watchdog not installed"): watch(tmp_path) + + +def test_rebuild_code_is_idempotent_when_cluster_ids_flap(tmp_path, monkeypatch): + from graphify import cluster as cluster_mod + from graphify.watch import _rebuild_code + + src = tmp_path / "app.py" + src.write_text("def alpha():\n return 1\n\ndef beta():\n return alpha()\n", encoding="utf-8") + + calls = {"n": 0} + + def flaky_cluster(G): + calls["n"] += 1 + nodes = sorted(G.nodes()) + if calls["n"] % 2 == 1: + return {100: nodes} + return {7: nodes} + + monkeypatch.setattr(cluster_mod, "cluster", flaky_cluster) + monkeypatch.setattr(cluster_mod, "score_all", lambda _G, comm: {cid: 1.0 for cid in comm}) + + assert _rebuild_code(tmp_path) + graph_path = tmp_path / "graphify-out" / "graph.json" + report_path = tmp_path / "graphify-out" / "GRAPH_REPORT.md" + first_graph = graph_path.read_text(encoding="utf-8") + first_report = report_path.read_text(encoding="utf-8") + + assert _rebuild_code(tmp_path) + second_graph = graph_path.read_text(encoding="utf-8") + second_report = report_path.read_text(encoding="utf-8") + + assert first_graph == second_graph + assert first_report == second_report From ef0e6ee681146e7a9fab3b9ec52d4bb8e5b41c33 Mon Sep 17 00:00:00 2001 From: Ahmad Fathallah Date: Tue, 12 May 2026 02:37:38 +0300 Subject: [PATCH 2/2] harden community serialization fallbacks Use safe JSON serialization fallbacks for deterministic sort keys in clustering and graph canonicalization, and skip invalid community IDs with a stderr warning instead of raising during update rebuilds. Co-authored-by: Cursor --- graphify/cluster.py | 6 +++++- graphify/watch.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/graphify/cluster.py b/graphify/cluster.py index b1f1df299..7959ec173 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -32,7 +32,11 @@ def _partition(G: nx.Graph) -> dict[str, int]: stable.add_nodes_from(sorted(G.nodes(), key=str)) edge_rows = sorted( G.edges(data=True), - key=lambda row: (str(row[0]), str(row[1]), json.dumps(row[2], sort_keys=True, ensure_ascii=False)), + key=lambda row: ( + str(row[0]), + str(row[1]), + json.dumps(row[2], sort_keys=True, ensure_ascii=False, default=str), + ), ) for src, tgt, attrs in edge_rows: stable.add_edge(src, tgt, **attrs) diff --git a/graphify/watch.py b/graphify/watch.py index c1cdee419..7c5dca639 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -123,7 +123,15 @@ def _node_community_map(graph_data: dict) -> dict[str, int]: cid = node.get("community") if node_id is None or cid is None: continue - out[str(node_id)] = int(cid) + try: + out[str(node_id)] = int(cid) + except (TypeError, ValueError): + print( + f"[graphify watch] Skipping node with invalid community id: " + f"node_id={node_id!r} community={cid!r}", + file=sys.stderr, + ) + continue return out @@ -134,7 +142,7 @@ def _canonical_graph_for_compare(graph_data: dict) -> dict: if key in canonical and isinstance(canonical[key], list): canonical[key] = sorted( canonical[key], - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False), + key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), ) return canonical