diff --git a/graphify/__main__.py b/graphify/__main__.py index 3bc27c336..78ed29cd0 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -4284,6 +4284,7 @@ def _parse_float(name: str, raw: str) -> float: } sem_cache_hits = 0 sem_cache_misses = 0 + fresh: dict = {"nodes": [], "edges": [], "hyperedges": []} if semantic_files: sem_paths_str = [str(p) for p in semantic_files] cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( @@ -4352,6 +4353,32 @@ def _progress(idx: int, total: int, _result: dict) -> None: file=sys.stderr, ) sys.exit(1) + # Incremental: never prune/replace changed files unless re-extraction + # actually produced nodes/edges. Invalid JSON and connection failures + # that yield empty chunks must abort before merge (#incremental-safe). + if incremental_mode and uncached_paths: + from graphify.build import paths_missing_from_extraction as _paths_missing + + _missing = _paths_missing(fresh, uncached_paths, target) + if fresh.get("failed_chunks", 0) > 0 or _missing: + if fresh.get("failed_chunks", 0) > 0: + print( + f"[graphify extract] error: {fresh['failed_chunks']} semantic " + f"chunk(s) failed during incremental re-extraction.", + file=sys.stderr, + ) + for _path in _missing: + print( + f"[graphify extract] error: re-extraction produced no " + f"nodes/edges for {_path}", + file=sys.stderr, + ) + print( + "[graphify extract] incremental update aborted — existing " + "graph.json and manifest unchanged.", + file=sys.stderr, + ) + sys.exit(1) try: _save_semantic_cache( fresh.get("nodes", []), @@ -4422,6 +4449,19 @@ def _progress(idx: int, total: int, _result: dict) -> None: for ftype, flist in files_by_type.items() } + _incremental_prune: list[str] | None = None + if incremental_mode: + from graphify.build import path_covered_by_extraction as _path_covered + + _changed_code = [str(p) for p in code_files] + _changed_sem = [ + str(p) for p in semantic_files + if _path_covered(str(p), sem_result, target) + ] + _incremental_prune = ( + list(dict.fromkeys(deleted_files + _changed_code + _changed_sem)) or None + ) + if no_cluster: # --no-cluster: dump the raw merged extraction as graph.json. # No NetworkX, no community detection, no analysis sidecar. @@ -4512,7 +4552,7 @@ def _progress(idx: int, total: int, _result: dict) -> None: G = _build_merge( [merged], graph_path=existing_graph_path, - prune_sources=deleted_files or None, + prune_sources=_incremental_prune, dedup=True, dedup_llm_backend=dedup_backend, root=target, diff --git a/graphify/build.py b/graphify/build.py index aa4cfff4e..eafad5d46 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -574,6 +574,55 @@ def _kept(item: dict) -> bool: return G +def source_path_aliases(p: str, root: str | Path | None = None) -> set[str]: + """All normalized forms of a scan/manifest path for matching source_file.""" + aliases: set[str] = set() + if not p: + return aliases + root_resolved = Path(root).resolve() if root else None + root_str = str(root_resolved) if root_resolved else None + aliases.add(p.replace("\\", "/")) + norm = _norm_source_file(p, root_str) + if norm: + aliases.add(norm) + if root_resolved is not None: + path = Path(p) + try: + abs_p = path if path.is_absolute() else root_resolved / path + aliases.add(abs_p.resolve().relative_to(root_resolved).as_posix()) + except ValueError: + pass + return aliases + + +def source_files_in_extraction(extraction: dict, root: str | Path | None = None) -> set[str]: + """Collect all source_file aliases present in an extraction dict.""" + found: set[str] = set() + for key in ("nodes", "edges", "hyperedges"): + for item in extraction.get(key, []): + sf = item.get("source_file") + if sf: + found |= source_path_aliases(str(sf), root) + return found + + +def path_covered_by_extraction(p: str, extraction: dict, root: str | Path | None = None) -> bool: + """True when extraction contains at least one node/edge/hyperedge for path p.""" + return bool(source_path_aliases(p, root) & source_files_in_extraction(extraction, root)) + + +def paths_missing_from_extraction( + extraction: dict, + paths: list[str], + root: str | Path | None = None, +) -> list[str]: + """Return paths whose re-extraction produced no nodes/edges/hyperedges.""" + if not paths: + return [] + extracted = source_files_in_extraction(extraction, root) + return [p for p in paths if not (source_path_aliases(p, root) & extracted)] + + def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: """Return a copy of G with all node IDs prefixed with repo_tag::. diff --git a/tests/test_build.py b/tests/test_build.py index 99e68c849..6c8a2b38d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -2,7 +2,17 @@ from pathlib import Path import networkx as nx from networkx.readwrite import json_graph -from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas, dedupe_edges, dedupe_nodes +from graphify.build import ( + build_from_json, + build, + build_merge, + dedupe_edges, + dedupe_nodes, + edge_data, + edge_datas, + path_covered_by_extraction, + paths_missing_from_extraction, +) FIXTURES = Path(__file__).parent / "fixtures"