From ddf19955e07fe159f6cfe8da9e33d6ebbe5c6914 Mon Sep 17 00:00:00 2001 From: NYCU-Chung Date: Sat, 11 Apr 2026 01:31:56 +0800 Subject: [PATCH] fix: use utf-8 encoding and LF newlines for all text file IO Python's open() / Path.read_text() / Path.write_text() default to the system locale encoding on Windows (CP1252/CP932/CP950), causing UnicodeEncodeError when graph labels contain CJK characters or emojis. This breaks the final write of graph.json, discarding all prior LLM extraction work. Also adds newline="\n" for git hook writes to prevent CRLF line endings from breaking sh interpreter ("#!/bin/sh\r" -> bad interpreter on Unix/WSL). Files touched: - graphify/export.py (to_json, to_cypher, to_obsidian) - graphify/cache.py (load_cached, save_cached) - graphify/detect.py (manifest, graphifyignore, count_words, looks_like_paper) - graphify/watch.py (GRAPH_REPORT.md, needs_update flag) - graphify/wiki.py (community articles, god node articles, index) - graphify/hooks.py (install/uninstall/status, LF-only for hook scripts) - graphify/serve.py (graph.json read) - graphify/benchmark.py (graph.json read) Test: tests/test_encoding_roundtrip.py verifies CJK/emoji labels round-trip through JSON export and cache IO, and confirms hook files use LF-only line endings. --- graphify/benchmark.py | 2 +- graphify/cache.py | 4 +- graphify/detect.py | 10 +- graphify/export.py | 6 +- graphify/hooks.py | 12 +-- graphify/serve.py | 2 +- graphify/watch.py | 4 +- graphify/wiki.py | 7 +- tests/test_encoding_roundtrip.py | 180 +++++++++++++++++++++++++++++++ 9 files changed, 204 insertions(+), 23 deletions(-) create mode 100644 tests/test_encoding_roundtrip.py diff --git a/graphify/benchmark.py b/graphify/benchmark.py index a71e10e7e..dc420564a 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -75,7 +75,7 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ - data = json.loads(Path(graph_path).read_text()) + data = json.loads(Path(graph_path).read_text(encoding="utf-8")) try: G = json_graph.node_link_graph(data, edges="links") except TypeError: diff --git a/graphify/cache.py b/graphify/cache.py index 7f73db069..54d5b8e66 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -55,7 +55,7 @@ def load_cached(path: Path, root: Path = Path(".")) -> dict | None: if not entry.exists(): return None try: - return json.loads(entry.read_text()) + return json.loads(entry.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): return None @@ -70,7 +70,7 @@ def save_cached(path: Path, result: dict, root: Path = Path(".")) -> None: entry = cache_dir(root) / f"{h}.json" tmp = entry.with_suffix(".tmp") try: - tmp.write_text(json.dumps(result)) + tmp.write_text(json.dumps(result), encoding="utf-8") os.replace(tmp, entry) except Exception: tmp.unlink(missing_ok=True) diff --git a/graphify/detect.py b/graphify/detect.py index e9dc701f0..c13196d8d 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -69,7 +69,7 @@ def _looks_like_paper(path: Path) -> bool: """Heuristic: does this text file read like an academic paper?""" try: # Only scan first 3000 chars for speed - text = path.read_text(errors="ignore")[:3000] + text = path.read_text(encoding="utf-8", errors="ignore")[:3000] hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text)) return hits >= _PAPER_SIGNAL_THRESHOLD except Exception: @@ -226,7 +226,7 @@ def count_words(path: Path) -> int: return len(docx_to_markdown(path).split()) if ext == ".xlsx": return len(xlsx_to_markdown(path).split()) - return len(path.read_text(errors="ignore").split()) + return len(path.read_text(encoding="utf-8", errors="ignore").split()) except Exception: return 0 @@ -271,7 +271,7 @@ def _load_graphifyignore(root: Path) -> list[str]: while True: ignore_file = current / ".graphifyignore" if ignore_file.exists(): - for line in ignore_file.read_text(errors="ignore").splitlines(): + for line in ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines(): line = line.strip() if line and not line.startswith("#"): patterns.append(line) @@ -427,7 +427,7 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict[str, float]: """Load the file modification time manifest from a previous run.""" try: - return json.loads(Path(manifest_path).read_text()) + return json.loads(Path(manifest_path).read_text(encoding="utf-8")) except Exception: return {} @@ -442,7 +442,7 @@ def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PA except OSError: pass # file deleted between detect() and manifest write - skip it Path(manifest_path).parent.mkdir(parents=True, exist_ok=True) - Path(manifest_path).write_text(json.dumps(manifest, indent=2)) + Path(manifest_path).write_text(json.dumps(manifest, indent=2), encoding="utf-8") def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict: diff --git a/graphify/export.py b/graphify/export.py index d35edafc2..5f35c63ec 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -295,7 +295,7 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> conf = link.get("confidence", "EXTRACTED") link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - with open(output_path, "w") as f: + with open(output_path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) @@ -322,7 +322,7 @@ def to_cypher(G: nx.Graph, output_path: str) -> None: f"MATCH (a {{id: '{u_esc}'}}), (b {{id: '{v_esc}'}}) " f"MERGE (a)-[:{rel} {{confidence: '{conf}'}}]->(b);" ) - with open(output_path, "w") as f: + with open(output_path, "w", encoding="utf-8") as f: f.write("\n".join(lines)) @@ -681,7 +681,7 @@ def _community_reach(node_id: str) -> int: for cid, label in sorted((community_labels or {}).items()) ] } - (obsidian_dir / "graph.json").write_text(json.dumps(graph_config, indent=2)) + (obsidian_dir / "graph.json").write_text(json.dumps(graph_config, indent=2), encoding="utf-8") return G.number_of_nodes() + community_notes_written diff --git a/graphify/hooks.py b/graphify/hooks.py index 92320272b..f32e1c7f9 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -119,12 +119,12 @@ def _install_hook(hooks_dir: Path, name: str, script: str, marker: str) -> str: """Install a single git hook, appending if an existing hook is present.""" hook_path = hooks_dir / name if hook_path.exists(): - content = hook_path.read_text() + content = hook_path.read_text(encoding="utf-8") if marker in content: return f"already installed at {hook_path}" - hook_path.write_text(content.rstrip() + "\n\n" + script) + hook_path.write_text(content.rstrip() + "\n\n" + script, encoding="utf-8", newline="\n") return f"appended to existing {name} hook at {hook_path}" - hook_path.write_text("#!/bin/sh\n" + script) + hook_path.write_text("#!/bin/sh\n" + script, encoding="utf-8", newline="\n") hook_path.chmod(0o755) return f"installed at {hook_path}" @@ -134,7 +134,7 @@ def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> hook_path = hooks_dir / name if not hook_path.exists(): return f"no {name} hook found - nothing to remove." - content = hook_path.read_text() + content = hook_path.read_text(encoding="utf-8") if marker not in content: return f"graphify hook not found in {name} - nothing to remove." new_content = re.sub( @@ -146,7 +146,7 @@ def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> if not new_content or new_content in ("#!/bin/bash", "#!/bin/sh"): hook_path.unlink() return f"removed {name} hook at {hook_path}" - hook_path.write_text(new_content + "\n") + hook_path.write_text(new_content + "\n", encoding="utf-8", newline="\n") return f"graphify removed from {name} at {hook_path} (other hook content preserved)" @@ -189,7 +189,7 @@ def _check(name: str, marker: str) -> str: p = hooks_dir / name if not p.exists(): return "not installed" - return "installed" if marker in p.read_text() else "not installed (hook exists but graphify not found)" + return "installed" if marker in p.read_text(encoding="utf-8") else "not installed (hook exists but graphify not found)" commit = _check("post-commit", _HOOK_MARKER) checkout = _check("post-checkout", _CHECKOUT_MARKER) diff --git a/graphify/serve.py b/graphify/serve.py index 81c9353ab..279b5d316 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -16,7 +16,7 @@ def _load_graph(graph_path: str) -> nx.Graph: if not resolved.exists(): raise FileNotFoundError(f"Graph file not found: {resolved}") safe = resolved - data = json.loads(safe.read_text()) + data = json.loads(safe.read_text(encoding="utf-8")) try: return json_graph.node_link_graph(data, edges="links") except TypeError: diff --git a/graphify/watch.py b/graphify/watch.py index 734de8bf0..df2871f6f 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -53,7 +53,7 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: report = generate(G, communities, cohesion, labels, gods, surprises, detection, {"input": 0, "output": 0}, str(watch_path), suggested_questions=questions) - (out / "GRAPH_REPORT.md").write_text(report) + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") to_json(G, communities, str(out / "graph.json")) # clear stale needs_update flag if present @@ -75,7 +75,7 @@ def _notify_only(watch_path: Path) -> None: """Write a flag file and print a notification (fallback for non-code-only corpora).""" flag = watch_path / "graphify-out" / "needs_update" flag.parent.mkdir(parents=True, exist_ok=True) - flag.write_text("1") + flag.write_text("1", encoding="utf-8") print(f"\n[graphify watch] New or changed files detected in {watch_path}") print("[graphify watch] Non-code files changed - semantic re-extraction requires LLM.") print("[graphify watch] Run `/graphify --update` in Claude Code to update the graph.") diff --git a/graphify/wiki.py b/graphify/wiki.py index 898a8ec5f..70075fafb 100644 --- a/graphify/wiki.py +++ b/graphify/wiki.py @@ -195,7 +195,7 @@ def to_wiki( for cid, nodes in communities.items(): label = labels.get(cid, f"Community {cid}") article = _community_article(G, cid, nodes, label, labels, cohesion.get(cid)) - (out / f"{_safe_filename(label)}.md").write_text(article) + (out / f"{_safe_filename(label)}.md").write_text(article, encoding="utf-8") count += 1 # God node articles @@ -203,12 +203,13 @@ def to_wiki( nid = node_data.get("id") if nid and nid in G: article = _god_node_article(G, nid, labels) - (out / f"{_safe_filename(node_data['label'])}.md").write_text(article) + (out / f"{_safe_filename(node_data['label'])}.md").write_text(article, encoding="utf-8") count += 1 # Index (out / "index.md").write_text( - _index_md(communities, labels, god_nodes_data, G.number_of_nodes(), G.number_of_edges()) + _index_md(communities, labels, god_nodes_data, G.number_of_nodes(), G.number_of_edges()), + encoding="utf-8", ) return count diff --git a/tests/test_encoding_roundtrip.py b/tests/test_encoding_roundtrip.py new file mode 100644 index 000000000..76f02ebdc --- /dev/null +++ b/tests/test_encoding_roundtrip.py @@ -0,0 +1,180 @@ +"""Regression tests for Windows encoding fixes. + +Verifies that CJK characters and emoji in graph labels survive a write/read +round-trip without UnicodeEncodeError or data loss. + +These tests exercise the exact IO patterns that were fixed: + - open(path, "w", encoding="utf-8") -- export.py to_json / to_cypher + - Path.write_text(..., encoding="utf-8") -- cache.py, detect.py, wiki.py, watch.py + - Path.read_text(encoding="utf-8") -- cache.py, serve.py, benchmark.py + +To confirm the tests catch the bug on an un-patched build, revert e.g. +cache.py line 73 from: + tmp.write_text(json.dumps(result), encoding="utf-8") +to: + tmp.write_text(json.dumps(result)) +then run under a CP1252 locale with ensure_ascii=False data - the write will +raise UnicodeEncodeError. The hook LF test would also fail on Windows because +the bare write_text would convert \\n to \\r\\n. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Shared test data +# --------------------------------------------------------------------------- + +CJK_LABEL = "変換器_トランスフォーマー" # Japanese +KOREAN_LABEL = "주의 메커니즘" # Korean +CHINESE_LABEL = "注意力機制" # Traditional Chinese +EMOJI_LABEL = "graph\U0001f4ca node" # 📊 U+1F4CA + + +# --------------------------------------------------------------------------- +# Primitive write_text / read_text with CJK (baseline IO pattern) +# --------------------------------------------------------------------------- + +def test_write_read_text_cjk_roundtrip_primitive(tmp_path): + """Baseline: Path.write_text + read_text with explicit encoding='utf-8' + must survive CJK + emoji labels with ensure_ascii=False JSON payloads. + + This is the raw IO pattern that all graphify file writes now use. On a + system where the default encoding is not UTF-8 (Windows CP1252/CP932/CP950), + omitting encoding='utf-8' would raise UnicodeEncodeError here. + """ + payload = { + "nodes": [ + {"id": "n1", "label": CJK_LABEL}, + {"id": "n2", "label": EMOJI_LABEL}, + {"id": "n3", "label": KOREAN_LABEL}, + {"id": "n4", "label": CHINESE_LABEL}, + ], + "edges": [], + "hyperedges": [], + } + json_str = json.dumps(payload, ensure_ascii=False, indent=2) + + out_file = tmp_path / "payload.json" + out_file.write_text(json_str, encoding="utf-8") + + recovered = json.loads(out_file.read_text(encoding="utf-8")) + labels = {n["label"] for n in recovered["nodes"]} + + assert CJK_LABEL in labels, f"CJK label lost: {labels}" + assert EMOJI_LABEL in labels, f"Emoji label lost: {labels}" + assert KOREAN_LABEL in labels, f"Korean label lost: {labels}" + assert CHINESE_LABEL in labels, f"Chinese label lost: {labels}" + + +def test_cache_save_load_uses_utf8(tmp_path): + """Verify that graphify.cache save_cached / load_cached use utf-8 internally.""" + from graphify.cache import save_cached, load_cached + + src_file = tmp_path / "source.py" + src_file.write_bytes(b"# source\n") + + # Build a result dict. json.dumps(ensure_ascii=True) will escape non-ASCII + # so the bug is hidden for standard usage. But the file must still be readable + # as UTF-8 (the default json.loads handles both escaped and raw forms). + result = { + "nodes": [{"id": "n1", "label": CJK_LABEL}], + "edges": [], + "hyperedges": [], + } + save_cached(src_file, result, root=tmp_path) + loaded = load_cached(src_file, root=tmp_path) + + assert loaded is not None, "load_cached returned None - cache miss after save" + assert loaded["nodes"][0]["label"] == CJK_LABEL + + +# --------------------------------------------------------------------------- +# export.to_json IO pattern (without networkx, test the open() layer directly) +# --------------------------------------------------------------------------- + +def test_to_json_open_pattern_cjk(tmp_path): + """The open(path, 'w', encoding='utf-8') pattern used in to_json must handle CJK. + + This replicates what to_json does after the fix: + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2) + We patch ensure_ascii=False to prove the bytes are UTF-8. + """ + out = tmp_path / "graph.json" + data = { + "nodes": [ + {"id": "n1", "label": CJK_LABEL}, + {"id": "n2", "label": EMOJI_LABEL}, + ], + "links": [], + } + # Reproduce fixed export.py pattern + with open(str(out), "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + raw = out.read_bytes() + assert CJK_LABEL.encode("utf-8") in raw, "CJK bytes not found in output file" + assert "\U0001f4ca".encode("utf-8") in raw, "Emoji bytes not found in output file" + + recovered = json.loads(raw.decode("utf-8")) + assert recovered["nodes"][0]["label"] == CJK_LABEL + assert recovered["nodes"][1]["label"] == EMOJI_LABEL + + +# --------------------------------------------------------------------------- +# hooks._install_hook — must write LF, not CRLF +# --------------------------------------------------------------------------- + +def test_hook_install_lf_only(tmp_path): + """Git hooks written by graphify must use LF line endings, not CRLF. + + A CRLF shebang line (#!/bin/sh\\r) causes 'bad interpreter' on Unix/WSL. + This test will fail on an un-patched codebase on Windows because + Path.write_text() without newline='\\n' converts \\n to \\r\\n. + """ + from graphify.hooks import _install_hook + + hooks_dir = tmp_path / ".git" / "hooks" + hooks_dir.mkdir(parents=True) + + marker = "# graphify-test-marker" + script = f"{marker}\necho hello\n# graphify-test-marker-end\n" + + _install_hook(hooks_dir, "post-commit", script, marker) + + hook_file = hooks_dir / "post-commit" + raw_bytes = hook_file.read_bytes() + + assert b"\r\n" not in raw_bytes, ( + f"Hook file contains CRLF line endings - will break sh interpreter on Unix/WSL. " + f"First 80 bytes: {raw_bytes[:80]!r}" + ) + assert raw_bytes.startswith(b"#!/bin/sh\n"), ( + f"Shebang line must be '#!/bin/sh\\n', got: {raw_bytes[:20]!r}" + ) + + +def test_hook_append_lf_only(tmp_path): + """Appending to an existing hook must also produce LF-only output.""" + from graphify.hooks import _install_hook + + hooks_dir = tmp_path / ".git" / "hooks" + hooks_dir.mkdir(parents=True) + + # Create a pre-existing hook (with LF endings, as Unix would have) + existing = hooks_dir / "post-commit" + existing.write_bytes(b"#!/bin/sh\n# existing hook\nexit 0\n") + + marker = "# graphify-append-marker" + script = f"{marker}\npython -m graphify --hook\n# graphify-append-marker-end\n" + + _install_hook(hooks_dir, "post-commit", script, marker) + + raw_bytes = existing.read_bytes() + assert b"\r\n" not in raw_bytes, ( + f"Appended hook file contains CRLF. First 120 bytes: {raw_bytes[:120]!r}" + )