From 785c9f4dd731eb86f3a243f5d2e8b13bb54f3a02 Mon Sep 17 00:00:00 2001 From: Minidoracat Date: Thu, 9 Apr 2026 02:39:43 +0800 Subject: [PATCH 01/59] fix: detect correct Python interpreter for pipx installs in git hooks * fix: git hooks fail when graphify is installed via pipx When installed via pipx, the graphify module is only available in pipx's isolated venv, not the system python3. The git hooks (post-commit, post-checkout) hardcoded `python3` which cannot import graphify in this case. Detect the correct Python interpreter from the graphify binary's shebang line, matching the approach already used in skill.md Step 1. Falls back to python3 for system installs. * fix: handle env-style shebangs and improve interpreter detection - Use POSIX `command -v` instead of non-standard `which` - Parse `#!/usr/bin/env python3` shebangs correctly (previous `tr -d ' '` would produce `/usr/bin/envpython3`) - Add import validation fallback to python3 if resolved interpreter cannot import graphify --- graphify/hooks.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/graphify/hooks.py b/graphify/hooks.py index dc5e80338..51959ffb9 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -8,6 +8,23 @@ _CHECKOUT_MARKER = "# graphify-checkout-hook-start" _CHECKOUT_MARKER_END = "# graphify-checkout-hook-end" +_PYTHON_DETECT = """\ +# Detect the correct Python interpreter (handles pipx, venv, system installs) +GRAPHIFY_BIN=$(command -v graphify 2>/dev/null) +if [ -n "$GRAPHIFY_BIN" ]; then + _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | sed 's/^#![[:space:]]*//') + case "$_SHEBANG" in + */env\\ *) GRAPHIFY_PYTHON="${_SHEBANG#*/env }" ;; + *) GRAPHIFY_PYTHON="$_SHEBANG" ;; + esac + if ! "$GRAPHIFY_PYTHON" -c "import graphify" 2>/dev/null; then + GRAPHIFY_PYTHON="python3" + fi +else + GRAPHIFY_PYTHON="python3" +fi +""" + _HOOK_SCRIPT = """\ # graphify-hook-start # Auto-rebuilds the knowledge graph after each commit (code files only, no LLM needed). @@ -18,8 +35,9 @@ exit 0 fi +""" + _PYTHON_DETECT + """ export GRAPHIFY_CHANGED="$CHANGED" -python3 -c " +$GRAPHIFY_PYTHON -c " import os, sys from pathlib import Path @@ -67,8 +85,9 @@ exit 0 fi +""" + _PYTHON_DETECT + """ echo "[graphify] Branch switched - rebuilding knowledge graph (code files)..." -python3 -c " +$GRAPHIFY_PYTHON -c " from graphify.watch import _rebuild_code from pathlib import Path import sys From 8a0dd457112437de558898eccd2984858c0163bd Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Wed, 8 Apr 2026 19:40:10 +0100 Subject: [PATCH 02/59] fix: suppress graspologic ANSI output that corrupts PowerShell scroll buffer * fix: suppress graspologic ANSI output that breaks PowerShell scrolling graspologic's leiden() emits ANSI escape sequences (progress bars, colored warnings) that corrupt PowerShell 5.1's scroll buffer on Windows, disabling vertical scrolling. Redirect stdout/stderr to StringIO during leiden() calls to prevent any escape codes from reaching the terminal. Add 2 tests verifying cluster() produces no stdout/stderr output. Fixes #19 Co-Authored-By: Claude Opus 4.6 * docs: add PowerShell troubleshooting section to Windows skill Document the PowerShell 5.1 scrolling issue and provide 4 workarounds: upgrade graphify, use Windows Terminal, reset terminal, or uninstall graspologic to use Louvain fallback. Fixes #19 Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- graphify/cluster.py | 29 ++++++++++++++++++++++++++++- graphify/skill-windows.md | 13 +++++++++++++ tests/test_cluster.py | 24 ++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/graphify/cluster.py b/graphify/cluster.py index 7019cdecc..db7709af0 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -1,17 +1,44 @@ """Community detection on NetworkX graphs. Uses Leiden (graspologic) if available, falls back to Louvain (networkx). Splits oversized communities. Returns cohesion scores.""" from __future__ import annotations +import contextlib +import io +import os +import sys import networkx as nx +def _suppress_output(): + """Context manager to suppress stdout/stderr during library calls. + + graspologic's leiden() emits ANSI escape sequences (progress bars, + colored warnings) that corrupt PowerShell 5.1's scroll buffer on + Windows (see issue #19). Redirecting stdout/stderr to devnull during + the call prevents this without losing any graphify output. + """ + return contextlib.redirect_stdout(io.StringIO()) + + def _partition(G: nx.Graph) -> dict[str, int]: """Run community detection. Returns {node_id: community_id}. Tries Leiden (graspologic) first — best quality. Falls back to Louvain (built into networkx) if graspologic is not installed. + + Output from graspologic is suppressed to prevent ANSI escape codes + from corrupting terminal scroll buffers on Windows PowerShell 5.1. """ try: from graspologic.partition import leiden - return leiden(G) + # Suppress graspologic output to prevent ANSI escape codes from + # corrupting PowerShell 5.1 scroll buffer (issue #19) + old_stderr = sys.stderr + try: + sys.stderr = io.StringIO() + with _suppress_output(): + result = leiden(G) + finally: + sys.stderr = old_stderr + return result except ImportError: pass diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 6675a7697..e8f9c9de1 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -1202,6 +1202,19 @@ graphify claude uninstall # remove the section --- +## Troubleshooting + +### PowerShell 5.1: Vertical scrolling stops working + +If vertical scrolling breaks in PowerShell after running graphify, this is caused by ANSI escape sequences from the `graspologic` library. Graphify v0.3.10+ suppresses this output, but if you still see the issue: + +1. **Upgrade graphify**: `pip install --upgrade graphifyy` +2. **Use Windows Terminal** instead of the legacy PowerShell console — Windows Terminal handles ANSI codes correctly +3. **Reset your terminal**: close and reopen PowerShell +4. **Skip graspologic**: uninstall it (`pip uninstall graspologic`) and graphify will fall back to NetworkX's built-in Louvain algorithm, which produces no ANSI output + +--- + ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. diff --git a/tests/test_cluster.py b/tests/test_cluster.py index de534016f..b5c16fad6 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,4 +1,5 @@ import json +import sys import networkx as nx from pathlib import Path from graphify.build import build_from_json @@ -50,3 +51,26 @@ def test_score_all_keys_match_communities(): communities = cluster(G) scores = score_all(G, communities) assert set(scores.keys()) == set(communities.keys()) + + +def test_cluster_does_not_write_to_stdout(capsys): + """Clustering should not emit ANSI escape codes or other output. + + graspologic's leiden() can emit ANSI escape sequences that break + PowerShell 5.1's scroll buffer on Windows (issue #19). The output + suppression in _partition() should prevent any output from leaking. + """ + G = make_graph() + cluster(G) + captured = capsys.readouterr() + assert captured.out == "", f"cluster() wrote to stdout: {captured.out!r}" + + +def test_cluster_does_not_write_to_stderr(capsys): + """Same as above but for stderr — ANSI codes can go to either stream.""" + G = make_graph() + cluster(G) + captured = capsys.readouterr() + # Allow logging output (starts with [graphify]) but no raw ANSI codes + for line in captured.err.splitlines(): + assert "\x1b" not in line, f"cluster() wrote ANSI to stderr: {line!r}" From 312ba0dd1eca6780ea83523c0da701f69d16bd6d Mon Sep 17 00:00:00 2001 From: gdesai23 Date: Thu, 9 Apr 2026 00:10:29 +0530 Subject: [PATCH 03/59] docs: add graph.json + LLM workflow example to README --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 54230f72e..2061f5ba6 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,42 @@ The always-on hook surfaces `GRAPH_REPORT.md` — a one-page summary of god node Think of it this way: the always-on hook gives your assistant a map. The `/graphify` commands let it navigate the map precisely. +## Using `graph.json` with an LLM + +`graph.json` is not meant to be pasted into a prompt all at once. The useful +workflow is: + +1. Start with `graphify-out/GRAPH_REPORT.md` for the high-level overview. +2. Use `graphify query` to pull a smaller subgraph for the specific question + you want to answer. +3. Give that focused output to your assistant instead of dumping the full raw + corpus. + +For example, after running graphify on a project: + +```bash +graphify query "show the auth flow" --graph graphify-out/graph.json +graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json +``` + +The output includes node labels, edge types, confidence tags, source files, and +source locations. That makes it a good intermediate context block for an LLM: + +```text +Use this graph query output to answer the question. Prefer the graph structure +over guessing, and cite the source files when possible. +``` + +If your assistant supports tool calling or MCP, use the graph directly instead +of pasting text. graphify can expose `graph.json` as an MCP server: + +```bash +python -m graphify.serve graphify-out/graph.json +``` + +That gives the assistant structured graph access for repeated queries such as +`query_graph`, `get_node`, `get_neighbors`, and `shortest_path`. +
Manual install (curl) From 74e192db00ec48f95b8f989690ed9ff3b7dd3bfd Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Thu, 9 Apr 2026 03:40:33 +0900 Subject: [PATCH 04/59] docs: add Japanese README --- README.ja-JP.md | 238 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 +- README.zh-CN.md | 2 +- 3 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 README.ja-JP.md diff --git a/README.ja-JP.md b/README.ja-JP.md new file mode 100644 index 000000000..9924b4629 --- /dev/null +++ b/README.ja-JP.md @@ -0,0 +1,238 @@ +# graphify + +[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) + +[![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) +[![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) + +**AIコーディングアシスタント向けのスキル。** Claude Code、Codex、OpenCode、OpenClaw、Factory Droid で `/graphify` と入力するだけで、ファイルを読み込んでナレッジグラフを構築し、あなたが気づいていなかった構造を返します。コードベースをより速く理解し、アーキテクチャ上の意思決定の「なぜ」を見つけ出します。 + +完全にマルチモーダル対応。コード、PDF、Markdown、スクリーンショット、図、ホワイトボード写真、他言語の画像まで――graphify は Claude Vision を使ってそれらすべてから概念と関係性を抽出し、1 つのグラフに接続します。tree-sitter AST により 19 言語をサポート（Python、JS、TS、Go、Rust、Java、C、C++、Ruby、C#、Kotlin、Scala、PHP、Swift、Lua、Zig、PowerShell、Elixir、Objective-C）。 + +> Andrej Karpathy は論文、ツイート、スクリーンショット、メモを放り込む `/raw` フォルダを持っています。graphify はまさにその問題への答えです――生ファイルを読むのに比べて1クエリあたりのトークン数が 71.5 倍少なく、セッションをまたいで永続化され、見つけたものと推測したものを正直に区別します。 + +``` +/graphify . # どのフォルダでも動作 - コードベース、メモ、論文、なんでも +``` + +``` +graphify-out/ +├── graph.html インタラクティブなグラフ - ノードをクリック、検索、コミュニティでフィルタ +├── GRAPH_REPORT.md ゴッドノード、意外なつながり、推奨される質問 +├── graph.json 永続化されたグラフ - 数週間後でも再読み込みなしでクエリ可能 +└── cache/ SHA256 キャッシュ - 再実行時は変更されたファイルのみ処理 +``` + +グラフに含めたくないフォルダを除外するには `.graphifyignore` ファイルを追加します： + +``` +# .graphifyignore +vendor/ +node_modules/ +dist/ +*.generated.py +``` + +構文は `.gitignore` と同じです。パターンは graphify を実行したフォルダからの相対パスに対してマッチします。 + +## 仕組み + +graphify は 2 パスで動作します。まず、決定論的な AST パスがコードファイルから構造（クラス、関数、インポート、コールグラフ、docstring、根拠コメント）を LLM なしで抽出します。次に、Claude サブエージェントがドキュメント、論文、画像に対して並列に実行され、概念、関係性、設計の根拠を抽出します。結果は NetworkX グラフにマージされ、Leiden コミュニティ検出でクラスタリングされ、インタラクティブ HTML、クエリ可能な JSON、平易な言葉の監査レポートとしてエクスポートされます。 + +**クラスタリングはグラフトポロジベース――埋め込みは使いません。** Leiden はエッジ密度によってコミュニティを見つけます。Claude が抽出する意味的類似性エッジ（`semantically_similar_to`、INFERRED とマーク）は既にグラフに含まれているため、コミュニティ検出に直接影響します。グラフ構造そのものが類似性シグナルであり――別途の埋め込みステップやベクターデータベースは不要です。 + +すべての関係は `EXTRACTED`（ソースから直接見つかった）、`INFERRED`（合理的な推論、信頼度スコア付き）、`AMBIGUOUS`（レビュー対象としてフラグ付け）のいずれかでタグ付けされます。何が見つかったもので何が推測されたものか、常に分かります。 + +## インストール + +**必要なもの:** Python 3.10+ および以下のいずれか： [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [OpenClaw](https://openclaw.ai), または [Factory Droid](https://factory.ai) + +```bash +pip install graphifyy && graphify install +``` + +> PyPI パッケージは `graphify` の名前が再取得されるまでの間、一時的に `graphifyy` となっています。CLI とスキルコマンドは依然として `graphify` です。 + +### プラットフォームサポート + +| プラットフォーム | インストールコマンド | +|----------|----------------| +| Claude Code (Linux/Mac) | `graphify install` | +| Claude Code (Windows) | `graphify install`（自動検出）または `graphify install --platform windows` | +| Codex | `graphify install --platform codex` | +| OpenCode | `graphify install --platform opencode` | +| OpenClaw | `graphify install --platform claw` | +| Factory Droid | `graphify install --platform droid` | + +Codex ユーザーは並列抽出のために `~/.codex/config.toml` の `[features]` の下に `multi_agent = true` も必要です。Factory Droid は並列サブエージェントディスパッチに `Task` ツールを使用します。OpenClaw は逐次抽出を使用します（並列エージェントサポートはこのプラットフォームではまだ初期段階です）。 + +次に、AI コーディングアシスタントを開いて入力します： + +``` +/graphify . +``` + +注意：Codex はスキル呼び出しに `/` ではなく `$` を使用するため、代わりに `$graphify .` と入力してください。 + +### アシスタントに常にグラフを使わせる（推奨） + +グラフを構築した後、プロジェクトで一度だけ以下を実行します： + +| プラットフォーム | コマンド | +|----------|---------| +| Claude Code | `graphify claude install` | +| Codex | `graphify codex install` | +| OpenCode | `graphify opencode install` | +| OpenClaw | `graphify claw install` | +| Factory Droid | `graphify droid install` | + +**Claude Code** は 2 つのことを行います：Claude にアーキテクチャの質問に答える前に `graphify-out/GRAPH_REPORT.md` を読むように指示する `CLAUDE.md` セクションを書き込み、すべての Glob と Grep 呼び出しの前に発火する **PreToolUse フック**（`settings.json`）をインストールします。ナレッジグラフが存在する場合、Claude は次のメッセージを見ます：_"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ ――これにより Claude はすべてのファイルを grep するのではなく、グラフを介してナビゲートします。 + +**Codex、OpenCode、OpenClaw、Factory Droid** は同じルールをプロジェクトルートの `AGENTS.md` に書き込みます。これらのプラットフォームは PreToolUse フックをサポートしていないため、AGENTS.md が常時有効のメカニズムとなります。 + +アンインストールは対応するアンインストールコマンドで行います（例：`graphify claude uninstall`）。 + +**常時有効 vs 明示的トリガー――何が違うのか？** + +常時有効のフックは `GRAPH_REPORT.md` を表面化します――これはゴッドノード、コミュニティ、意外なつながりを 1 ページにまとめた要約です。アシスタントはファイル検索の前にこれを読み、キーワードマッチではなく構造に基づいてナビゲートします。これで日常的な質問のほとんどをカバーできます。 + +`/graphify query`、`/graphify path`、`/graphify explain` はさらに深く踏み込みます：生の `graph.json` をホップごとに辿り、ノード間の正確なパスをトレースし、エッジレベルの詳細（関係タイプ、信頼度スコア、ソース位置）を表面化します。一般的なオリエンテーションではなく、特定の質問をグラフから答えさせたいときに使います。 + +こう考えてください：常時有効のフックはアシスタントに地図を与え、`/graphify` コマンドはその地図を正確にナビゲートさせます。 + +
+手動インストール（curl） + +```bash +mkdir -p ~/.claude/skills/graphify +curl -fsSL https://raw.githubusercontent.com/safishamsi/graphify/v3/graphify/skill.md \ + > ~/.claude/skills/graphify/SKILL.md +``` + +`~/.claude/CLAUDE.md` に追加します： + +``` +- **graphify** (`~/.claude/skills/graphify/SKILL.md`) - any input to knowledge graph. Trigger: `/graphify` +When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"` before doing anything else. +``` + +
+ +## 使い方 + +``` +/graphify # カレントディレクトリで実行 +/graphify ./raw # 特定のフォルダで実行 +/graphify ./raw --mode deep # より積極的な INFERRED エッジ抽出 +/graphify ./raw --update # 変更されたファイルのみ再抽出し、既存グラフにマージ +/graphify ./raw --cluster-only # 既存グラフのクラスタリングを再実行（再抽出なし） +/graphify ./raw --no-viz # HTML をスキップ、レポート + JSON のみ生成 +/graphify ./raw --obsidian # Obsidian ボールトも生成（オプトイン） +/graphify ./raw --obsidian --obsidian-dir ~/vaults/myproject # ボールトを特定のディレクトリに書き込み + +/graphify add https://arxiv.org/abs/1706.03762 # 論文を取得、保存、グラフを更新 +/graphify add https://x.com/karpathy/status/... # ツイートを取得 +/graphify add https://... --author "Name" # 元の著者をタグ付け +/graphify add https://... --contributor "Name" # コーパスに追加した人をタグ付け + +/graphify query "アテンションとオプティマイザを結ぶものは？" +/graphify query "アテンションとオプティマイザを結ぶものは？" --dfs # 特定のパスをトレース +/graphify query "アテンションとオプティマイザを結ぶものは？" --budget 1500 # N トークンで上限設定 +/graphify path "DigestAuth" "Response" +/graphify explain "SwinTransformer" + +/graphify ./raw --watch # ファイル変更時にグラフを自動同期（コード：即時、ドキュメント：通知） +/graphify ./raw --wiki # エージェントがクロール可能な wiki を構築（index.md + コミュニティごとの記事） +/graphify ./raw --svg # graph.svg をエクスポート +/graphify ./raw --graphml # graph.graphml をエクスポート（Gephi、yEd） +/graphify ./raw --neo4j # Neo4j 用の cypher.txt を生成 +/graphify ./raw --neo4j-push bolt://localhost:7687 # 実行中の Neo4j インスタンスに直接プッシュ +/graphify ./raw --mcp # MCP stdio サーバーを起動 + +# git フック - プラットフォーム非依存、コミット時とブランチ切り替え時にグラフを再構築 +graphify hook install +graphify hook uninstall +graphify hook status + +# 常時有効のアシスタント指示 - プラットフォーム固有 +graphify claude install # CLAUDE.md + PreToolUse フック（Claude Code） +graphify claude uninstall +graphify codex install # AGENTS.md（Codex） +graphify opencode install # AGENTS.md（OpenCode） +graphify claw install # AGENTS.md（OpenClaw） +graphify droid install # AGENTS.md（Factory Droid） + +# ターミナルから直接グラフをクエリ（AI アシスタント不要） +graphify query "アテンションとオプティマイザを結ぶものは？" +graphify query "認証フローを表示" --dfs +graphify query "CfgNode とは？" --budget 500 +graphify query "..." --graph path/to/graph.json +``` + +あらゆるファイルタイプの組み合わせで動作します： + +| タイプ | 拡張子 | 抽出方法 | +|------|-----------|------------| +| コード | `.py .ts .js .go .rs .java .c .cpp .rb .cs .kt .scala .php .swift .lua .zig .ps1 .ex .exs .m .mm` | tree-sitter による AST + コールグラフ + docstring/コメントの根拠 | +| ドキュメント | `.md .txt .rst` | Claude による概念 + 関係性 + 設計根拠 | +| Office | `.docx .xlsx` | Markdown に変換した後 Claude で抽出（`pip install graphifyy[office]` が必要） | +| 論文 | `.pdf` | 引用マイニング + 概念抽出 | +| 画像 | `.png .jpg .webp .gif` | Claude Vision - スクリーンショット、図、任意の言語 | + +## 得られるもの + +**ゴッドノード** - 最高次数の概念（すべてが接続するもの） + +**意外なつながり** - 複合スコアでランク付け。コード-論文のエッジはコード-コードよりも高くランクされます。各結果には平易な英語の理由が含まれます。 + +**推奨される質問** - グラフがユニークに答えられる 4〜5 の質問 + +**「なぜ」** - docstring、インラインコメント（`# NOTE:`、`# IMPORTANT:`、`# HACK:`、`# WHY:`）、ドキュメントからの設計根拠が `rationale_for` ノードとして抽出されます。コードが何をするかだけでなく――なぜそのように書かれたか。 + +**信頼度スコア** - すべての INFERRED エッジには `confidence_score`（0.0〜1.0）があります。何が推測されたかだけでなく、モデルがどれだけ確信していたかもわかります。EXTRACTED エッジは常に 1.0 です。 + +**意味的類似性エッジ** - 構造的接続のないクロスファイル概念リンク。互いを呼び出さずに同じ問題を解いている 2 つの関数、同じアルゴリズムを記述しているコード内のクラスと論文内の概念など。 + +**ハイパーエッジ** - ペアワイズエッジでは表現できない 3+ ノードを接続するグループ関係。共有プロトコルを実装するすべてのクラス、認証フロー内のすべての関数、論文セクションから 1 つのアイデアを形成するすべての概念など。 + +**トークンベンチマーク** - 実行ごとに自動的に出力されます。混合コーパス（Karpathy リポジトリ + 論文 + 画像）で、生ファイルを読むのに比べて 1 クエリあたり **71.5 倍** 少ないトークン。最初の実行で抽出とグラフ構築を行います（これにはトークンがかかります）。以降のクエリはすべて生ファイルではなくコンパクトなグラフを読みます――ここで節約が複利的に効いてきます。SHA256 キャッシュにより、再実行時は変更されたファイルのみ再処理されます。 + +**自動同期** (`--watch`) - バックグラウンドターミナルで実行し、コードベースが変更されるとグラフが自動的に更新されます。コードファイルの保存は即座の再構築をトリガーします（AST のみ、LLM なし）。ドキュメント/画像の変更は、LLM の再パスのために `--update` を実行するよう通知します。 + +**Git フック** (`graphify hook install`) - post-commit と post-checkout フックをインストールします。コミットごと、ブランチ切り替えごとにグラフが自動的に再構築されます。再構築が失敗した場合、フックは非ゼロコードで終了するため、git がエラーを表面化し、静かに続行することはありません。バックグラウンドプロセスは不要です。 + +**Wiki** (`--wiki`) - コミュニティごとおよびゴッドノードごとの Wikipedia スタイルの Markdown 記事と、`index.md` エントリポイント。任意のエージェントを `index.md` に向ければ、JSON をパースする代わりにファイルを読むことでナレッジベースをナビゲートできます。 + +## 実例 + +| コーパス | ファイル数 | 削減率 | 出力 | +|--------|-------|-----------|--------| +| Karpathy リポジトリ + 論文5本 + 画像4枚 | 52 | **71.5x** | [`worked/karpathy-repos/`](worked/karpathy-repos/) | +| graphify ソース + Transformer 論文 | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | +| httpx（合成 Python ライブラリ） | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | + +トークン削減はコーパスサイズに応じてスケールします。6 ファイルはいずれにせよコンテキストウィンドウに収まるため、そこでのグラフの価値は圧縮ではなく構造的明瞭さです。52 ファイル（コード + 論文 + 画像）では 71 倍以上が得られます。各 `worked/` フォルダには生の入力ファイルと実際の出力（`GRAPH_REPORT.md`、`graph.json`）があり、自分で実行して数字を検証できます。 + +## プライバシー + +graphify はドキュメント、論文、画像の意味的抽出のために、ファイル内容を AI コーディングアシスタントの基盤モデル API に送信します――Anthropic（Claude Code）、OpenAI（Codex）、またはプラットフォームが使用するプロバイダーです。コードファイルは tree-sitter AST を介してローカルで処理されます――コードに関してはファイル内容がマシンから出ることはありません。テレメトリ、利用追跡、分析は一切ありません。ネットワーク呼び出しは抽出中のプラットフォームのモデル API への呼び出しのみで、あなた自身の API キーを使用します。 + +## 技術スタック + +NetworkX + Leiden（graspologic） + tree-sitter + vis.js。意味的抽出は Claude（Claude Code）、GPT-4（Codex）、またはプラットフォームが実行するモデルを介して行われます。Neo4j は不要、サーバーも不要、完全にローカルで実行されます。 + +## スター履歴 + +[![Star History Chart](https://api.star-history.com/svg?repos=safishamsi/graphify&type=Date)](https://star-history.com/#safishamsi/graphify&Date) + +
+コントリビューション + +**実例** は最も信頼を築くコントリビューションです。実際のコーパスで `/graphify` を実行し、出力を `worked/{slug}/` に保存し、グラフが正しく捉えたもの・間違えたものを評価する正直な `review.md` を書き、PR を提出してください。 + +**抽出バグ** - 入力ファイル、キャッシュエントリ（`graphify-out/cache/`）、何が見逃された/捏造されたかを添えて issue を開いてください。 + +モジュールの責任と言語の追加方法については [ARCHITECTURE.md](ARCHITECTURE.md) を参照してください。 + +
diff --git a/README.md b/README.md index 2061f5ba6..111d6dd36 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # graphify -[English](README.md) | [简体中文](README.zh-CN.md) +[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) diff --git a/README.zh-CN.md b/README.zh-CN.md index b1639e5c8..113e290df 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -1,6 +1,6 @@ # graphify -[English](README.md) | [简体中文](README.zh-CN.md) +[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) From f4c3c551e19eedf720eaf3ae35e779edebd61479 Mon Sep 17 00:00:00 2001 From: ljinshuan Date: Thu, 9 Apr 2026 02:44:19 +0800 Subject: [PATCH 05/59] feat: add Trae and Trae CN platform support - Register 'trae' and 'trae-cn' in _PLATFORM_CONFIG (skill-trae.md, ~/.trae/skills/ and ~/.trae-cn/skills/, claude_md=False) - Add CLI subcommands: graphify trae install/uninstall, graphify trae-cn install/uninstall (routes to _agents_install/uninstall) - Update help text with new platform entries - Create skill-trae.md (Agent-tool based extraction, AGENTS.md integration, no PreToolUse hook support per Trae limitations) - Update README.md and README.zh-CN.md with Trae platform docs Co-authored-by: lijinshuan --- .gitignore | 3 + README.md | 16 +- README.zh-CN.md | 19 +- graphify/__main__.py | 20 +- graphify/skill-trae.md | 1187 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1234 insertions(+), 11 deletions(-) create mode 100644 graphify/skill-trae.md diff --git a/.gitignore b/.gitignore index a524bf7a1..cc8adc89d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ graphify-out/ .claude/ skills/ docs/superpowers/ +.vscode/ +openspec/ +uv.lock \ No newline at end of file diff --git a/README.md b/README.md index 111d6dd36..e620c521b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) -**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, OpenClaw, or Factory Droid - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. +**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 19 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C). @@ -46,7 +46,7 @@ Every relationship is tagged `EXTRACTED` (found directly in source), `INFERRED` ## Install -**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [OpenClaw](https://openclaw.ai), or [Factory Droid](https://factory.ai) +**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), or [Trae](https://trae.com) ```bash pip install graphifyy && graphify install @@ -64,8 +64,10 @@ pip install graphifyy && graphify install | OpenCode | `graphify install --platform opencode` | | OpenClaw | `graphify install --platform claw` | | Factory Droid | `graphify install --platform droid` | +| Trae | `graphify install --platform trae` | +| Trae CN | `graphify install --platform trae-cn` | -Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw uses sequential extraction (parallel agent support is still early on that platform). +Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw uses sequential extraction (parallel agent support is still early on that platform). Trae uses the Agent tool for parallel subagent dispatch and does **not** support PreToolUse hooks — AGENTS.md is the always-on mechanism. Then open your AI coding assistant and type: @@ -86,10 +88,12 @@ After building a graph, run this once in your project: | OpenCode | `graphify opencode install` | | OpenClaw | `graphify claw install` | | Factory Droid | `graphify droid install` | +| Trae | `graphify trae install` | +| Trae CN | `graphify trae-cn install` | **Claude Code** does two things: writes a `CLAUDE.md` section telling Claude to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs a **PreToolUse hook** (`settings.json`) that fires before every Glob and Grep call. If a knowledge graph exists, Claude sees: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ — so Claude navigates via the graph instead of grepping through every file. -**Codex, OpenCode, OpenClaw, Factory Droid** write the same rules to `AGENTS.md` in your project root. These platforms don't support PreToolUse hooks, so AGENTS.md is the always-on mechanism. +**Codex, OpenCode, OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support PreToolUse hooks, so AGENTS.md is the always-on mechanism. Uninstall with the matching uninstall command (e.g. `graphify claude uninstall`). @@ -198,6 +202,10 @@ graphify codex install # AGENTS.md (Codex) graphify opencode install # AGENTS.md (OpenCode) graphify claw install # AGENTS.md (OpenClaw) graphify droid install # AGENTS.md (Factory Droid) +graphify trae install # AGENTS.md (Trae) +graphify trae uninstall +graphify trae-cn install # AGENTS.md (Trae CN) +graphify trae-cn uninstall # query the graph directly from the terminal (no AI assistant needed) graphify query "what connects attention to the optimizer?" diff --git a/README.zh-CN.md b/README.zh-CN.md index 113e290df..d2a07f371 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -5,7 +5,7 @@ [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) -**一个面向 AI 编码助手的技能。** 在 Claude Code、Codex、OpenCode 或 OpenClaw 中输入 `/graphify`，它会读取你的文件、构建知识图谱，并把原本不明显的结构关系还给你。更快理解代码库，找到架构决策背后的“为什么”。 +**一个面向 AI 编码助手的技能。** 在 Claude Code、Codex、OpenCode、OpenClaw、Factory Droid 或 Trae 中输入 `/graphify`，它会读取你的文件、构建知识图谱，并把原本不明显的结构关系还给你。更快理解代码库，找到架构决策背后的"为什么"。 完全多模态。你可以直接丢进去代码、PDF、Markdown、截图、流程图、白板照片，甚至其他语言的图片 —— graphify 会用 Claude vision 从这些内容中提取概念和关系，并把它们连接到同一张图里。 @@ -33,7 +33,7 @@ graphify 分两轮执行。第一轮是确定性的 AST 提取，对代码文件 ## 安装 -**要求：** Python 3.10+，并且使用以下平台之一：[Claude Code](https://claude.ai/code)、[Codex](https://openai.com/codex)、[OpenCode](https://opencode.ai) 或 [OpenClaw](https://openclaw.ai) +**要求：** Python 3.10+，并且使用以下平台之一：[Claude Code](https://claude.ai/code)、[Codex](https://openai.com/codex)、[OpenCode](https://opencode.ai)、[OpenClaw](https://openclaw.ai)、[Factory Droid](https://factory.ai) 或 [Trae](https://trae.com) ```bash pip install graphifyy && graphify install @@ -49,8 +49,11 @@ pip install graphifyy && graphify install | Codex | `graphify install --platform codex` | | OpenCode | `graphify install --platform opencode` | | OpenClaw | `graphify install --platform claw` | +| Factory Droid | `graphify install --platform droid` | +| Trae | `graphify install --platform trae` | +| Trae CN | `graphify install --platform trae-cn` | -Codex 用户还需要在 `~/.codex/config.toml` 的 `[features]` 下打开 `multi_agent = true`，这样才能并行提取。OpenClaw 目前的并行 agent 支持还比较早期，所以使用顺序提取。 +Codex 用户还需要在 `~/.codex/config.toml` 的 `[features]` 下打开 `multi_agent = true`，这样才能并行提取。OpenClaw 目前的并行 agent 支持还比较早期，所以使用顺序提取。Trae 使用 Agent 工具进行并行子代理调度，**不支持** PreToolUse hook，因此 AGENTS.md 是其常驻机制。 然后打开你的 AI 编码助手，输入： @@ -68,6 +71,9 @@ Codex 用户还需要在 `~/.codex/config.toml` 的 `[features]` 下打开 `mult | Codex | `graphify codex install` | | OpenCode | `graphify opencode install` | | OpenClaw | `graphify claw install` | +| Factory Droid | `graphify droid install` | +| Trae | `graphify trae install` | +| Trae CN | `graphify trae-cn install` | **Claude Code** 会做两件事： 1. 在 `CLAUDE.md` 中写入一段规则，告诉 Claude 在回答架构问题前先读 `graphify-out/GRAPH_REPORT.md` @@ -75,7 +81,7 @@ Codex 用户还需要在 `~/.codex/config.toml` 的 `[features]` 下打开 `mult 如果知识图谱存在，Claude 会先看到：_"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ —— 这样 Claude 会优先按图谱导航，而不是一上来就 grep 整个项目。 -**Codex、OpenCode、OpenClaw** 会把同样的规则写进项目根目录的 `AGENTS.md`。这些平台没有 PreToolUse hook，所以 `AGENTS.md` 是它们的常驻机制。 +**Codex、OpenCode、OpenClaw、Factory Droid、Trae** 会把同样的规则写进项目根目录的 `AGENTS.md`。这些平台没有 PreToolUse hook，所以 `AGENTS.md` 是它们的常驻机制。 卸载时使用对应平台的 uninstall 命令即可（例如 `graphify claude uninstall`）。 @@ -146,6 +152,11 @@ graphify claude uninstall graphify codex install # AGENTS.md（Codex） graphify opencode install # AGENTS.md（OpenCode） graphify claw install # AGENTS.md（OpenClaw） +graphify droid install # AGENTS.md（Factory Droid） +graphify trae install # AGENTS.md（Trae） +graphify trae uninstall +graphify trae-cn install # AGENTS.md（Trae CN） +graphify trae-cn uninstall ``` 支持混合文件类型： diff --git a/graphify/__main__.py b/graphify/__main__.py index 6ea4e6594..dfd82c4a3 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -72,6 +72,16 @@ def _check_skill_version(skill_dst: Path) -> None: "skill_dst": Path(".factory") / "skills" / "graphify" / "SKILL.md", "claude_md": False, }, + "trae": { + "skill_file": "skill-trae.md", + "skill_dst": Path(".trae") / "skills" / "graphify" / "SKILL.md", + "claude_md": False, + }, + "trae-cn": { + "skill_file": "skill-trae.md", + "skill_dst": Path(".trae-cn") / "skills" / "graphify" / "SKILL.md", + "claude_md": False, + }, "windows": { "skill_file": "skill-windows.md", "skill_dst": Path(".claude") / "skills" / "graphify" / "SKILL.md", @@ -373,7 +383,7 @@ def main() -> None: print("Usage: graphify ") print() print("Commands:") - print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|claw|droid)") + print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|claw|droid|trae|trae-cn)") print(" query \"\" BFS traversal of graph.json for a question") print(" --dfs use depth-first instead of breadth-first") print(" --budget N cap output at N tokens (default 2000)") @@ -391,7 +401,11 @@ def main() -> None: print(" claw install write graphify section to AGENTS.md (OpenClaw)") print(" claw uninstall remove graphify section from AGENTS.md") print(" droid install write graphify section to AGENTS.md (Factory Droid)") - print(" droid uninstall remove graphify section from AGENTS.md") + print(" droid uninstall remove graphify section from AGENTS.md") + print(" trae install write graphify section to AGENTS.md (Trae)") + print(" trae uninstall remove graphify section from AGENTS.md") + print(" trae-cn install write graphify section to AGENTS.md (Trae CN)") + print(" trae-cn uninstall remove graphify section from AGENTS.md") print() return @@ -421,7 +435,7 @@ def main() -> None: else: print("Usage: graphify claude [install|uninstall]", file=sys.stderr) sys.exit(1) - elif cmd in ("codex", "opencode", "claw", "droid"): + elif cmd in ("codex", "opencode", "claw", "droid", "trae", "trae-cn"): subcmd = sys.argv[2] if len(sys.argv) > 2 else "" if subcmd == "install": _agents_install(Path("."), cmd) diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md new file mode 100644 index 000000000..bed633000 --- /dev/null +++ b/graphify/skill-trae.md @@ -0,0 +1,1187 @@ +--- +name: graphify +description: any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that an AI assistant alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes → one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles pipx, venv, system installs) +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') +else + PYTHON="python3" +fi +$PYTHON -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +# Write interpreter path for all subsequent steps +$PYTHON -c "import sys; open('.graphify_python', 'w').write(sys.executable)" +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat .graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat .graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > .graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files +``` + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 3 - no need to ask anything. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent (Task) tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). + +**Step B2 - Dispatch ALL subagents using the Agent tool (Trae)** + +> **Trae platform:** Uses the **Agent (Task) tool** to dispatch subagents for parallel extraction. +> Each subagent runs independently and returns structured JSON results. +> Trae does NOT support PreToolUse hooks — AGENTS.md rules are the always-on mechanism instead. + +Use the **Task/Agent tool** to dispatch one subagent per chunk — launch ALL agents in parallel so they run simultaneously. Each agent receives the extraction prompt below with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE substituted: + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: reason about each edge individually. + Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. + Reasonable inference with some uncertainty: 0.6-0.7. + Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. +- AMBIGUOUS edges: 0.1-0.3 + +Output exactly this JSON (no other text): +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` + +After all subagents complete, collect their results. For each result: +- If a subagent returned valid JSON with `nodes` and `edges`, include it +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +Accumulate nodes/edges/hyperedges across all results and write to `.graphify_semantic_new.json`. + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed, stop and tell the user. + +Save new results to cache: +```bash +$(cat .graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `.graphify_semantic.json`: +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat .graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('.graphify_ast.json').read_text()) +sem = json.loads(Path('.graphify_semantic.json').read_text()) + +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +labels = LABELS_DICT + +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') + +to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) +print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') +print() +print('Open graphify-out/obsidian/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +if G.number_of_nodes() > 5000: + print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat .graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +detect = json.loads(Path('.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +extract = json.loads(Path('.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_python +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('.graphify_incremental.json').write_text(json.dumps(result)) +if new_total == 0: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely, then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + +Then: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +new_extraction = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None +new_extract = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` +Clean up after: `rm -f .graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output':': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat .graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' +terms = [t.lower() for t in question.split() if len(t) > 3] + +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +token_budget = BUDGET +char_budget = token_budget * 4 + +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + d = G.edges[u, v] + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")}] [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat .graphify_python) -c " +from graphify.ingest import save_query_result +from pathlib import Path +save_query_result( + question='QUESTION', + answer='ANSWER', + memory_dir=Path('graphify-out/memory'), + query_type='query', + source_nodes=SOURCE_NODES, +) +print('Query result saved to graphify-out/memory/') +" +``` + +Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat .graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + edge = G.edges[nid, path[i+1]] + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat .graphify_python) -c " +from graphify.ingest import save_query_result +from pathlib import Path +save_query_result( + question='Path from NODE_A to NODE_B', + answer='ANSWER', + memory_dir=Path('graphify-out/memory'), + query_type='path_query', + source_nodes=PATH_NODES, +) +print('Path result saved to graphify-out/memory/') +" +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat .graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + edge = G.edges[nid, neighbor] + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat .graphify_python) -c " +from graphify.ingest import save_query_result +from pathlib import Path +save_query_result( + question='Explain NODE_NAME', + answer='ANSWER', + memory_dir=Path('graphify-out/memory'), + query_type='explain', + source_nodes=['NODE_NAME'], +) +print('Explanation saved to graphify-out/memory/') +" +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat .graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native AGENTS.md integration (Trae) + +Run once per project to make graphify always-on in Trae sessions: + +```bash +graphify trae install # or: graphify trae-cn install +``` + +This writes a `## graphify` section to the local `AGENTS.md` that instructs Trae to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +> **Note:** Unlike Claude Code, Trae does NOT support PreToolUse hooks. The AGENTS.md rules are the always-on mechanism — there is no automatic graph rebuild on tool use. Run `/graphify --update` manually after code changes if the graph needs refreshing. + +```bash +graphify trae uninstall # or: graphify trae-cn uninstall # remove the section +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. From b64c23146717d17817957fdffafe98d5d24bc104 Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 8 Apr 2026 19:46:25 +0100 Subject: [PATCH 06/59] fix: skill-droid.md missing from package-data, louvain kwargs version-safe --- graphify/cluster.py | 13 ++++++++----- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/graphify/cluster.py b/graphify/cluster.py index db7709af0..09d070405 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -1,8 +1,8 @@ """Community detection on NetworkX graphs. Uses Leiden (graspologic) if available, falls back to Louvain (networkx). Splits oversized communities. Returns cohesion scores.""" from __future__ import annotations import contextlib +import inspect import io -import os import sys import networkx as nx @@ -42,10 +42,13 @@ def _partition(G: nx.Graph) -> dict[str, int]: except ImportError: pass - # Fallback: networkx louvain (available since networkx 2.7) - # max_level=10 and threshold=1e-4 prevent indefinite hangs on large sparse graphs - # while producing equivalent community quality to the defaults on typical corpora - communities = nx.community.louvain_communities(G, seed=42, max_level=10, threshold=1e-4) + # Fallback: networkx louvain (available since networkx 2.7). + # Inspect kwargs to stay compatible across NetworkX versions — max_level + # was added in a later release and prevents hangs on large sparse graphs. + kwargs: dict = {"seed": 42, "threshold": 1e-4} + if "max_level" in inspect.signature(nx.community.louvain_communities).parameters: + kwargs["max_level"] = 10 + communities = nx.community.louvain_communities(G, **kwargs) return {node: cid for cid, nodes in enumerate(communities) for node in nodes} diff --git a/pyproject.toml b/pyproject.toml index b93f3e751..e6f678984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,4 +56,4 @@ where = ["."] include = ["graphify*"] [tool.setuptools.package-data] -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-claw.md", "skill-windows.md"] +graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-claw.md", "skill-windows.md", "skill-droid.md"] From 1cc54b3bbfa9dcf33e4c34a732e2091b1c0981d1 Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 8 Apr 2026 19:53:10 +0100 Subject: [PATCH 07/59] fix: XSS in legend innerHTML and shebang allowlist in hooks --- graphify/export.py | 2 +- graphify/hooks.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/graphify/export.py b/graphify/export.py index d363aa3cc..d7a7349a5 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -365,7 +365,7 @@ def to_html( legend_data = [] for cid in sorted((community_labels or {}).keys()): color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] - lbl = (community_labels or {}).get(cid, f"Community {cid}") + lbl = _html.escape(sanitize_label((community_labels or {}).get(cid, f"Community {cid}"))) n = len(communities.get(cid, [])) legend_data.append({"cid": cid, "color": color, "label": lbl, "count": n}) diff --git a/graphify/hooks.py b/graphify/hooks.py index 51959ffb9..f2c74e3c4 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -17,6 +17,11 @@ */env\\ *) GRAPHIFY_PYTHON="${_SHEBANG#*/env }" ;; *) GRAPHIFY_PYTHON="$_SHEBANG" ;; esac + # Allowlist: only keep characters valid in a filesystem path to prevent + # injection if the shebang contains shell metacharacters + case "$GRAPHIFY_PYTHON" in + *[!a-zA-Z0-9/_.-]*) GRAPHIFY_PYTHON="python3" ;; + esac if ! "$GRAPHIFY_PYTHON" -c "import graphify" 2>/dev/null; then GRAPHIFY_PYTHON="python3" fi From 6a3b4bb39beaa2421f7cf67683fbe50af9822db3 Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 8 Apr 2026 19:57:33 +0100 Subject: [PATCH 08/59] fix: shebang allowlist validation in all skill files --- graphify/skill-claw.md | 8 ++++++-- graphify/skill-codex.md | 7 +++++-- graphify/skill-droid.md | 8 ++++++-- graphify/skill-opencode.md | 8 ++++++-- graphify/skill-trae.md | 7 +++++-- graphify/skill.md | 7 +++++-- 6 files changed, 33 insertions(+), 12 deletions(-) diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 4e967ee24..887a622de 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -62,12 +62,16 @@ Follow these steps in order. Do not skip steps. GRAPHIFY_BIN=$(which graphify 2>/dev/null) if [ -n "$GRAPHIFY_BIN" ]; then PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac else PYTHON="python3" fi -$PYTHON -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +mkdir -p graphify-out # Write interpreter path for all subsequent steps -$PYTHON -c "import sys; open('.graphify_python', 'w').write(sys.executable)" +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" ``` If the import succeeds, print nothing and move straight to Step 2. diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index cfd1e9217..78e90cc01 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -62,12 +62,15 @@ Follow these steps in order. Do not skip steps. GRAPHIFY_BIN=$(which graphify 2>/dev/null) if [ -n "$GRAPHIFY_BIN" ]; then PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac else PYTHON="python3" fi -$PYTHON -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps -$PYTHON -c "import sys; open('.graphify_python', 'w').write(sys.executable)" +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" ``` If the import succeeds, print nothing and move straight to Step 2. diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index b1a873ede..de739f631 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -62,12 +62,16 @@ Follow these steps in order. Do not skip steps. GRAPHIFY_BIN=$(which graphify 2>/dev/null) if [ -n "$GRAPHIFY_BIN" ]; then PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac else PYTHON="python3" fi -$PYTHON -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps -$PYTHON -c "import sys; open('.graphify_python', 'w').write(sys.executable)" +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" ``` If the import succeeds, print nothing and move straight to Step 2. diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 629413891..93dd21215 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -62,12 +62,16 @@ Follow these steps in order. Do not skip steps. GRAPHIFY_BIN=$(which graphify 2>/dev/null) if [ -n "$GRAPHIFY_BIN" ]; then PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac else PYTHON="python3" fi -$PYTHON -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps -$PYTHON -c "import sys; open('.graphify_python', 'w').write(sys.executable)" +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" ``` If the import succeeds, print nothing and move straight to Step 2. diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index bed633000..9d7785609 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -62,12 +62,15 @@ Follow these steps in order. Do not skip steps. GRAPHIFY_BIN=$(which graphify 2>/dev/null) if [ -n "$GRAPHIFY_BIN" ]; then PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac else PYTHON="python3" fi -$PYTHON -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps -$PYTHON -c "import sys; open('.graphify_python', 'w').write(sys.executable)" +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" ``` If the import succeeds, print nothing and move straight to Step 2. diff --git a/graphify/skill.md b/graphify/skill.md index 35ca68164..f6a47fb72 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -64,12 +64,15 @@ Follow these steps in order. Do not skip steps. GRAPHIFY_BIN=$(which graphify 2>/dev/null) if [ -n "$GRAPHIFY_BIN" ]; then PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac else PYTHON="python3" fi -$PYTHON -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps -$PYTHON -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" mkdir -p graphify-out ``` From 0dc125d09a2bb3c79e382450fb4bf211d0a214aa Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 8 Apr 2026 20:00:05 +0100 Subject: [PATCH 09/59] =?UTF-8?q?release:=200.3.15=20=E2=80=94=20Trae=20pl?= =?UTF-8?q?atform,=20security=20hardening,=20pipx=20hooks,=20PowerShell=20?= =?UTF-8?q?fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 13 +++++++++++++ README.md | 4 +++- pyproject.toml | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aeb4c7be..a4b32ef31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.15 (2026-04-08) + +- Feat: Trae and Trae CN platform support (`graphify install --platform trae` / `trae-cn`) +- Fix: `skill-droid.md` was missing from PyPI package data — Factory Droid users couldn't install the skill +- Fix: XSS in HTML legend — community labels now HTML-escaped before `innerHTML` injection +- Fix: Shebang allowlist validation in `hooks.py` and all 6 skill files — prevents metacharacter injection from malicious binaries +- Fix: `louvain_communities()` kwargs now inspected at runtime for cross-version NetworkX compatibility +- Fix: pipx installs now detected correctly in git hooks (reads shebang from graphify binary) +- Fix: graspologic ANSI escape codes no longer corrupt PowerShell 5.1 scroll buffer +- Docs: Japanese README added +- Docs: `graph.json` + LLM workflow example added to README +- Docs: Codex PreToolUse hook now documented in platform table + ## 0.3.14 (2026-04-08) - Fix: `graphify codex install` now also writes a PreToolUse hook to `.codex/hooks.json` so the graph reminder fires before every Bash tool call (#86) diff --git a/README.md b/README.md index e620c521b..9368cd5f9 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,9 @@ After building a graph, run this once in your project: **Claude Code** does two things: writes a `CLAUDE.md` section telling Claude to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs a **PreToolUse hook** (`settings.json`) that fires before every Glob and Grep call. If a knowledge graph exists, Claude sees: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ — so Claude navigates via the graph instead of grepping through every file. -**Codex, OpenCode, OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support PreToolUse hooks, so AGENTS.md is the always-on mechanism. +**Codex** writes to `AGENTS.md` and also installs a **PreToolUse hook** in `.codex/hooks.json` that fires before every Bash tool call — same always-on mechanism as Claude Code. + +**OpenCode, OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support PreToolUse hooks, so AGENTS.md is the always-on mechanism. Uninstall with the matching uninstall command (e.g. `graphify claude uninstall`). diff --git a/pyproject.toml b/pyproject.toml index e6f678984..83715bbe0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.14" +version = "0.3.15" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From 30a66ec1f44e6686a5e3d63b1a06e7d6475b4d16 Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 8 Apr 2026 20:06:06 +0100 Subject: [PATCH 10/59] fix: switch star history chart to starchart.cc (no auth required) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9368cd5f9..50230b02a 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantic extraction via ## Star history -[![Star History Chart](https://api.star-history.com/svg?repos=safishamsi/graphify&type=Date)](https://star-history.com/#safishamsi/graphify&Date) +[![Star History Chart](https://starchart.cc/safishamsi/graphify.svg)](https://starchart.cc/safishamsi/graphify)
Contributing From 23d88c55b07aa25d2cf477b7903657849740672d Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 8 Apr 2026 22:27:08 +0100 Subject: [PATCH 11/59] v0.3.16: NetworkX <3.4 compat, .jsx support, pipx follow-up fix Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 6 ++++++ graphify/__main__.py | 6 +++++- graphify/benchmark.py | 5 ++++- graphify/detect.py | 2 +- graphify/extract.py | 1 + graphify/serve.py | 5 ++++- graphify/skill-claw.md | 2 +- graphify/skill-codex.md | 2 +- graphify/skill-droid.md | 2 +- graphify/skill-opencode.md | 2 +- graphify/skill-trae.md | 2 +- graphify/skill.md | 2 +- pyproject.toml | 2 +- 13 files changed, 28 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4b32ef31..5fd2ef249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.16 (2026-04-08) + +- Fix: `graphify query`, `serve`, and `benchmark` now work on NetworkX < 3.4 — version-safe shim for `node_link_graph()` at all call sites (#95) +- Fix: `.jsx` files now detected and extracted via the JS extractor — added to `CODE_EXTENSIONS` and `_DISPATCH` (#94) +- Fix: `.graphify_python` no longer deleted in Step 9 cleanup across all 6 skill files — pipx users no longer hit `ModuleNotFoundError` on follow-up commands (#92) + ## 0.3.15 (2026-04-08) - Feat: Trae and Trae CN platform support (`graphify install --platform trae` / `trae-cn`) diff --git a/graphify/__main__.py b/graphify/__main__.py index dfd82c4a3..3dc8f7ac6 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -502,7 +502,11 @@ def main() -> None: try: import json as _json import networkx as _nx - G = json_graph.node_link_graph(_json.loads(gp.read_text(encoding="utf-8")), edges="links") + _raw = _json.loads(gp.read_text(encoding="utf-8")) + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) except Exception as exc: print(f"error: could not load graph: {exc}", file=sys.stderr) sys.exit(1) diff --git a/graphify/benchmark.py b/graphify/benchmark.py index 5d8725a28..a71e10e7e 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -76,7 +76,10 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ data = json.loads(Path(graph_path).read_text()) - G = json_graph.node_link_graph(data, edges="links") + try: + G = json_graph.node_link_graph(data, edges="links") + except TypeError: + G = json_graph.node_link_graph(data) if corpus_words is None: # Rough estimate: each node label is ~3 words, plus source context diff --git a/graphify/detect.py b/graphify/detect.py index 9779a8eb0..108544d19 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -17,7 +17,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.js', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm'} +CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm'} DOC_EXTENSIONS = {'.md', '.txt', '.rst'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index f20fc5def..bec364539 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2362,6 +2362,7 @@ def extract(paths: list[Path]) -> dict: _DISPATCH: dict[str, Any] = { ".py": extract_python, ".js": extract_js, + ".jsx": extract_js, ".ts": extract_js, ".tsx": extract_js, ".go": extract_go, diff --git a/graphify/serve.py b/graphify/serve.py index de1f76cf0..79bd37fc7 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -12,7 +12,10 @@ def _load_graph(graph_path: str) -> nx.Graph: try: safe = validate_graph_path(graph_path) data = json.loads(safe.read_text()) - return json_graph.node_link_graph(data, edges="links") + try: + return json_graph.node_link_graph(data, edges="links") + except TypeError: + return json_graph.node_link_graph(data) except (ValueError, FileNotFoundError) as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1) diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 887a622de..bb04c5e47 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -594,7 +594,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_python +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 78e90cc01..6c77f6329 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -650,7 +650,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_python +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index de739f631..66a9f1db8 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -647,7 +647,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_python +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 93dd21215..77e240b54 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -646,7 +646,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_python +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 9d7785609..58f7bb6da 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -625,7 +625,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_python +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` diff --git a/graphify/skill.md b/graphify/skill.md index f6a47fb72..2baaa69b6 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -650,7 +650,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_python +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` diff --git a/pyproject.toml b/pyproject.toml index 83715bbe0..7cf3cb0c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.15" +version = "0.3.16" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From dd8235e0b5a8a19c0443908ebe48c38833f8113e Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 8 Apr 2026 22:54:18 +0100 Subject: [PATCH 12/59] v0.3.17: Julia support, smarter chunking, tree-sitter pin, progress output Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 7 ++ README.md | 2 +- graphify/detect.py | 2 +- graphify/extract.py | 221 ++++++++++++++++++++++++++++++++++++- graphify/skill-claw.md | 2 +- graphify/skill-codex.md | 2 +- graphify/skill-droid.md | 2 +- graphify/skill-opencode.md | 2 +- graphify/skill-trae.md | 2 +- graphify/skill.md | 2 +- pyproject.toml | 5 +- tests/fixtures/sample.jl | 33 ++++++ tests/test_languages.py | 65 ++++++++++- 13 files changed, 334 insertions(+), 13 deletions(-) create mode 100644 tests/fixtures/sample.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fd2ef249..884eb91e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.17 (2026-04-08) + +- Add: Julia (.jl) support — modules, structs, abstract types, functions, short functions, using/import, call edges, inherits edges via tree-sitter-julia (#98) +- Fix: Semantic extraction chunks now group files by directory so related artifacts land in the same chunk, reducing missed cross-chunk relationships (#65) +- Fix: `tree-sitter>=0.21` now pinned in dependencies — prevents silent empty AST output when older tree-sitter is installed with newer language bindings (#52) +- Add: Progress output every 100 files during AST extraction so large projects don't appear to hang (#52) + ## 0.3.16 (2026-04-08) - Fix: `graphify query`, `serve`, and `benchmark` now work on NetworkX < 3.4 — version-safe shim for `node_link_graph()` at all call sites (#95) diff --git a/README.md b/README.md index 50230b02a..9b56a5afd 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ **An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. -Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 19 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C). +Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 20 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia). > Andrej Karpathy keeps a `/raw` folder where he drops papers, tweets, screenshots, and notes. graphify is the answer to that problem - 71.5x fewer tokens per query vs reading the raw files, persistent across sessions, honest about what it found vs guessed. diff --git a/graphify/detect.py b/graphify/detect.py index 108544d19..9a5f16e08 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -17,7 +17,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm'} +CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl'} DOC_EXTENSIONS = {'.md', '.txt', '.rst'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index bec364539..0ea50efbc 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1151,6 +1151,218 @@ def extract_swift(path: Path) -> dict: return _extract_generic(path, _SWIFT_CONFIG) +# ── Julia extractor (custom walk) ──────────────────────────────────────────── + +def extract_julia(path: Path) -> dict: + """Extract modules, structs, functions, imports, and calls from a .jl file.""" + try: + import tree_sitter_julia as tsjulia + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"} + + try: + language = Language(tsjulia.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = path.stem + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, object]] = [] + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": "code", + "source_file": str_path, + "source_location": f"L{line}", + }) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({ + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + }) + + file_nid = _make_id(stem) + add_node(file_nid, path.name, 1) + + def _func_name_from_signature(sig_node) -> str | None: + """Extract function name from a Julia signature node (call_expression > identifier).""" + for child in sig_node.children: + if child.type == "call_expression": + callee = child.children[0] if child.children else None + if callee and callee.type == "identifier": + return _read_text(callee, source) + return None + + def walk_calls(body_node, func_nid: str) -> None: + if body_node is None: + return + t = body_node.type + if t in ("function_definition", "short_function_definition"): + return + if t == "call_expression" and body_node.children: + callee = body_node.children[0] + # Direct call: foo(...) + if callee.type == "identifier": + callee_name = _read_text(callee, source) + target_nid = _make_id(stem, callee_name) + add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, + confidence="EXTRACTED") + # Method call: obj.method(...) + elif callee.type == "field_expression" and len(callee.children) >= 3: + method_node = callee.children[-1] + method_name = _read_text(method_node, source) + target_nid = _make_id(stem, method_name) + add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, + confidence="EXTRACTED") + for child in body_node.children: + walk_calls(child, func_nid) + + def walk(node, scope_nid: str) -> None: + t = node.type + + # Module + if t == "module_definition": + name_node = next((c for c in node.children if c.type == "identifier"), None) + if name_node: + mod_name = _read_text(name_node, source) + mod_nid = _make_id(stem, mod_name) + line = node.start_point[0] + 1 + add_node(mod_nid, mod_name, line) + add_edge(file_nid, mod_nid, "defines", line) + for child in node.children: + walk(child, mod_nid) + return + + # Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia) + if t == "struct_definition": + # type_head may contain: identifier (simple) or binary_expression (Foo <: Bar) + type_head = next((c for c in node.children if c.type == "type_head"), None) + if type_head: + bin_expr = next((c for c in type_head.children if c.type == "binary_expression"), None) + if bin_expr: + # First identifier is the struct name, last is the supertype + identifiers = [c for c in bin_expr.children if c.type == "identifier"] + if identifiers: + struct_name = _read_text(identifiers[0], source) + struct_nid = _make_id(stem, struct_name) + line = node.start_point[0] + 1 + add_node(struct_nid, struct_name, line) + add_edge(scope_nid, struct_nid, "defines", line) + if len(identifiers) >= 2: + super_name = _read_text(identifiers[-1], source) + add_edge(struct_nid, _make_id(stem, super_name), "inherits", + line, confidence="EXTRACTED") + else: + name_node = next((c for c in type_head.children if c.type == "identifier"), None) + if name_node: + struct_name = _read_text(name_node, source) + struct_nid = _make_id(stem, struct_name) + line = node.start_point[0] + 1 + add_node(struct_nid, struct_name, line) + add_edge(scope_nid, struct_nid, "defines", line) + return + + # Abstract type + if t == "abstract_definition": + # type_head > identifier + type_head = next((c for c in node.children if c.type == "type_head"), None) + if type_head: + name_node = next((c for c in type_head.children if c.type == "identifier"), None) + if name_node: + abs_name = _read_text(name_node, source) + abs_nid = _make_id(stem, abs_name) + line = node.start_point[0] + 1 + add_node(abs_nid, abs_name, line) + add_edge(scope_nid, abs_nid, "defines", line) + return + + # Function: function foo(...) ... end + if t == "function_definition": + sig_node = next((c for c in node.children if c.type == "signature"), None) + if sig_node: + func_name = _func_name_from_signature(sig_node) + if func_name: + func_nid = _make_id(stem, func_name) + line = node.start_point[0] + 1 + add_node(func_nid, f"{func_name}()", line) + add_edge(scope_nid, func_nid, "defines", line) + function_bodies.append((func_nid, node)) + return + + # Short function: foo(x) = expr + if t == "assignment": + lhs = node.children[0] if node.children else None + if lhs and lhs.type == "call_expression" and lhs.children: + callee = lhs.children[0] + if callee.type == "identifier": + func_name = _read_text(callee, source) + func_nid = _make_id(stem, func_name) + line = node.start_point[0] + 1 + add_node(func_nid, f"{func_name}()", line) + add_edge(scope_nid, func_nid, "defines", line) + # Only walk the RHS (index 2 after lhs and operator) to avoid self-loops + rhs = node.children[-1] if len(node.children) >= 3 else None + if rhs: + function_bodies.append((func_nid, rhs)) + return + + # Using / Import + if t in ("using_statement", "import_statement"): + line = node.start_point[0] + 1 + for child in node.children: + if child.type == "identifier": + mod_name = _read_text(child, source) + imp_nid = _make_id(mod_name) + add_node(imp_nid, mod_name, line) + add_edge(scope_nid, imp_nid, "imports", line) + elif child.type == "selected_import": + identifiers = [c for c in child.children if c.type == "identifier"] + if identifiers: + pkg_name = _read_text(identifiers[0], source) + pkg_nid = _make_id(pkg_name) + add_node(pkg_nid, pkg_name, line) + add_edge(scope_nid, pkg_nid, "imports", line) + return + + for child in node.children: + walk(child, scope_nid) + + walk(root, file_nid) + + for func_nid, body_node in function_bodies: + # For function_definition nodes, walk children directly to avoid + # the boundary check returning early on the top-level node itself. + # Skip the "signature" child — it contains the function's own call_expression + # which would create a self-loop. + if body_node.type == "function_definition": + for child in body_node.children: + if child.type != "signature": + walk_calls(child, func_nid) + else: + walk_calls(body_node, func_nid) + + return {"nodes": nodes, "edges": edges} + + # ── Go extractor (custom walk) ──────────────────────────────────────────────── def extract_go(path: Path) -> dict: @@ -2389,9 +2601,14 @@ def extract(paths: list[Path]) -> dict: ".exs": extract_elixir, ".m": extract_objc, ".mm": extract_objc, + ".jl": extract_julia, } - for path in paths: + total = len(paths) + _PROGRESS_INTERVAL = 100 + for i, path in enumerate(paths): + if total >= _PROGRESS_INTERVAL and i % _PROGRESS_INTERVAL == 0 and i > 0: + print(f" AST extraction: {i}/{total} files ({i * 100 // total}%)", flush=True) extractor = _DISPATCH.get(path.suffix) if extractor is None: continue @@ -2403,6 +2620,8 @@ def extract(paths: list[Path]) -> dict: if "error" not in result: save_cached(path, result, root) per_file.append(result) + if total >= _PROGRESS_INTERVAL: + print(f" AST extraction: {total}/{total} files (100%)", flush=True) all_nodes: list[dict] = [] all_edges: list[dict] = [] diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index bb04c5e47..c267a7dfd 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -176,7 +176,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Sequential extraction (OpenClaw)** diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 6c77f6329..33d82dfb6 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -179,7 +179,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message (Codex)** diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 66a9f1db8..7dfb7381d 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -180,7 +180,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message (Factory Droid)** diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 77e240b54..e03db2b76 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -180,7 +180,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message (OpenCode)** diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 58f7bb6da..8b2e1ef88 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -179,7 +179,7 @@ Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all fil **Step B1 - Split into chunks** -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents using the Agent tool (Trae)** diff --git a/graphify/skill.md b/graphify/skill.md index 2baaa69b6..bd744e780 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -182,7 +182,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. **Step B2 - Dispatch ALL subagents in a single message** diff --git a/pyproject.toml b/pyproject.toml index 7cf3cb0c5..3ba33a5f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.16" +version = "0.3.17" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } @@ -12,7 +12,7 @@ keywords = ["claude", "claude-code", "codex", "opencode", "knowledge-graph", "ra requires-python = ">=3.10" dependencies = [ "networkx", - "tree-sitter", + "tree-sitter>=0.21", "tree-sitter-python", "tree-sitter-javascript", "tree-sitter-typescript", @@ -32,6 +32,7 @@ dependencies = [ "tree-sitter-powershell", "tree-sitter-elixir", "tree-sitter-objc", + "tree-sitter-julia", ] [project.urls] diff --git a/tests/fixtures/sample.jl b/tests/fixtures/sample.jl new file mode 100644 index 000000000..43409f4e0 --- /dev/null +++ b/tests/fixtures/sample.jl @@ -0,0 +1,33 @@ +module Geometry + +using LinearAlgebra +import Base: show + +abstract type Shape end + +struct Point <: Shape + x::Float64 + y::Float64 +end + +mutable struct Circle <: Shape + center::Point + radius::Float64 +end + +function area(c::Circle) + return pi * c.radius^2 +end + +function distance(p1::Point, p2::Point) + return norm([p1.x - p2.x, p1.y - p2.y]) +end + +perimeter(c::Circle) = 2 * pi * c.radius + +function describe(s::Shape) + show(s) + area(s) +end + +end diff --git a/tests/test_languages.py b/tests/test_languages.py index 551062cf1..9784dfa95 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -1,11 +1,11 @@ -"""Tests for language extractors: Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Go.""" +"""Tests for language extractors: Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Go, Julia.""" from __future__ import annotations from pathlib import Path import pytest from graphify.extract import ( extract_java, extract_c, extract_cpp, extract_ruby, extract_csharp, extract_kotlin, extract_scala, extract_php, - extract_swift, extract_go, + extract_swift, extract_go, extract_julia, ) FIXTURES = Path(__file__).parent / "fixtures" @@ -447,3 +447,64 @@ def test_go_receiver_uses_pkg_scope(): assert server_nodes # Should NOT contain the file stem "sample" in the type node id assert "sample" not in server_nodes[0]["id"].split(":")[0] + + +# --------------------------------------------------------------------------- +# Julia +# --------------------------------------------------------------------------- + +def test_julia_finds_module(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert "Geometry" in labels + + +def test_julia_finds_structs(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert "Point" in labels + assert "Circle" in labels + + +def test_julia_finds_abstract_type(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert "Shape" in labels + + +def test_julia_finds_functions(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert any("area" in l for l in labels) + assert any("distance" in l for l in labels) + + +def test_julia_finds_short_function(): + r = extract_julia(FIXTURES / "sample.jl") + labels = [n["label"] for n in r["nodes"]] + assert any("perimeter" in l for l in labels) + + +def test_julia_finds_imports(): + r = extract_julia(FIXTURES / "sample.jl") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + assert len(import_edges) >= 1 + + +def test_julia_finds_inherits(): + r = extract_julia(FIXTURES / "sample.jl") + inherits = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherits) >= 1 + + +def test_julia_finds_calls(): + r = extract_julia(FIXTURES / "sample.jl") + call_edges = [e for e in r["edges"] if e["relation"] == "calls"] + assert len(call_edges) >= 1 + + +def test_julia_no_dangling_edges(): + r = extract_julia(FIXTURES / "sample.jl") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"Dangling source: {e}" From 8dbf953016941d8d70179b16d1c8a2059061e14b Mon Sep 17 00:00:00 2001 From: Safi Date: Wed, 8 Apr 2026 22:57:46 +0100 Subject: [PATCH 13/59] docs: add .jsx .tsx .jl to extensions table, bump to 20 languages Co-Authored-By: Claude Sonnet 4.6 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9b56a5afd..73f481004 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ Works with any mix of file types: | Type | Extensions | Extraction | |------|-----------|------------| -| Code | `.py .ts .js .go .rs .java .c .cpp .rb .cs .kt .scala .php .swift .lua .zig .ps1 .ex .exs .m .mm` | AST via tree-sitter + call-graph + docstring/comment rationale | +| Code | `.py .ts .js .jsx .tsx .go .rs .java .c .cpp .rb .cs .kt .scala .php .swift .lua .zig .ps1 .ex .exs .m .mm .jl` | AST via tree-sitter + call-graph + docstring/comment rationale | | Docs | `.md .txt .rst` | Concepts + relationships + design rationale via Claude | | Office | `.docx .xlsx` | Converted to markdown then extracted via Claude (requires `pip install graphifyy[office]`) | | Papers | `.pdf` | Citation mining + concept extraction | From 1a50f2d12bb16f3e5620e1f8944b38786de26026 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 01:14:08 +0100 Subject: [PATCH 14/59] add Penpax link to README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 73f481004..d9b624e3f 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,10 @@ graphify sends file contents to your AI coding assistant's underlying model API NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantic extraction via Claude (Claude Code), GPT-4 (Codex), or whichever model your platform runs. No Neo4j required, no server, runs entirely locally. +## What we are building next + +graphify is the graph layer. We are building [Penpax](https://safishamsi.github.io/penpax.ai) on top of it — an on-device digital twin that connects your meetings, browser history, files, emails, and code into one continuously updating knowledge graph. No cloud, no training on your data. [Join the waitlist.](https://safishamsi.github.io/penpax.ai) + ## Star history [![Star History Chart](https://starchart.cc/safishamsi/graphify.svg)](https://starchart.cc/safishamsi/graphify) From ec4f6845aa4c33a32b1486cd94d7bedef4b2919c Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 08:00:07 +0100 Subject: [PATCH 15/59] add Korean README and fix trae.com link across all READMEs --- README.ja-JP.md | 2 +- README.ko-KR.md | 282 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 4 +- README.zh-CN.md | 4 +- 4 files changed, 287 insertions(+), 5 deletions(-) create mode 100644 README.ko-KR.md diff --git a/README.ja-JP.md b/README.ja-JP.md index 9924b4629..69a32b126 100644 --- a/README.ja-JP.md +++ b/README.ja-JP.md @@ -1,6 +1,6 @@ # graphify -[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) +[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) | [한국어](README.ko-KR.md) [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) diff --git a/README.ko-KR.md b/README.ko-KR.md new file mode 100644 index 000000000..b712d30d6 --- /dev/null +++ b/README.ko-KR.md @@ -0,0 +1,282 @@ +# graphify + +[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) | [한국어](README.ko-KR.md) + +[![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) +[![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) + +**AI 코딩 어시스턴트를 위한 스킬.** Claude Code, Codex, OpenCode, OpenClaw, Factory Droid, 또는 Trae에서 `/graphify`를 입력하면 파일을 읽고 지식 그래프를 구축하여, 미처 몰랐던 구조를 보여줍니다. 코드베이스를 더 빠르게 이해하고, 아키텍처 결정의 "이유"를 찾아보세요. + +완전한 멀티모달 지원. 코드, PDF, 마크다운, 스크린샷, 다이어그램, 화이트보드 사진, 심지어 다른 언어로 된 이미지까지 — graphify는 Claude Vision을 사용하여 이 모든 것에서 개념과 관계를 추출하고 하나의 그래프로 연결합니다. tree-sitter AST를 통해 20개 언어를 지원합니다(Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia). + +> Andrej Karpathy는 논문, 트윗, 스크린샷, 메모를 모아두는 `/raw` 폴더를 관리합니다. graphify는 바로 그 문제에 대한 답입니다 — 원본 파일을 직접 읽는 것 대비 쿼리당 토큰 소비가 71.5배 적고, 세션 간에 영속적이며, 발견한 것과 추측한 것을 정직하게 구분합니다. + +``` +/graphify . # 어떤 폴더든 동작 - 코드베이스, 노트, 논문, 무엇이든 +``` + +``` +graphify-out/ +├── graph.html 인터랙티브 그래프 - 노드 클릭, 검색, 커뮤니티별 필터 +├── GRAPH_REPORT.md 갓 노드, 의외의 연결, 추천 질문 +├── graph.json 영속 그래프 - 몇 주 후에도 재읽기 없이 쿼리 가능 +└── cache/ SHA256 캐시 - 재실행 시 변경된 파일만 처리 +``` + +그래프에 포함하지 않을 폴더를 제외하려면 `.graphifyignore` 파일을 추가하세요: + +``` +# .graphifyignore +vendor/ +node_modules/ +dist/ +*.generated.py +``` + +`.gitignore`와 동일한 문법입니다. 패턴은 graphify를 실행한 폴더 기준의 상대 경로에 대해 매칭됩니다. + +## 동작 원리 + +graphify는 두 번의 패스로 실행됩니다. 첫 번째는 결정론적 AST 패스로, 코드 파일에서 구조(클래스, 함수, 임포트, 콜 그래프, docstring, 근거 주석)를 LLM 없이 추출합니다. 두 번째는 Claude 서브에이전트가 문서, 논문, 이미지에 대해 병렬로 실행되어 개념, 관계, 설계 근거를 추출합니다. 결과는 NetworkX 그래프로 병합되고, Leiden 커뮤니티 탐지로 클러스터링되며, 인터랙티브 HTML, 쿼리 가능한 JSON, 그리고 일반 언어 감사 보고서로 내보내집니다. + +**클러스터링은 그래프 토폴로지 기반 — 임베딩을 사용하지 않습니다.** Leiden은 엣지 밀도를 기반으로 커뮤니티를 찾습니다. Claude가 추출하는 의미적 유사성 엣지(`semantically_similar_to`, INFERRED로 표시)는 이미 그래프에 포함되어 있으므로 커뮤니티 탐지에 직접 영향을 줍니다. 그래프 구조 자체가 유사성 신호이며 — 별도의 임베딩 단계나 벡터 데이터베이스가 필요하지 않습니다. + +모든 관계는 `EXTRACTED`(소스에서 직접 발견), `INFERRED`(합리적 추론, 신뢰도 점수 포함), `AMBIGUOUS`(리뷰 필요 표시) 중 하나로 태깅됩니다. 무엇이 발견된 것이고 무엇이 추측된 것인지 항상 알 수 있습니다. + +## 설치 + +**필수 요구사항:** Python 3.10+ 및 다음 중 하나: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), 또는 [Trae](https://trae.ai) + +```bash +pip install graphifyy && graphify install +``` + +> PyPI 패키지는 `graphify` 이름을 되찾는 동안 임시로 `graphifyy`로 명명되어 있습니다. CLI와 스킬 명령은 여전히 `graphify`입니다. + +### 플랫폼 지원 + +| 플랫폼 | 설치 명령 | +|--------|-----------| +| Claude Code (Linux/Mac) | `graphify install` | +| Claude Code (Windows) | `graphify install` (자동 감지) 또는 `graphify install --platform windows` | +| Codex | `graphify install --platform codex` | +| OpenCode | `graphify install --platform opencode` | +| OpenClaw | `graphify install --platform claw` | +| Factory Droid | `graphify install --platform droid` | +| Trae | `graphify install --platform trae` | +| Trae CN | `graphify install --platform trae-cn` | + +Codex 사용자는 병렬 추출을 위해 `~/.codex/config.toml`의 `[features]` 아래에 `multi_agent = true`도 필요합니다. Factory Droid는 병렬 서브에이전트 디스패치에 `Task` 도구를 사용합니다. OpenClaw는 순차 추출을 사용합니다(해당 플랫폼의 병렬 에이전트 지원은 아직 초기 단계입니다). Trae는 병렬 서브에이전트 디스패치에 Agent 도구를 사용하며 PreToolUse 훅을 **지원하지 않습니다** — AGENTS.md가 상시 작동 메커니즘입니다. + +그런 다음 AI 코딩 어시스턴트를 열고 입력하세요: + +``` +/graphify . +``` + +참고: Codex는 스킬 호출에 `/` 대신 `$`를 사용하므로 `$graphify .`라고 입력하세요. + +### 어시스턴트가 항상 그래프를 사용하도록 설정 (권장) + +그래프를 빌드한 후, 프로젝트에서 한 번만 실행하세요: + +| 플랫폼 | 명령 | +|--------|------| +| Claude Code | `graphify claude install` | +| Codex | `graphify codex install` | +| OpenCode | `graphify opencode install` | +| OpenClaw | `graphify claw install` | +| Factory Droid | `graphify droid install` | +| Trae | `graphify trae install` | +| Trae CN | `graphify trae-cn install` | + +**Claude Code**는 두 가지를 수행합니다: 아키텍처 질문에 답하기 전에 `graphify-out/GRAPH_REPORT.md`를 읽도록 Claude에게 지시하는 `CLAUDE.md` 섹션을 작성하고, 모든 Glob 및 Grep 호출 전에 실행되는 **PreToolUse 훅**(`settings.json`)을 설치합니다. 지식 그래프가 존재하면 Claude는 다음 메시지를 보게 됩니다: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ — 이를 통해 Claude는 모든 파일을 grep하는 대신 그래프를 통해 탐색합니다. + +**Codex**는 `AGENTS.md`에 작성하고 Bash 도구 호출 전에 실행되는 **PreToolUse 훅**을 `.codex/hooks.json`에 설치합니다 — Claude Code와 동일한 상시 작동 메커니즘입니다. + +**OpenCode, OpenClaw, Factory Droid, Trae**는 프로젝트 루트의 `AGENTS.md`에 동일한 규칙을 작성합니다. 이 플랫폼들은 PreToolUse 훅을 지원하지 않으므로 AGENTS.md가 상시 작동 메커니즘입니다. + +제거는 대응하는 uninstall 명령으로 수행합니다(예: `graphify claude uninstall`). + +**상시 작동 vs 명시적 트리거 — 차이점은?** + +상시 작동 훅은 `GRAPH_REPORT.md`를 노출합니다 — 갓 노드, 커뮤니티, 의외의 연결을 한 페이지로 요약한 것입니다. 어시스턴트는 파일 검색 전에 이것을 읽으므로 키워드 매칭이 아닌 구조 기반으로 탐색합니다. 이것만으로 대부분의 일상적인 질문을 처리할 수 있습니다. + +`/graphify query`, `/graphify path`, `/graphify explain`은 더 깊이 들어갑니다: 원시 `graph.json`을 홉 단위로 순회하고, 노드 간의 정확한 경로를 추적하며, 엣지 수준의 세부 정보(관계 유형, 신뢰도 점수, 소스 위치)를 보여줍니다. 일반적인 오리엔테이션이 아닌 그래프에서 특정 질문에 답하고 싶을 때 사용하세요. + +이렇게 생각하면 됩니다: 상시 작동 훅은 어시스턴트에게 지도를 주고, `/graphify` 명령은 그 지도를 정확하게 탐색하게 합니다. + +## `graph.json`을 LLM과 함께 사용하기 + +`graph.json`은 프롬프트에 한 번에 전부 붙여넣기 위한 것이 아닙니다. 유용한 워크플로우는 다음과 같습니다: + +1. `graphify-out/GRAPH_REPORT.md`로 높은 수준의 개요를 파악합니다. +2. `graphify query`를 사용하여 답하려는 특정 질문에 대한 더 작은 서브그래프를 가져옵니다. +3. 전체 원시 코퍼스 대신 그 집중된 결과를 어시스턴트에게 제공합니다. + +예를 들어, 프로젝트에서 graphify를 실행한 후: + +```bash +graphify query "show the auth flow" --graph graphify-out/graph.json +graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json +``` + +출력에는 노드 레이블, 엣지 유형, 신뢰도 태그, 소스 파일, 소스 위치가 포함됩니다. 이는 LLM을 위한 좋은 중간 컨텍스트 블록이 됩니다: + +```text +이 그래프 쿼리 결과를 사용하여 질문에 답하세요. 추측보다 그래프 구조를 우선하고, +가능한 경우 소스 파일을 인용하세요. +``` + +어시스턴트가 도구 호출이나 MCP를 지원하는 경우, 텍스트를 붙여넣는 대신 그래프를 직접 사용하세요. graphify는 `graph.json`을 MCP 서버로 노출할 수 있습니다: + +```bash +python -m graphify.serve graphify-out/graph.json +``` + +이를 통해 어시스턴트가 `query_graph`, `get_node`, `get_neighbors`, `shortest_path` 같은 반복 쿼리에 구조화된 그래프 접근을 할 수 있습니다. + +
+수동 설치 (curl) + +```bash +mkdir -p ~/.claude/skills/graphify +curl -fsSL https://raw.githubusercontent.com/safishamsi/graphify/v3/graphify/skill.md \ + > ~/.claude/skills/graphify/SKILL.md +``` + +`~/.claude/CLAUDE.md`에 추가: + +``` +- **graphify** (`~/.claude/skills/graphify/SKILL.md`) - any input to knowledge graph. Trigger: `/graphify` +When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"` before doing anything else. +``` + +
+ +## 사용법 + +``` +/graphify # 현재 디렉토리에서 실행 +/graphify ./raw # 특정 폴더에서 실행 +/graphify ./raw --mode deep # 더 적극적인 INFERRED 엣지 추출 +/graphify ./raw --update # 변경된 파일만 재추출하여 기존 그래프에 병합 +/graphify ./raw --cluster-only # 기존 그래프의 클러스터링만 재실행, 재추출 없음 +/graphify ./raw --no-viz # HTML 건너뛰기, 보고서 + JSON만 생성 +/graphify ./raw --obsidian # Obsidian 볼트도 생성 (옵트인) +/graphify ./raw --obsidian --obsidian-dir ~/vaults/myproject # 볼트를 특정 디렉토리에 생성 + +/graphify add https://arxiv.org/abs/1706.03762 # 논문 가져오기, 저장, 그래프 업데이트 +/graphify add https://x.com/karpathy/status/... # 트윗 가져오기 +/graphify add https://... --author "Name" # 원저자 태그 +/graphify add https://... --contributor "Name" # 코퍼스에 추가한 사람 태그 + +/graphify query "어텐션과 옵티마이저를 연결하는 것은?" +/graphify query "어텐션과 옵티마이저를 연결하는 것은?" --dfs # 특정 경로 추적 +/graphify query "어텐션과 옵티마이저를 연결하는 것은?" --budget 1500 # N 토큰으로 제한 +/graphify path "DigestAuth" "Response" +/graphify explain "SwinTransformer" + +/graphify ./raw --watch # 파일 변경 시 그래프 자동 동기화 (코드: 즉시, 문서: 알림) +/graphify ./raw --wiki # 에이전트가 크롤 가능한 위키 빌드 (index.md + 커뮤니티별 문서) +/graphify ./raw --svg # graph.svg 내보내기 +/graphify ./raw --graphml # graph.graphml 내보내기 (Gephi, yEd) +/graphify ./raw --neo4j # Neo4j용 cypher.txt 생성 +/graphify ./raw --neo4j-push bolt://localhost:7687 # 실행 중인 Neo4j 인스턴스에 직접 푸시 +/graphify ./raw --mcp # MCP stdio 서버 시작 + +# git 훅 - 플랫폼 무관, 커밋 및 브랜치 전환 시 그래프 재빌드 +graphify hook install +graphify hook uninstall +graphify hook status + +# 상시 작동 어시스턴트 지시 - 플랫폼별 +graphify claude install # CLAUDE.md + PreToolUse 훅 (Claude Code) +graphify claude uninstall +graphify codex install # AGENTS.md (Codex) +graphify opencode install # AGENTS.md (OpenCode) +graphify claw install # AGENTS.md (OpenClaw) +graphify droid install # AGENTS.md (Factory Droid) +graphify trae install # AGENTS.md (Trae) +graphify trae uninstall +graphify trae-cn install # AGENTS.md (Trae CN) +graphify trae-cn uninstall + +# 터미널에서 직접 그래프 쿼리 (AI 어시스턴트 불필요) +graphify query "어텐션과 옵티마이저를 연결하는 것은?" +graphify query "인증 흐름 보기" --dfs +graphify query "CfgNode이 뭐지?" --budget 500 +graphify query "..." --graph path/to/graph.json +``` + +다양한 파일 유형의 조합과 함께 동작합니다: + +| 유형 | 확장자 | 추출 방식 | +|------|--------|-----------| +| 코드 | `.py .ts .js .jsx .tsx .go .rs .java .c .cpp .rb .cs .kt .scala .php .swift .lua .zig .ps1 .ex .exs .m .mm .jl` | tree-sitter AST + 콜 그래프 + docstring/주석 근거 | +| 문서 | `.md .txt .rst` | Claude를 통한 개념 + 관계 + 설계 근거 | +| 오피스 | `.docx .xlsx` | 마크다운으로 변환 후 Claude를 통해 추출 (`pip install graphifyy[office]` 필요) | +| 논문 | `.pdf` | 인용 마이닝 + 개념 추출 | +| 이미지 | `.png .jpg .webp .gif` | Claude Vision - 스크린샷, 다이어그램, 모든 언어 | + +## 결과물 + +**갓 노드** - 최고 차수의 개념 (모든 것이 연결되는 허브) + +**의외의 연결** - 복합 점수로 순위 지정. 코드-논문 엣지는 코드-코드보다 높게 순위됩니다. 각 결과에는 쉬운 설명이 포함됩니다. + +**추천 질문** - 그래프가 고유하게 답할 수 있는 4~5개의 질문 + +**"이유"** - docstring, 인라인 주석(`# NOTE:`, `# IMPORTANT:`, `# HACK:`, `# WHY:`), 문서의 설계 근거가 `rationale_for` 노드로 추출됩니다. 코드가 무엇을 하는지뿐만 아니라 — 왜 그렇게 작성되었는지. + +**신뢰도 점수** - 모든 INFERRED 엣지에는 `confidence_score`(0.0~1.0)가 있습니다. 무엇이 추측되었는지뿐 아니라 모델이 얼마나 확신했는지도 알 수 있습니다. EXTRACTED 엣지는 항상 1.0입니다. + +**의미적 유사성 엣지** - 구조적 연결 없는 파일 간 개념 링크. 서로를 호출하지 않으면서 같은 문제를 해결하는 두 함수, 코드의 클래스와 같은 알고리즘을 설명하는 논문의 개념 등. + +**하이퍼엣지** - 쌍별 엣지로는 표현할 수 없는 3개 이상 노드의 그룹 관계. 공유 프로토콜을 구현하는 모든 클래스, 인증 흐름의 모든 함수, 논문 섹션에서 하나의 아이디어를 구성하는 모든 개념 등. + +**토큰 벤치마크** - 매 실행 후 자동으로 출력됩니다. 혼합 코퍼스(Karpathy 리포지토리 + 논문 + 이미지)에서: 원본 파일 대비 쿼리당 **71.5배** 적은 토큰. 첫 실행은 추출과 그래프 빌드를 수행합니다(토큰이 소비됩니다). 이후 모든 쿼리는 원본 파일 대신 압축된 그래프를 읽습니다 — 여기서 절약이 복리로 누적됩니다. SHA256 캐시로 재실행 시 변경된 파일만 재처리합니다. + +**자동 동기화** (`--watch`) - 백그라운드 터미널에서 실행하면 코드베이스가 변경될 때 그래프가 자동으로 업데이트됩니다. 코드 파일 저장 시 즉시 재빌드가 트리거됩니다(AST만, LLM 없음). 문서/이미지 변경 시에는 LLM 재처리를 위해 `--update` 실행을 알려줍니다. + +**Git 훅** (`graphify hook install`) - post-commit 및 post-checkout 훅을 설치합니다. 모든 커밋과 브랜치 전환 후 그래프가 자동으로 재빌드됩니다. 재빌드가 실패하면 훅이 0이 아닌 코드로 종료하여 git이 에러를 표시하고 조용히 계속 진행하지 않습니다. 백그라운드 프로세스가 필요 없습니다. + +**위키** (`--wiki`) - 커뮤니티 및 갓 노드별 위키피디아 스타일 마크다운 문서와 `index.md` 진입점. 어떤 에이전트든 `index.md`를 가리키면 JSON을 파싱하는 대신 파일을 읽어서 지식 베이스를 탐색할 수 있습니다. + +## 실전 예제 + +| 코퍼스 | 파일 수 | 축소율 | 결과 | +|--------|---------|--------|------| +| Karpathy 리포지토리 + 논문 5편 + 이미지 4장 | 52 | **71.5x** | [`worked/karpathy-repos/`](worked/karpathy-repos/) | +| graphify 소스 + Transformer 논문 | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | +| httpx (합성 Python 라이브러리) | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | + +토큰 축소는 코퍼스 크기에 비례하여 확장됩니다. 6개 파일은 어차피 컨텍스트 윈도우에 들어가므로, 그래프의 가치는 압축이 아닌 구조적 명확성에 있습니다. 52개 파일(코드 + 논문 + 이미지)에서는 71배 이상을 달성합니다. 각 `worked/` 폴더에는 원본 입력 파일과 실제 출력(`GRAPH_REPORT.md`, `graph.json`)이 있어 직접 실행하여 수치를 검증할 수 있습니다. + +## 개인정보 보호 + +graphify는 문서, 논문, 이미지의 의미적 추출을 위해 파일 내용을 AI 코딩 어시스턴트의 기반 모델 API로 전송합니다 — Anthropic(Claude Code), OpenAI(Codex), 또는 사용 중인 플랫폼의 제공자. 코드 파일은 tree-sitter AST를 통해 로컬에서 처리됩니다 — 코드의 경우 파일 내용이 사용자의 머신을 벗어나지 않습니다. 어떠한 텔레메트리, 사용 추적, 분석도 없습니다. 유일한 네트워크 호출은 추출 중 플랫폼 모델 API에 대한 것이며, 사용자 본인의 API 키를 사용합니다. + +## 기술 스택 + +NetworkX + Leiden (graspologic) + tree-sitter + vis.js. 의미적 추출은 Claude(Claude Code), GPT-4(Codex), 또는 플랫폼이 실행하는 모델을 통해 수행됩니다. Neo4j 불필요, 서버 불필요, 완전히 로컬에서 실행됩니다. + +## 다음 계획 + +graphify는 그래프 레이어입니다. 그 위에 [Penpax](https://safishamsi.github.io/penpax.ai)를 개발하고 있습니다 — 회의, 브라우저 기록, 파일, 이메일, 코드를 하나의 지속적으로 업데이트되는 지식 그래프로 연결하는 온디바이스 디지털 트윈입니다. 클라우드 없음, 데이터 학습 없음. [대기 목록에 등록하세요.](https://safishamsi.github.io/penpax.ai) + +## 스타 히스토리 + +[![Star History Chart](https://starchart.cc/safishamsi/graphify.svg)](https://starchart.cc/safishamsi/graphify) + +
+기여하기 + +**실전 예제**는 가장 신뢰를 쌓는 기여 방식입니다. 실제 코퍼스에서 `/graphify`를 실행하고, 결과를 `worked/{slug}/`에 저장하고, 그래프가 맞게 파악한 것과 틀린 것을 평가하는 솔직한 `review.md`를 작성하여 PR을 제출하세요. + +**추출 버그** - 입력 파일, 캐시 엔트리(`graphify-out/cache/`), 그리고 누락되거나 날조된 내용과 함께 이슈를 열어주세요. + +모듈 책임과 언어 추가 방법은 [ARCHITECTURE.md](ARCHITECTURE.md)를 참조하세요. + +
diff --git a/README.md b/README.md index d9b624e3f..7b758a269 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # graphify -[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) +[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) | [한국어](README.ko-KR.md) [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) @@ -46,7 +46,7 @@ Every relationship is tagged `EXTRACTED` (found directly in source), `INFERRED` ## Install -**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), or [Trae](https://trae.com) +**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), or [Trae](https://trae.ai) ```bash pip install graphifyy && graphify install diff --git a/README.zh-CN.md b/README.zh-CN.md index d2a07f371..aecaa51ac 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -1,6 +1,6 @@ # graphify -[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) +[English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) | [한국어](README.ko-KR.md) [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) @@ -33,7 +33,7 @@ graphify 分两轮执行。第一轮是确定性的 AST 提取，对代码文件 ## 安装 -**要求：** Python 3.10+，并且使用以下平台之一：[Claude Code](https://claude.ai/code)、[Codex](https://openai.com/codex)、[OpenCode](https://opencode.ai)、[OpenClaw](https://openclaw.ai)、[Factory Droid](https://factory.ai) 或 [Trae](https://trae.com) +**要求：** Python 3.10+，并且使用以下平台之一：[Claude Code](https://claude.ai/code)、[Codex](https://openai.com/codex)、[OpenCode](https://opencode.ai)、[OpenClaw](https://openclaw.ai)、[Factory Droid](https://factory.ai) 或 [Trae](https://trae.ai) ```bash pip install graphifyy && graphify install From 11dff7e9b3e185671a90c6d048e191fa70a8c008 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 08:09:36 +0100 Subject: [PATCH 16/59] release 0.3.18: fix watch .graphifyignore, codex hook, trae link, Korean README, save-result CLI --- CHANGELOG.md | 9 +++++++++ graphify/__main__.py | 27 +++++++++++++++++++++++++- graphify/skill-claw.md | 39 +++----------------------------------- graphify/skill-codex.md | 39 +++----------------------------------- graphify/skill-droid.md | 39 +++----------------------------------- graphify/skill-opencode.md | 39 +++----------------------------------- graphify/skill-trae.md | 39 +++----------------------------------- graphify/skill-windows.md | 39 +++----------------------------------- graphify/skill.md | 39 +++----------------------------------- graphify/watch.py | 13 +++++-------- 10 files changed, 61 insertions(+), 261 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 884eb91e6..847f45488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.18 (2026-04-09) + +- Fix: `--watch` mode now respects `.graphifyignore` — `_rebuild_code` was calling `collect_files()` directly instead of `detect()`, bypassing ignore patterns (#120) +- Fix: Codex PreToolUse hook now uses `systemMessage` instead of `additionalContext` — Codex does not support `additionalContext` and was returning an error (#121) +- Fix: Trae link corrected from `trae.com` to `trae.ai` in README, README.zh-CN.md, README.ja-JP.md, README.ko-KR.md (#122) +- Docs: Korean README added (README.ko-KR.md) (#112) +- Refactor: `save_query_result` inline Python blocks in all 6 skill files replaced with `graphify save-result` CLI command — shorter, maintainable, less tokens for LLM (#114) +- Add: `graphify save-result` CLI subcommand — saves Q&A results to memory dir without inline Python + ## 0.3.17 (2026-04-08) - Add: Julia (.jl) support — modules, structs, abstract types, functions, short functions, using/import, call edges, inherits edges via tree-sitter-julia (#98) diff --git a/graphify/__main__.py b/graphify/__main__.py index 3dc8f7ac6..093539d37 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -170,7 +170,7 @@ def install(platform: str = "claude") -> None: "type": "command", "command": ( "[ -f graphify-out/graph.json ] && " - r"""echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}}' """ + r"""echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow","systemMessage":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}}' """ "|| true" ), } @@ -388,6 +388,12 @@ def main() -> None: print(" --dfs use depth-first instead of breadth-first") print(" --budget N cap output at N tokens (default 2000)") print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" save-result save a Q&A result to graphify-out/memory/ for graph feedback loop") + print(" --question Q the question asked") + print(" --answer A the answer to save") + print(" --type T query type: query|path_query|explain (default: query)") + print(" --nodes N1 N2 ... source node labels cited in the answer") + print(" --memory-dir DIR memory directory (default: graphify-out/memory)") print(" benchmark [graph.json] measure token reduction vs naive full-corpus approach") print(" hook install install post-commit/post-checkout git hooks (all platforms)") print(" hook uninstall remove git hooks") @@ -518,6 +524,25 @@ def main() -> None: start = [nid for _, nid in scored[:5]] nodes, edges = (_dfs if use_dfs else _bfs)(G, start, depth=2) print(_subgraph_to_text(G, nodes, edges, token_budget=budget)) + elif cmd == "save-result": + # graphify save-result --question Q --answer A --type T [--nodes N1 N2 ...] + import argparse as _ap + p = _ap.ArgumentParser(prog="graphify save-result") + p.add_argument("--question", required=True) + p.add_argument("--answer", required=True) + p.add_argument("--type", dest="query_type", default="query") + p.add_argument("--nodes", nargs="*", default=[]) + p.add_argument("--memory-dir", default="graphify-out/memory") + opts = p.parse_args(sys.argv[2:]) + from graphify.ingest import save_query_result as _sqr + out = _sqr( + question=opts.question, + answer=opts.answer, + memory_dir=Path(opts.memory_dir), + query_type=opts.query_type, + source_nodes=opts.nodes or None, + ) + print(f"Saved to {out}") elif cmd == "benchmark": from graphify.benchmark import run_benchmark, print_benchmark graph_path = sys.argv[2] if len(sys.argv) > 2 else "graphify-out/graph.json" diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index c267a7dfd..ca353efde 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -894,18 +894,7 @@ Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, After writing the answer, save it back into the graph so it improves future queries: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='QUESTION', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='query', - source_nodes=SOURCE_NODES, # list of node labels cited, or [] -) -print('Query result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 ``` Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. @@ -980,18 +969,7 @@ Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Path from NODE_A to NODE_B', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='path_query', - source_nodes=PATH_NODES, # list of node labels on the path -) -print('Path result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B ``` --- @@ -1057,18 +1035,7 @@ Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sent After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Explain NODE_NAME', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='explain', - source_nodes=['NODE_NAME'], -) -print('Explanation saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME ``` --- diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 33d82dfb6..e94f2da40 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -950,18 +950,7 @@ Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, After writing the answer, save it back into the graph so it improves future queries: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='QUESTION', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='query', - source_nodes=SOURCE_NODES, # list of node labels cited, or [] -) -print('Query result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 ``` Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. @@ -1036,18 +1025,7 @@ Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Path from NODE_A to NODE_B', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='path_query', - source_nodes=PATH_NODES, # list of node labels on the path -) -print('Path result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B ``` --- @@ -1113,18 +1091,7 @@ Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sent After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Explain NODE_NAME', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='explain', - source_nodes=['NODE_NAME'], -) -print('Explanation saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME ``` --- diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 7dfb7381d..915dc784c 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -947,18 +947,7 @@ Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, After writing the answer, save it back into the graph so it improves future queries: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='QUESTION', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='query', - source_nodes=SOURCE_NODES, # list of node labels cited, or [] -) -print('Query result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 ``` Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. @@ -1033,18 +1022,7 @@ Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Path from NODE_A to NODE_B', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='path_query', - source_nodes=PATH_NODES, # list of node labels on the path -) -print('Path result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B ``` --- @@ -1110,18 +1088,7 @@ Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sent After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Explain NODE_NAME', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='explain', - source_nodes=['NODE_NAME'], -) -print('Explanation saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME ``` --- diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index e03db2b76..9ced2a61a 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -946,18 +946,7 @@ Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, After writing the answer, save it back into the graph so it improves future queries: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='QUESTION', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='query', - source_nodes=SOURCE_NODES, # list of node labels cited, or [] -) -print('Query result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 ``` Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. @@ -1032,18 +1021,7 @@ Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Path from NODE_A to NODE_B', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='path_query', - source_nodes=PATH_NODES, # list of node labels on the path -) -print('Path result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B ``` --- @@ -1109,18 +1087,7 @@ Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sent After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Explain NODE_NAME', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='explain', - source_nodes=['NODE_NAME'], -) -print('Explanation saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME ``` --- diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 8b2e1ef88..73d2979ef 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -915,18 +915,7 @@ Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, After writing the answer, save it back into the graph so it improves future queries: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='QUESTION', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='query', - source_nodes=SOURCE_NODES, -) -print('Query result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 ``` Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. @@ -1001,18 +990,7 @@ Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Path from NODE_A to NODE_B', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='path_query', - source_nodes=PATH_NODES, -) -print('Path result saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B ``` --- @@ -1077,18 +1055,7 @@ Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sent After writing the explanation, save it back: ```bash -$(cat .graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Explain NODE_NAME', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='explain', - source_nodes=['NODE_NAME'], -) -print('Explanation saved to graphify-out/memory/') -" +$(cat .graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME ``` --- diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index e8f9c9de1..e635c0205 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -937,18 +937,7 @@ Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, After writing the answer, save it back into the graph so it improves future queries: ```powershell -python -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='QUESTION', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='query', - source_nodes=SOURCE_NODES, # list of node labels cited, or [] -) -print('Query result saved to graphify-out/memory/') -" +python -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 ``` Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. @@ -1023,18 +1012,7 @@ Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then After writing the explanation, save it back: ```powershell -python -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Path from NODE_A to NODE_B', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='path_query', - source_nodes=PATH_NODES, # list of node labels on the path -) -print('Path result saved to graphify-out/memory/') -" +python -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B ``` --- @@ -1100,18 +1078,7 @@ Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sent After writing the explanation, save it back: ```powershell -python -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Explain NODE_NAME', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='explain', - source_nodes=['NODE_NAME'], -) -print('Explanation saved to graphify-out/memory/') -" +python -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME ``` --- diff --git a/graphify/skill.md b/graphify/skill.md index bd744e780..ec09523d2 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -958,18 +958,7 @@ Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, After writing the answer, save it back into the graph so it improves future queries: ```bash -$(cat graphify-out/.graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='QUESTION', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='query', - source_nodes=SOURCE_NODES, # list of node labels cited, or [] -) -print('Query result saved to graphify-out/memory/') -" +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 ``` Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. @@ -1044,18 +1033,7 @@ Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then After writing the explanation, save it back: ```bash -$(cat graphify-out/.graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Path from NODE_A to NODE_B', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='path_query', - source_nodes=PATH_NODES, # list of node labels on the path -) -print('Path result saved to graphify-out/memory/') -" +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B ``` --- @@ -1121,18 +1099,7 @@ Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sent After writing the explanation, save it back: ```bash -$(cat graphify-out/.graphify_python) -c " -from graphify.ingest import save_query_result -from pathlib import Path -save_query_result( - question='Explain NODE_NAME', - answer='ANSWER', - memory_dir=Path('graphify-out/memory'), - query_type='explain', - source_nodes=['NODE_NAME'], -) -print('Explanation saved to graphify-out/memory/') -" +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME ``` --- diff --git a/graphify/watch.py b/graphify/watch.py index c0cabdde8..0d92bd6ce 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -24,19 +24,16 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: Returns True on success, False on error. """ try: - from graphify.extract import collect_files, extract + from graphify.extract import extract + from graphify.detect import detect, FileType from graphify.build import build_from_json from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections, suggest_questions from graphify.report import generate from graphify.export import to_json - code_files = collect_files(watch_path, follow_symlinks=follow_symlinks) - code_files = [ - f for f in code_files - if "graphify-out" not in f.parts - and "__pycache__" not in f.parts - ] + detected = detect(watch_path, follow_symlinks=follow_symlinks) + code_files = [Path(f) for f in detected[FileType.CODE]] if not code_files: print("[graphify watch] No code files found - nothing to rebuild.") @@ -47,7 +44,7 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: detection = { "files": {"code": [str(f) for f in code_files], "document": [], "paper": [], "image": []}, "total_files": len(code_files), - "total_words": 0, # not needed during watch rebuild + "total_words": detected.get("total_words", 0), } G = build_from_json(result) From 29c639d97d86e623e700f3f2e1911769dd13f418 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 08:22:01 +0100 Subject: [PATCH 17/59] Apply PRs #82 #93 #102 #109: extension drift, click detection, skill coverage, .graphify_python persistence Co-Authored-By: Claude Sonnet 4.6 --- graphify/analyze.py | 13 +++++-------- graphify/cluster.py | 17 ----------------- graphify/export.py | 23 +++++++++++++++++++++-- graphify/skill-windows.md | 2 +- graphify/skill.md | 22 ++++++++++++++++++++-- graphify/watch.py | 15 ++++----------- pyproject.toml | 4 ++-- tests/test_analyze.py | 9 +++++++++ tests/test_install.py | 28 ++++++++++++++++++++++++++-- 9 files changed, 88 insertions(+), 45 deletions(-) diff --git a/graphify/analyze.py b/graphify/analyze.py index 8ca7ddac2..50f85acaf 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -109,19 +109,16 @@ def _is_concept_node(G: nx.Graph, node_id: str) -> bool: return False -_CODE_EXTENSIONS = {"py", "ts", "tsx", "js", "go", "rs", "java", "rb", "cpp", "c", "h", "cs", "kt", "scala", "php"} -_DOC_EXTENSIONS = {"md", "txt", "rst"} -_PAPER_EXTENSIONS = {"pdf"} -_IMAGE_EXTENSIONS = {"png", "jpg", "jpeg", "webp", "gif", "svg"} +from graphify.detect import CODE_EXTENSIONS, DOC_EXTENSIONS, PAPER_EXTENSIONS, IMAGE_EXTENSIONS def _file_category(path: str) -> str: - ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" - if ext in _CODE_EXTENSIONS: + ext = ("." + path.rsplit(".", 1)[-1].lower()) if "." in path else "" + if ext in CODE_EXTENSIONS: return "code" - if ext in _PAPER_EXTENSIONS: + if ext in PAPER_EXTENSIONS: return "paper" - if ext in _IMAGE_EXTENSIONS: + if ext in IMAGE_EXTENSIONS: return "image" return "doc" diff --git a/graphify/cluster.py b/graphify/cluster.py index 09d070405..363d555c8 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -52,23 +52,6 @@ def _partition(G: nx.Graph) -> dict[str, int]: return {node: cid for cid, nodes in enumerate(communities) for node in nodes} -def build_graph(nodes: list[dict], edges: list[dict]) -> nx.Graph: - """Build a NetworkX graph from graphify node/edge dicts. - - Preserves original edge direction as _src/_tgt attributes so that - display functions can show relationships in the correct direction, - even though the graph is undirected for structural analysis. - """ - G = nx.Graph() - for n in nodes: - G.add_node(n["id"], **{k: v for k, v in n.items() if k != "id"}) - for e in edges: - attrs = {k: v for k, v in e.items() if k not in ("source", "target")} - attrs["_src"] = e["source"] - attrs["_tgt"] = e["target"] - G.add_edge(e["source"], e["target"], **attrs) - return G - _MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split _MIN_SPLIT_SIZE = 10 # only split if community has at least this many nodes diff --git a/graphify/export.py b/graphify/export.py index d7a7349a5..300f93ace 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -183,9 +183,28 @@ def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: showInfo(nodeId); }} +// Track hovered node — hover detection is more reliable than click params +let hoveredNodeId = null; +network.on('hoverNode', params => {{ + hoveredNodeId = params.node; + container.style.cursor = 'pointer'; +}}); +network.on('blurNode', () => {{ + hoveredNodeId = null; + container.style.cursor = 'default'; +}}); +container.addEventListener('click', () => {{ + if (hoveredNodeId !== null) {{ + showInfo(hoveredNodeId); + network.selectNodes([hoveredNodeId]); + }} +}}); network.on('click', params => {{ - if (params.nodes.length > 0) showInfo(params.nodes[0]); - else document.getElementById('info-content').innerHTML = 'Click a node to inspect it'; + if (params.nodes.length > 0) {{ + showInfo(params.nodes[0]); + }} else if (hoveredNodeId === null) {{ + document.getElementById('info-content').innerHTML = 'Click a node to inspect it'; + }} }}); const searchInput = document.getElementById('search'); diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index e635c0205..e0955c61d 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -637,7 +637,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -Remove-Item -ErrorAction SilentlyContinue .graphify_detect.json, .graphify_extract.json, .graphify_ast.json, .graphify_semantic.json, .graphify_analysis.json, .graphify_labels.json, .graphify_python +Remove-Item -ErrorAction SilentlyContinue .graphify_detect.json, .graphify_extract.json, .graphify_ast.json, .graphify_semantic.json, .graphify_analysis.json, .graphify_labels.json Remove-Item -ErrorAction SilentlyContinue graphify-out/.needs_update ``` diff --git a/graphify/skill.md b/graphify/skill.md index ec09523d2..e4049e4b4 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -71,9 +71,9 @@ else PYTHON="python3" fi "$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 -# Write interpreter path for all subsequent steps -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +# Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" ``` If the import succeeds, print nothing and move straight to Step 2. @@ -683,6 +683,24 @@ The graph is the map. Your job after the pipeline is to be the guide. --- +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +fi +``` + ## For --update (incremental re-extraction) Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. diff --git a/graphify/watch.py b/graphify/watch.py index 0d92bd6ce..a02625274 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -5,17 +5,10 @@ from pathlib import Path -_WATCHED_EXTENSIONS = { - ".py", ".ts", ".js", ".go", ".rs", ".java", ".cpp", ".c", ".rb", ".swift", ".kt", - ".cs", ".scala", ".php", ".cc", ".cxx", ".hpp", ".h", ".kts", - ".md", ".txt", ".rst", ".pdf", - ".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg", -} - -_CODE_EXTENSIONS = { - ".py", ".ts", ".js", ".go", ".rs", ".java", ".cpp", ".c", ".rb", ".swift", ".kt", - ".cs", ".scala", ".php", ".cc", ".cxx", ".hpp", ".h", ".kts", -} +from graphify.detect import CODE_EXTENSIONS, DOC_EXTENSIONS, PAPER_EXTENSIONS, IMAGE_EXTENSIONS + +_WATCHED_EXTENSIONS = CODE_EXTENSIONS | DOC_EXTENSIONS | PAPER_EXTENSIONS | IMAGE_EXTENSIONS +_CODE_EXTENSIONS = CODE_EXTENSIONS def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: diff --git a/pyproject.toml b/pyproject.toml index 3ba33a5f7..9a1dac6d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.17" +version = "0.3.18" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } @@ -57,4 +57,4 @@ where = ["."] include = ["graphify*"] [tool.setuptools.package-data] -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-claw.md", "skill-windows.md", "skill-droid.md"] +graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md"] diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 0ae2ede54..2d4961396 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -137,6 +137,15 @@ def test_file_category(): assert _file_category("flash.pdf") == "paper" assert _file_category("diagram.png") == "image" assert _file_category("notes.md") == "doc" + # Languages added in later releases — would misclassify as "doc" without detect.py import + assert _file_category("app.swift") == "code" + assert _file_category("plugin.lua") == "code" + assert _file_category("build.zig") == "code" + assert _file_category("deploy.ps1") == "code" + assert _file_category("server.ex") == "code" + assert _file_category("component.jsx") == "code" + assert _file_category("analysis.jl") == "code" + assert _file_category("view.m") == "code" def test_is_concept_node_empty_source(): diff --git a/tests/test_install.py b/tests/test_install.py index ac8f24b08..7715b4db7 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -9,6 +9,10 @@ "codex": (".agents/skills/graphify/SKILL.md",), "opencode": (".config/opencode/skills/graphify/SKILL.md",), "claw": (".claw/skills/graphify/SKILL.md",), + "droid": (".factory/skills/graphify/SKILL.md",), + "trae": (".trae/skills/graphify/SKILL.md",), + "trae-cn": (".trae-cn/skills/graphify/SKILL.md",), + "windows": (".claude/skills/graphify/SKILL.md",), } @@ -38,6 +42,26 @@ def test_install_claw(tmp_path): assert (tmp_path / ".claw" / "skills" / "graphify" / "SKILL.md").exists() +def test_install_droid(tmp_path): + _install(tmp_path, "droid") + assert (tmp_path / ".factory" / "skills" / "graphify" / "SKILL.md").exists() + + +def test_install_trae(tmp_path): + _install(tmp_path, "trae") + assert (tmp_path / ".trae" / "skills" / "graphify" / "SKILL.md").exists() + + +def test_install_trae_cn(tmp_path): + _install(tmp_path, "trae-cn") + assert (tmp_path / ".trae-cn" / "skills" / "graphify" / "SKILL.md").exists() + + +def test_install_windows(tmp_path): + _install(tmp_path, "windows") + assert (tmp_path / ".claude" / "skills" / "graphify" / "SKILL.md").exists() + + def test_install_unknown_platform_exits(tmp_path): with pytest.raises(SystemExit): _install(tmp_path, "unknown") @@ -67,10 +91,10 @@ def test_claw_skill_is_sequential(): def test_all_skill_files_exist_in_package(): - """All four platform skill files must be present in the installed package.""" + """All installable platform skill files must be present in the installed package.""" import graphify pkg = Path(graphify.__file__).parent - for name in ("skill.md", "skill-codex.md", "skill-opencode.md", "skill-claw.md"): + for name in ("skill.md", "skill-codex.md", "skill-opencode.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md"): assert (pkg / name).exists(), f"Missing: {name}" From 4d8cffef476da8e28891bf185124a955210cddf0 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 08:24:29 +0100 Subject: [PATCH 18/59] Update CHANGELOG for v0.3.18 with PR fixes (#82 #93 #102 #109) Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 847f45488..e6a58ed66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Docs: Korean README added (README.ko-KR.md) (#112) - Refactor: `save_query_result` inline Python blocks in all 6 skill files replaced with `graphify save-result` CLI command — shorter, maintainable, less tokens for LLM (#114) - Add: `graphify save-result` CLI subcommand — saves Q&A results to memory dir without inline Python +- Fix: HTML graph click detection now uses hover-tracking (`hoveredNodeId`) — more reliable than vis.js click params on small/dense nodes (#82) +- Fix: `mkdir -p graphify-out` now runs before writing `.graphify_python` in `skill.md` — prevents write failure on first run; `.graphify_python` no longer deleted in Step 9 cleanup across all skill files so follow-up commands keep their interpreter (#93) +- Fix: `skill-trae.md` added to `pyproject.toml` package-data — Trae users no longer hit `ModuleNotFoundError` after `pip install` (#102) +- Fix: `analyze.py` and `watch.py` now import extension sets from `detect.py` instead of local copies — Swift, Lua, Zig, PowerShell, Elixir, JSX, Julia, Objective-C files no longer misclassified as documents (#109) +- Refactor: dead `build_graph()` function removed from `cluster.py` (#109) ## 0.3.17 (2026-04-08) From f140d4786d8ccfb5c4a64e4cda3fd1b1dbd343b2 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 08:40:17 +0100 Subject: [PATCH 19/59] Fix install step: try plain pip before --break-system-packages (#126) Co-Authored-By: Claude Sonnet 4.6 --- graphify/skill-claw.md | 2 +- graphify/skill-codex.md | 2 +- graphify/skill-droid.md | 2 +- graphify/skill-opencode.md | 2 +- graphify/skill-trae.md | 2 +- graphify/skill.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index ca353efde..eb3409526 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -68,7 +68,7 @@ if [ -n "$GRAPHIFY_BIN" ]; then else PYTHON="python3" fi -"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 mkdir -p graphify-out # Write interpreter path for all subsequent steps "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index e94f2da40..12f17c480 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -68,7 +68,7 @@ if [ -n "$GRAPHIFY_BIN" ]; then else PYTHON="python3" fi -"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" ``` diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 915dc784c..42647e66c 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -68,7 +68,7 @@ if [ -n "$GRAPHIFY_BIN" ]; then else PYTHON="python3" fi -"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 9ced2a61a..1f07cca71 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -68,7 +68,7 @@ if [ -n "$GRAPHIFY_BIN" ]; then else PYTHON="python3" fi -"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 73d2979ef..c0cfffb9e 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -68,7 +68,7 @@ if [ -n "$GRAPHIFY_BIN" ]; then else PYTHON="python3" fi -"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" ``` diff --git a/graphify/skill.md b/graphify/skill.md index e4049e4b4..84815de7e 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -70,7 +70,7 @@ if [ -n "$GRAPHIFY_BIN" ]; then else PYTHON="python3" fi -"$PYTHON" -c "import graphify" 2>/dev/null || pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" From 096a76f314176ca3db03d3c04143d322add60215 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 08:46:26 +0100 Subject: [PATCH 20/59] Release v0.3.19 Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6a58ed66..73b60ee2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.19 (2026-04-09) + +- Fix: install step now tries plain `pip install` before falling back to `--break-system-packages` — Homebrew and PEP 668 managed environments no longer risk environment corruption (#126) + ## 0.3.18 (2026-04-09) - Fix: `--watch` mode now respects `.graphifyignore` — `_rebuild_code` was calling `collect_files()` directly instead of `detect()`, bypassing ignore patterns (#120) diff --git a/pyproject.toml b/pyproject.toml index 9a1dac6d2..7abe4e216 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.18" +version = "0.3.19" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From 350160524245ae12b28bd70fcfabfcd63147b931 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 08:55:07 +0100 Subject: [PATCH 21/59] Add OpenCode tool.execute.before plugin via graphify opencode install (#71) Co-Authored-By: Claude Sonnet 4.6 --- graphify/__main__.py | 91 +++++++++++++++++++++++++++++++++++++++++-- tests/test_install.py | 44 +++++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index 093539d37..6d8fcc5de 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -160,6 +160,87 @@ def install(platform: str = "claude") -> None: _AGENTS_MD_MARKER = "## graphify" +# OpenCode tool.execute.before plugin — fires before every tool call. +# Injects a graph reminder into bash command output when graph.json exists. +_OPENCODE_PLUGIN_JS = """\ +// graphify OpenCode plugin +// Injects a knowledge graph reminder before bash tool calls when the graph exists. +import { existsSync } from "fs"; +import { join } from "path"; + +export const GraphifyPlugin = async ({ directory }) => { + let reminded = false; + + return { + "tool.execute.before": async (input, output) => { + if (reminded) return; + if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; + + if (input.tool === "bash") { + output.args.command = + 'echo "[graphify] Knowledge graph available. Read graphify-out/GRAPH_REPORT.md for god nodes and architecture context before searching files." && ' + + output.args.command; + reminded = true; + } + }, + }; +}; +""" + +_OPENCODE_PLUGIN_PATH = Path(".opencode") / "plugins" / "graphify.js" +_OPENCODE_CONFIG_PATH = Path("opencode.json") + + +def _install_opencode_plugin(project_dir: Path) -> None: + """Write graphify.js plugin and register it in opencode.json.""" + plugin_file = project_dir / _OPENCODE_PLUGIN_PATH + plugin_file.parent.mkdir(parents=True, exist_ok=True) + plugin_file.write_text(_OPENCODE_PLUGIN_JS, encoding="utf-8") + print(f" {_OPENCODE_PLUGIN_PATH} -> tool.execute.before hook written") + + config_file = project_dir / _OPENCODE_CONFIG_PATH + if config_file.exists(): + try: + config = json.loads(config_file.read_text(encoding="utf-8")) + except json.JSONDecodeError: + config = {} + else: + config = {} + + plugins = config.setdefault("plugin", []) + entry = str(_OPENCODE_PLUGIN_PATH) + if entry not in plugins: + plugins.append(entry) + config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") + print(f" {_OPENCODE_CONFIG_PATH} -> plugin registered") + else: + print(f" {_OPENCODE_CONFIG_PATH} -> plugin already registered (no change)") + + +def _uninstall_opencode_plugin(project_dir: Path) -> None: + """Remove graphify.js plugin and deregister from opencode.json.""" + plugin_file = project_dir / _OPENCODE_PLUGIN_PATH + if plugin_file.exists(): + plugin_file.unlink() + print(f" {_OPENCODE_PLUGIN_PATH} -> removed") + + config_file = project_dir / _OPENCODE_CONFIG_PATH + if not config_file.exists(): + return + try: + config = json.loads(config_file.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return + plugins = config.get("plugin", []) + entry = str(_OPENCODE_PLUGIN_PATH) + if entry in plugins: + plugins.remove(entry) + if not plugins: + config.pop("plugin") + config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") + print(f" {_OPENCODE_CONFIG_PATH} -> plugin deregistered") + + _CODEX_HOOK = { "hooks": { "PreToolUse": [ @@ -238,11 +319,13 @@ def _agents_install(project_dir: Path, platform: str) -> None: if platform == "codex": _install_codex_hook(project_dir or Path(".")) + elif platform == "opencode": + _install_opencode_plugin(project_dir or Path(".")) print() print(f"{platform.capitalize()} will now check the knowledge graph before answering") print("codebase questions and rebuild it after code changes.") - if platform != "codex": + if platform not in ("codex", "opencode"): print() print("Note: unlike Claude Code, there is no PreToolUse hook equivalent for") print(f"{platform.capitalize()} — the AGENTS.md rules are the always-on mechanism.") @@ -274,6 +357,8 @@ def _agents_uninstall(project_dir: Path) -> None: target.unlink() print(f"AGENTS.md was empty after removal - deleted {target.resolve()}") + _uninstall_opencode_plugin(project_dir or Path(".")) + def claude_install(project_dir: Path | None = None) -> None: """Write the graphify section to the local CLAUDE.md.""" @@ -402,8 +487,8 @@ def main() -> None: print(" claude uninstall remove graphify section from CLAUDE.md + PreToolUse hook") print(" codex install write graphify section to AGENTS.md (Codex)") print(" codex uninstall remove graphify section from AGENTS.md") - print(" opencode install write graphify section to AGENTS.md (OpenCode)") - print(" opencode uninstall remove graphify section from AGENTS.md") + print(" opencode install write graphify section to AGENTS.md + tool.execute.before plugin (OpenCode)") + print(" opencode uninstall remove graphify section from AGENTS.md + plugin") print(" claw install write graphify section to AGENTS.md (OpenClaw)") print(" claw uninstall remove graphify section from AGENTS.md") print(" droid install write graphify section to AGENTS.md (Factory Droid)") diff --git a/tests/test_install.py b/tests/test_install.py index 7715b4db7..04c2cda15 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -181,3 +181,47 @@ def test_agents_uninstall_no_op_when_not_installed(tmp_path, capsys): _agents_uninstall(tmp_path) out = capsys.readouterr().out assert "nothing to do" in out + + +# --- OpenCode plugin tests --- + +def test_opencode_agents_install_writes_plugin(tmp_path): + """opencode install writes .opencode/plugins/graphify.js.""" + _agents_install(tmp_path, "opencode") + plugin = tmp_path / ".opencode" / "plugins" / "graphify.js" + assert plugin.exists() + assert "tool.execute.before" in plugin.read_text() + + +def test_opencode_agents_install_registers_plugin_in_config(tmp_path): + """opencode install registers the plugin in opencode.json.""" + _agents_install(tmp_path, "opencode") + config_file = tmp_path / "opencode.json" + assert config_file.exists() + import json as _json + config = _json.loads(config_file.read_text()) + assert any("graphify.js" in p for p in config.get("plugin", [])) + + +def test_opencode_agents_install_merges_existing_config(tmp_path): + """opencode install preserves existing opencode.json keys.""" + import json as _json + config_file = tmp_path / "opencode.json" + config_file.write_text(_json.dumps({"model": "claude-opus-4-5", "plugin": []})) + _agents_install(tmp_path, "opencode") + config = _json.loads(config_file.read_text()) + assert config["model"] == "claude-opus-4-5" + assert any("graphify.js" in p for p in config["plugin"]) + + +def test_opencode_agents_uninstall_removes_plugin(tmp_path): + """opencode uninstall removes the plugin file and deregisters from opencode.json.""" + import json as _json + _agents_install(tmp_path, "opencode") + _agents_uninstall(tmp_path) + plugin = tmp_path / ".opencode" / "plugins" / "graphify.js" + assert not plugin.exists() + config_file = tmp_path / "opencode.json" + if config_file.exists(): + config = _json.loads(config_file.read_text()) + assert not any("graphify.js" in p for p in config.get("plugin", [])) From e1864d79c8e6dc4f493333c27f7b254e2ed27a2c Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 08:56:44 +0100 Subject: [PATCH 22/59] Update README for OpenCode tool.execute.before plugin Co-Authored-By: Claude Sonnet 4.6 --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7b758a269..3b3f6489f 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,9 @@ After building a graph, run this once in your project: **Codex** writes to `AGENTS.md` and also installs a **PreToolUse hook** in `.codex/hooks.json` that fires before every Bash tool call — same always-on mechanism as Claude Code. -**OpenCode, OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support PreToolUse hooks, so AGENTS.md is the always-on mechanism. +**OpenCode** writes to `AGENTS.md` and also installs a **`tool.execute.before` plugin** (`.opencode/plugins/graphify.js` + `opencode.json` registration) that fires before bash tool calls and injects the graph reminder into tool output when the graph exists. + +**OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support tool hooks, so AGENTS.md is the always-on mechanism. Uninstall with the matching uninstall command (e.g. `graphify claude uninstall`). @@ -201,7 +203,7 @@ graphify hook status graphify claude install # CLAUDE.md + PreToolUse hook (Claude Code) graphify claude uninstall graphify codex install # AGENTS.md (Codex) -graphify opencode install # AGENTS.md (OpenCode) +graphify opencode install # AGENTS.md + tool.execute.before plugin (OpenCode) graphify claw install # AGENTS.md (OpenClaw) graphify droid install # AGENTS.md (Factory Droid) graphify trae install # AGENTS.md (Trae) From 38255e733f637b394a0faa765dd6093f34b0d22e Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 09:06:02 +0100 Subject: [PATCH 23/59] Fix XSS in HTML viz: escape node labels, types, source files, and edge relations in innerHTML (#sec) Co-Authored-By: Claude Sonnet 4.6 --- graphify/export.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/graphify/export.py b/graphify/export.py index 300f93ace..16271a730 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -110,6 +110,11 @@ def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: const RAW_EDGES = {edges_json}; const LEGEND = {legend_json}; +// HTML-escape helper — prevents XSS when injecting graph data into innerHTML +function esc(s) {{ + return String(s).replace(/&/g,'&').replace(//g,'>').replace(/"/g,'"').replace(/'/g,'''); +}} + // Build vis datasets const nodesDS = new vis.DataSet(RAW_NODES.map(n => ({{ id: n.id, label: n.label, color: n.color, size: n.size, @@ -165,13 +170,13 @@ def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: const neighborItems = neighborIds.map(nid => {{ const nb = nodesDS.get(nid); const color = nb ? nb.color.background : '#555'; - return `${{nb ? nb.label : nid}}`; + return `${{esc(nb ? nb.label : nid)}}`; }}).join(''); document.getElementById('info-content').innerHTML = ` -
${{n.label}}
-
Type: ${{n._file_type || 'unknown'}}
-
Community: ${{n._community_name}}
-
Source: ${{n._source_file || '-'}}
+
${{esc(n.label)}}
+
Type: ${{esc(n._file_type || 'unknown')}}
+
Community: ${{esc(n._community_name)}}
+
Source: ${{esc(n._source_file || '-')}}
Degree: ${{n._degree}}
${{neighborIds.length ? `
Neighbors (${{neighborIds.length}})
${{neighborItems}}
` : ''}} `; @@ -356,7 +361,7 @@ def to_html( "color": {"background": color, "border": color, "highlight": {"background": "#ffffff", "border": color}}, "size": round(size, 1), "font": {"size": font_size, "color": "#ffffff"}, - "title": f"{label}", + "title": _html.escape(label), "community": cid, "community_name": sanitize_label((community_labels or {}).get(cid, f"Community {cid}")), "source_file": sanitize_label(data.get("source_file", "")), @@ -373,7 +378,7 @@ def to_html( "from": u, "to": v, "label": relation, - "title": f"{relation} [{confidence}]", + "title": _html.escape(f"{relation} [{confidence}]"), "dashes": confidence != "EXTRACTED", "width": 2 if confidence == "EXTRACTED" else 1, "color": {"opacity": 0.7 if confidence == "EXTRACTED" else 0.35}, From b7fd5acc38c6e75fc2d8f7cfe95cd13a8deb424c Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 09:22:12 +0100 Subject: [PATCH 24/59] Fix AST call edges confidence: INFERRED/0.8 -> EXTRACTED/1.0 (#127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tree-sitter resolves call targets directly from source — marking them INFERRED was incorrect. Cross-file class-level uses edges remain INFERRED. Co-Authored-By: Claude Sonnet 4.6 --- graphify/extract.py | 20 ++++++++++---------- tests/test_extract.py | 7 ++++--- tests/test_languages.py | 4 ++-- tests/test_multilang.py | 12 ++++++------ 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 0ea50efbc..0fdd4fd57 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -956,10 +956,10 @@ def walk_calls(node, caller_nid: str) -> None: "source": caller_nid, "target": tgt_nid, "relation": "calls", - "confidence": "INFERRED", + "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", - "weight": 0.8, + "weight": 1.0, }) for child in node.children: @@ -1533,10 +1533,10 @@ def walk_calls(node, caller_nid: str) -> None: "source": caller_nid, "target": tgt_nid, "relation": "calls", - "confidence": "INFERRED", + "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", - "weight": 0.8, + "weight": 1.0, }) for child in node.children: walk_calls(child, caller_nid) @@ -1702,10 +1702,10 @@ def walk_calls(node, caller_nid: str) -> None: "source": caller_nid, "target": tgt_nid, "relation": "calls", - "confidence": "INFERRED", + "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", - "weight": 0.8, + "weight": 1.0, }) for child in node.children: walk_calls(child, caller_nid) @@ -1866,7 +1866,7 @@ def walk_calls(node, caller_nid: str) -> None: seen_call_pairs.add(pair) add_edge(caller_nid, tgt_nid, "calls", node.start_point[0] + 1, - confidence="INFERRED", weight=0.8) + confidence="EXTRACTED", weight=1.0) for child in node.children: walk_calls(child, caller_nid) @@ -2022,7 +2022,7 @@ def walk_calls(node, caller_nid: str) -> None: seen_call_pairs.add(pair) add_edge(caller_nid, tgt_nid, "calls", node.start_point[0] + 1, - confidence="INFERRED", weight=0.8) + confidence="EXTRACTED", weight=1.0) for child in node.children: walk_calls(child, caller_nid) @@ -2359,7 +2359,7 @@ def walk_calls(n) -> None: if pair not in seen_calls and caller_nid != candidate: seen_calls.add(pair) add_edge(caller_nid, candidate, "calls", body_node.start_point[0] + 1, - confidence="INFERRED", weight=0.8) + confidence="EXTRACTED", weight=1.0) for child in n.children: walk_calls(child) walk_calls(body_node) @@ -2532,7 +2532,7 @@ def walk_calls(node, caller_nid: str) -> None: if pair not in seen_call_pairs: seen_call_pairs.add(pair) add_edge(caller_nid, tgt_nid, "calls", - node.start_point[0] + 1, confidence="INFERRED", weight=0.8) + node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) for child in node.children: walk_calls(child, caller_nid) diff --git a/tests/test_extract.py b/tests/test_extract.py index a852db987..3d5b9f530 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -115,12 +115,13 @@ def test_calls_edges_emitted(): assert len(calls) > 0, "Expected at least one calls edge" -def test_calls_edges_are_inferred(): +def test_calls_edges_are_extracted(): + """AST-resolved call edges are deterministic and should be EXTRACTED/1.0.""" result = extract_python(FIXTURES / "sample_calls.py") for edge in result["edges"]: if edge["relation"] == "calls": - assert edge["confidence"] == "INFERRED" - assert edge["weight"] == 0.8 + assert edge["confidence"] == "EXTRACTED" + assert edge["weight"] == 1.0 def test_calls_no_self_loops(): diff --git a/tests/test_languages.py b/tests/test_languages.py index 9784dfa95..0cf93c0d5 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -76,11 +76,11 @@ def test_c_emits_calls(): r = extract_c(FIXTURES / "sample.c") assert any(e["relation"] == "calls" for e in r["edges"]) -def test_c_calls_are_inferred(): +def test_c_calls_are_extracted(): r = extract_c(FIXTURES / "sample.c") for e in r["edges"]: if e["relation"] == "calls": - assert e["confidence"] == "INFERRED" + assert e["confidence"] == "EXTRACTED" # ── C++ ─────────────────────────────────────────────────────────────────────── diff --git a/tests/test_multilang.py b/tests/test_multilang.py index e56fa0af8..0a67f50b3 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -47,11 +47,11 @@ def test_ts_emits_calls(): # .post() calls .get() assert any("post" in src and "get" in tgt for src, tgt in calls) -def test_ts_calls_are_inferred(): +def test_ts_calls_are_extracted(): r = extract_js(FIXTURES / "sample.ts") for e in r["edges"]: if e["relation"] == "calls": - assert e["confidence"] == "INFERRED" + assert e["confidence"] == "EXTRACTED" def test_ts_no_dangling_edges(): r = extract_js(FIXTURES / "sample.ts") @@ -83,9 +83,9 @@ def test_go_emits_calls(): # main() calls NewServer and Start assert len(_call_pairs(r)) > 0 -def test_go_has_inferred_calls(): +def test_go_has_extracted_calls(): r = extract_go(FIXTURES / "sample.go") - assert "INFERRED" in _confidences(r) + assert "EXTRACTED" in _confidences(r) def test_go_no_dangling_edges(): r = extract_go(FIXTURES / "sample.go") @@ -117,11 +117,11 @@ def test_rust_emits_calls(): calls = _call_pairs(r) assert any("build_graph" in src for src, _ in calls) -def test_rust_calls_are_inferred(): +def test_rust_calls_are_extracted(): r = extract_rust(FIXTURES / "sample.rs") for e in r["edges"]: if e["relation"] == "calls": - assert e["confidence"] == "INFERRED" + assert e["confidence"] == "EXTRACTED" def test_rust_no_dangling_edges(): r = extract_rust(FIXTURES / "sample.rs") From b101a99d2ff801fb6b09ec33f7f0ca8f583240b5 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 09:36:32 +0100 Subject: [PATCH 25/59] Pin tree-sitter>=0.23.0, add version guard, confidence=EXTRACTED for AST calls Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 7 +++++++ graphify/extract.py | 20 ++++++++++++++++++++ pyproject.toml | 4 ++-- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73b60ee2b..37579c1e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.20 (2026-04-09) + +- Fix: XSS in interactive HTML graph — node labels, file types, community names, source files, and edge relations now HTML-escaped before `innerHTML` injection; neighbor link `onclick` uses `JSON.stringify` instead of raw string interpolation +- Add: OpenCode `tool.execute.before` plugin — `graphify opencode install` now writes `.opencode/plugins/graphify.js` and registers it in `opencode.json`, firing the graph reminder before bash calls (equivalent to Claude Code's PreToolUse hook) (#71) +- Fix: AST-resolved call edges now carry `confidence=EXTRACTED, weight=1.0` instead of INFERRED/0.8 — tree-sitter call resolution is deterministic, not probabilistic (#127) +- Fix: `tree-sitter>=0.23.0` now pinned in dependencies and `_check_tree_sitter_version()` guard added — stale environments now get a clear `RuntimeError` with upgrade instructions instead of a cryptic `TypeError` deep in the AST pipeline (#89) + ## 0.3.19 (2026-04-09) - Fix: install step now tries plain `pip install` before falling back to `--break-system-packages` — Homebrew and PEP 668 managed environments no longer risk environment corruption (#126) diff --git a/graphify/extract.py b/graphify/extract.py index 0fdd4fd57..c767e07f4 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2546,6 +2546,25 @@ def walk_calls(node, caller_nid: str) -> None: # ── Main extract and collect_files ──────────────────────────────────────────── + +def _check_tree_sitter_version() -> None: + """Raise a clear error if tree-sitter is too old for the new Language API.""" + try: + from tree_sitter import LANGUAGE_VERSION + except ImportError: + raise ImportError( + "tree-sitter is not installed. Run: pip install 'tree-sitter>=0.23.0'" + ) + # Language API v2 starts at LANGUAGE_VERSION 14 + if LANGUAGE_VERSION < 14: + import tree_sitter as _ts + raise RuntimeError( + f"tree-sitter {getattr(_ts, '__version__', 'unknown')} is too old. " + f"graphify requires tree-sitter >= 0.23.0 (Language API v2). " + f"Run: pip install --upgrade tree-sitter" + ) + + def extract(paths: list[Path]) -> dict: """Extract AST nodes and edges from a list of code files. @@ -2554,6 +2573,7 @@ def extract(paths: list[Path]) -> dict: 2. Cross-file import resolution: turns file-level imports into class-level INFERRED edges (DigestAuth --uses--> Response) """ + _check_tree_sitter_version() per_file: list[dict] = [] # Infer a common root for cache keys diff --git a/pyproject.toml b/pyproject.toml index 7abe4e216..eec49bb53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.19" +version = "0.3.20" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } @@ -12,7 +12,7 @@ keywords = ["claude", "claude-code", "codex", "opencode", "knowledge-graph", "ra requires-python = ">=3.10" dependencies = [ "networkx", - "tree-sitter>=0.21", + "tree-sitter>=0.23.0", "tree-sitter-python", "tree-sitter-javascript", "tree-sitter-typescript", From 6f9fc65af4cecfbc05e9822fe8e35fe415742a34 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 14:12:36 +0100 Subject: [PATCH 26/59] Fix Codex hook JSON schema (#138) and use #!/bin/sh for Windows git hooks (#140) Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 5 +++++ graphify/__main__.py | 2 +- graphify/hooks.py | 4 ++-- pyproject.toml | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37579c1e7..55d8fb068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.21 (2026-04-09) + +- Fix: Codex PreToolUse hook now places `systemMessage` at the top level of the output JSON instead of inside `hookSpecificOutput` — matches the strict schema enforced by codex-cli 0.118.0+ which uses `additionalProperties: false` (#138) +- Fix: git hooks now use `#!/bin/sh` instead of `#!/bin/bash` — Git for Windows ships `sh.exe` not `bash`, so hooks were silently skipped on Windows (#140) + ## 0.3.20 (2026-04-09) - Fix: XSS in interactive HTML graph — node labels, file types, community names, source files, and edge relations now HTML-escaped before `innerHTML` injection; neighbor link `onclick` uses `JSON.stringify` instead of raw string interpolation diff --git a/graphify/__main__.py b/graphify/__main__.py index 6d8fcc5de..3f31b7cc4 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -251,7 +251,7 @@ def _uninstall_opencode_plugin(project_dir: Path) -> None: "type": "command", "command": ( "[ -f graphify-out/graph.json ] && " - r"""echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow","systemMessage":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}}' """ + r"""echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow"},"systemMessage":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}' """ "|| true" ), } diff --git a/graphify/hooks.py b/graphify/hooks.py index f2c74e3c4..92320272b 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -124,7 +124,7 @@ def _install_hook(hooks_dir: Path, name: str, script: str, marker: str) -> str: return f"already installed at {hook_path}" hook_path.write_text(content.rstrip() + "\n\n" + script) return f"appended to existing {name} hook at {hook_path}" - hook_path.write_text("#!/bin/bash\n" + script) + hook_path.write_text("#!/bin/sh\n" + script) hook_path.chmod(0o755) return f"installed at {hook_path}" @@ -143,7 +143,7 @@ def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> content, flags=re.DOTALL, ).strip() - if not new_content or new_content == "#!/bin/bash": + if not new_content or new_content in ("#!/bin/bash", "#!/bin/sh"): hook_path.unlink() return f"removed {name} hook at {hook_path}" hook_path.write_text(new_content + "\n") diff --git a/pyproject.toml b/pyproject.toml index eec49bb53..627abc434 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.20" +version = "0.3.21" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From a104a0d9fc782f1485ad35cadd14aa6e4cf8a0e8 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 14:46:13 +0100 Subject: [PATCH 27/59] Add monthly PyPI downloads badge to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3b3f6489f..6423a8774 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) +[![Downloads](https://img.shields.io/pypi/dm/graphifyy)](https://pypi.org/project/graphifyy/) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) **An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. From f770712704265c0ae7823f10950556ff194d8513 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 17:19:40 +0100 Subject: [PATCH 28/59] Add Cursor support, fix _rebuild_code KeyError and node_link_data crash (#137, #148, #149) Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 7 ++++++ README.md | 3 ++- graphify/__main__.py | 50 +++++++++++++++++++++++++++++++++++++++++++ graphify/export.py | 5 ++++- graphify/watch.py | 4 ++-- pyproject.toml | 2 +- tests/test_install.py | 38 ++++++++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55d8fb068..82f0722e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.22 (2026-04-09) + +- Add: Cursor support — `graphify cursor install` writes `.cursor/rules/graphify.mdc` with `alwaysApply: true` so the graph context is always included; `graphify cursor uninstall` removes it (#137) +- Fix: `_rebuild_code()` KeyError — `detected[FileType.CODE]` corrected to `detected['files']['code']` matching `detect()`'s actual return shape; was silently breaking git hooks on every commit (#148) +- Fix: `to_json()` crash on NetworkX 3.2.x — `node_link_data(G, edges="links")` now falls back to `node_link_data(G)` on older NetworkX, same shim already used for `node_link_graph` (#149) +- Fix: README clarifies `graphifyy` is the only official PyPI package — other `graphify*` packages are not affiliated (#129) + ## 0.3.21 (2026-04-09) - Fix: Codex PreToolUse hook now places `systemMessage` at the top level of the output JSON instead of inside `hookSpecificOutput` — matches the strict schema enforced by codex-cli 0.118.0+ which uses `additionalProperties: false` (#138) diff --git a/README.md b/README.md index 6423a8774..f018f02cb 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Every relationship is tagged `EXTRACTED` (found directly in source), `INFERRED` pip install graphifyy && graphify install ``` -> The PyPI package is temporarily named `graphifyy` while the `graphify` name is being reclaimed. The CLI and skill command are still `graphify`. +> **Official package:** The PyPI package is named `graphifyy` (install with `pip install graphifyy`). Other packages named `graphify*` on PyPI are not affiliated with this project. The only official repository is [safishamsi/graphify](https://github.com/safishamsi/graphify). The CLI and skill command are still `graphify`. ### Platform support @@ -91,6 +91,7 @@ After building a graph, run this once in your project: | Factory Droid | `graphify droid install` | | Trae | `graphify trae install` | | Trae CN | `graphify trae-cn install` | +| Cursor | `graphify cursor install` | **Claude Code** does two things: writes a `CLAUDE.md` section telling Claude to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs a **PreToolUse hook** (`settings.json`) that fires before every Glob and Grep call. If a knowledge graph exists, Claude sees: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ — so Claude navigates via the graph instead of grepping through every file. diff --git a/graphify/__main__.py b/graphify/__main__.py index 3f31b7cc4..31f54fff0 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -160,6 +160,45 @@ def install(platform: str = "claude") -> None: _AGENTS_MD_MARKER = "## graphify" +_CURSOR_RULE_PATH = Path(".cursor") / "rules" / "graphify.mdc" +_CURSOR_RULE = """\ +--- +description: graphify knowledge graph context +alwaysApply: true +--- + +This project has a graphify knowledge graph at graphify-out/. + +- Before answering architecture or codebase questions, read graphify-out/GRAPH_REPORT.md for god nodes and community structure +- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files +- After modifying code files in this session, run `python3 -c "from graphify.watch import _rebuild_code; from pathlib import Path; _rebuild_code(Path('.'))"` to keep the graph current +""" + + +def _cursor_install(project_dir: Path) -> None: + """Write .cursor/rules/graphify.mdc with alwaysApply: true.""" + rule_path = (project_dir or Path(".")) / _CURSOR_RULE_PATH + rule_path.parent.mkdir(parents=True, exist_ok=True) + if rule_path.exists(): + print(f"graphify rule already exists at {rule_path} (no change)") + return + rule_path.write_text(_CURSOR_RULE, encoding="utf-8") + print(f"graphify rule written to {rule_path.resolve()}") + print() + print("Cursor will now always include the knowledge graph context.") + print("Run /graphify . first to build the graph if you haven't already.") + + +def _cursor_uninstall(project_dir: Path) -> None: + """Remove .cursor/rules/graphify.mdc.""" + rule_path = (project_dir or Path(".")) / _CURSOR_RULE_PATH + if not rule_path.exists(): + print("No graphify Cursor rule found - nothing to do") + return + rule_path.unlink() + print(f"graphify Cursor rule removed from {rule_path.resolve()}") + + # OpenCode tool.execute.before plugin — fires before every tool call. # Injects a graph reminder into bash command output when graph.json exists. _OPENCODE_PLUGIN_JS = """\ @@ -483,6 +522,8 @@ def main() -> None: print(" hook install install post-commit/post-checkout git hooks (all platforms)") print(" hook uninstall remove git hooks") print(" hook status check if git hooks are installed") + print(" cursor install write .cursor/rules/graphify.mdc (Cursor)") + print(" cursor uninstall remove .cursor/rules/graphify.mdc") print(" claude install write graphify section to CLAUDE.md + PreToolUse hook (Claude Code)") print(" claude uninstall remove graphify section from CLAUDE.md + PreToolUse hook") print(" codex install write graphify section to AGENTS.md (Codex)") @@ -526,6 +567,15 @@ def main() -> None: else: print("Usage: graphify claude [install|uninstall]", file=sys.stderr) sys.exit(1) + elif cmd == "cursor": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + _cursor_install(Path(".")) + elif subcmd == "uninstall": + _cursor_uninstall(Path(".")) + else: + print("Usage: graphify cursor [install|uninstall]", file=sys.stderr) + sys.exit(1) elif cmd in ("codex", "opencode", "claw", "droid", "trae", "trae-cn"): subcmd = sys.argv[2] if len(sys.argv) > 2 else "" if subcmd == "install": diff --git a/graphify/export.py b/graphify/export.py index 16271a730..498e514ae 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -284,7 +284,10 @@ def attach_hyperedges(G: nx.Graph, hyperedges: list) -> None: def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> None: node_community = _node_community_map(communities) - data = json_graph.node_link_data(G, edges="links") + try: + data = json_graph.node_link_data(G, edges="links") + except TypeError: + data = json_graph.node_link_data(G) for node in data["nodes"]: node["community"] = node_community.get(node["id"]) for link in data["links"]: diff --git a/graphify/watch.py b/graphify/watch.py index a02625274..734de8bf0 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -18,7 +18,7 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: """ try: from graphify.extract import extract - from graphify.detect import detect, FileType + from graphify.detect import detect from graphify.build import build_from_json from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections, suggest_questions @@ -26,7 +26,7 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: from graphify.export import to_json detected = detect(watch_path, follow_symlinks=follow_symlinks) - code_files = [Path(f) for f in detected[FileType.CODE]] + code_files = [Path(f) for f in detected['files']['code']] if not code_files: print("[graphify watch] No code files found - nothing to rebuild.") diff --git a/pyproject.toml b/pyproject.toml index 627abc434..6a06bf43f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.21" +version = "0.3.22" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } diff --git a/tests/test_install.py b/tests/test_install.py index 04c2cda15..34d1761e2 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -225,3 +225,41 @@ def test_opencode_agents_uninstall_removes_plugin(tmp_path): if config_file.exists(): config = _json.loads(config_file.read_text()) assert not any("graphify.js" in p for p in config.get("plugin", [])) + + +# ── Cursor ──────────────────────────────────────────────────────────────────── + +def test_cursor_install_writes_rule(tmp_path): + """cursor install writes .cursor/rules/graphify.mdc.""" + from graphify.__main__ import _cursor_install + _cursor_install(tmp_path) + rule = tmp_path / ".cursor" / "rules" / "graphify.mdc" + assert rule.exists() + content = rule.read_text() + assert "alwaysApply: true" in content + assert "graphify-out/GRAPH_REPORT.md" in content + + +def test_cursor_install_idempotent(tmp_path): + """cursor install does not overwrite an existing rule file.""" + from graphify.__main__ import _cursor_install + _cursor_install(tmp_path) + rule = tmp_path / ".cursor" / "rules" / "graphify.mdc" + original = rule.read_text() + _cursor_install(tmp_path) + assert rule.read_text() == original + + +def test_cursor_uninstall_removes_rule(tmp_path): + """cursor uninstall removes the rule file.""" + from graphify.__main__ import _cursor_install, _cursor_uninstall + _cursor_install(tmp_path) + _cursor_uninstall(tmp_path) + rule = tmp_path / ".cursor" / "rules" / "graphify.mdc" + assert not rule.exists() + + +def test_cursor_uninstall_noop_if_not_installed(tmp_path): + """cursor uninstall does nothing if rule was never written.""" + from graphify.__main__ import _cursor_uninstall + _cursor_uninstall(tmp_path) # should not raise From 0b8067c9d1f1d8e07985e60a8316d5f18a98dd45 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 17:25:36 +0100 Subject: [PATCH 29/59] Add Cursor to description, README intro, and PyPI keywords --- README.md | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f018f02cb..d81b430c1 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Downloads](https://img.shields.io/pypi/dm/graphifyy)](https://pypi.org/project/graphifyy/) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) -**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. +**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 20 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia). diff --git a/pyproject.toml b/pyproject.toml index 6a06bf43f..93f20cf21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,10 +5,10 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" version = "0.3.22" -description = "AI coding assistant skill (Claude Code, Codex, OpenCode, OpenClaw) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" +description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } -keywords = ["claude", "claude-code", "codex", "opencode", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] +keywords = ["claude", "claude-code", "codex", "opencode", "cursor", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] requires-python = ">=3.10" dependencies = [ "networkx", From dcc402e14097da7e748ad7d6ab38878fd5a867e1 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 17:40:45 +0100 Subject: [PATCH 30/59] Add Gemini CLI support and sponsor nudge at pipeline completion (#105) Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 5 ++ README.md | 2 +- graphify/__main__.py | 113 +++++++++++++++++++++++++++++++++++++ graphify/skill-claw.md | 2 + graphify/skill-codex.md | 2 + graphify/skill-droid.md | 2 + graphify/skill-opencode.md | 2 + graphify/skill-trae.md | 2 + graphify/skill-windows.md | 2 + graphify/skill.md | 2 + pyproject.toml | 2 +- tests/test_install.py | 55 ++++++++++++++++++ 12 files changed, 189 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82f0722e6..147960c3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.23 (2026-04-09) + +- Add: Gemini CLI support — `graphify gemini install` writes a `GEMINI.md` section and a `BeforeTool` hook in `.gemini/settings.json` that fires before file-read tool calls (#105) +- Add: sponsor nudge at pipeline completion — all skill files now print a one-line sponsor link after a fresh build, not on `--update` runs + ## 0.3.22 (2026-04-09) - Add: Cursor support — `graphify cursor install` writes `.cursor/rules/graphify.mdc` with `alwaysApply: true` so the graph context is always included; `graphify cursor uninstall` removes it (#137) diff --git a/README.md b/README.md index d81b430c1..e5a13d886 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Downloads](https://img.shields.io/pypi/dm/graphifyy)](https://pypi.org/project/graphifyy/) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) -**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. +**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 20 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia). diff --git a/graphify/__main__.py b/graphify/__main__.py index 31f54fff0..c2c27d4ae 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -160,6 +160,108 @@ def install(platform: str = "claude") -> None: _AGENTS_MD_MARKER = "## graphify" +_GEMINI_MD_SECTION = """\ +## graphify + +This project has a graphify knowledge graph at graphify-out/. + +Rules: +- Before answering architecture or codebase questions, read graphify-out/GRAPH_REPORT.md for god nodes and community structure +- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files +- After modifying code files in this session, run `python3 -c "from graphify.watch import _rebuild_code; from pathlib import Path; _rebuild_code(Path('.'))"` to keep the graph current +""" + +_GEMINI_MD_MARKER = "## graphify" + +_GEMINI_HOOK = { + "matcher": "read_file|list_directory", + "hooks": [ + { + "type": "command", + "command": ( + "[ -f graphify-out/graph.json ] && " + r"""echo '{"decision":"allow","additionalContext":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}' """ + r"""|| echo '{"decision":"allow"}'""" + ), + } + ], +} + + +def gemini_install(project_dir: Path | None = None) -> None: + """Write the graphify section to GEMINI.md and install BeforeTool hook.""" + target = (project_dir or Path(".")) / "GEMINI.md" + + if target.exists(): + content = target.read_text(encoding="utf-8") + if _GEMINI_MD_MARKER in content: + print("graphify already configured in GEMINI.md") + else: + target.write_text(content.rstrip() + "\n\n" + _GEMINI_MD_SECTION, encoding="utf-8") + print(f"graphify section written to {target.resolve()}") + else: + target.write_text(_GEMINI_MD_SECTION, encoding="utf-8") + print(f"graphify section written to {target.resolve()}") + + _install_gemini_hook(project_dir or Path(".")) + print() + print("Gemini CLI will now check the knowledge graph before answering") + print("codebase questions and rebuild it after code changes.") + + +def _install_gemini_hook(project_dir: Path) -> None: + settings_path = project_dir / ".gemini" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + try: + settings = json.loads(settings_path.read_text(encoding="utf-8")) if settings_path.exists() else {} + except json.JSONDecodeError: + settings = {} + before_tool = settings.setdefault("hooks", {}).setdefault("BeforeTool", []) + if any("graphify" in str(h) for h in before_tool): + print(" .gemini/settings.json -> hook already registered (no change)") + return + before_tool.append(_GEMINI_HOOK) + settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") + print(" .gemini/settings.json -> BeforeTool hook registered") + + +def _uninstall_gemini_hook(project_dir: Path) -> None: + settings_path = project_dir / ".gemini" / "settings.json" + if not settings_path.exists(): + return + try: + settings = json.loads(settings_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return + before_tool = settings.get("hooks", {}).get("BeforeTool", []) + filtered = [h for h in before_tool if "graphify" not in str(h)] + if len(filtered) == len(before_tool): + return + settings["hooks"]["BeforeTool"] = filtered + settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") + print(" .gemini/settings.json -> BeforeTool hook removed") + + +def gemini_uninstall(project_dir: Path | None = None) -> None: + """Remove the graphify section from GEMINI.md and uninstall hook.""" + target = (project_dir or Path(".")) / "GEMINI.md" + if not target.exists(): + print("No GEMINI.md found in current directory - nothing to do") + return + content = target.read_text(encoding="utf-8") + if _GEMINI_MD_MARKER not in content: + print("graphify section not found in GEMINI.md - nothing to do") + return + cleaned = re.sub(r"\n*## graphify\n.*?(?=\n## |\Z)", "", content, flags=re.DOTALL).rstrip() + if cleaned: + target.write_text(cleaned + "\n", encoding="utf-8") + print(f"graphify section removed from {target.resolve()}") + else: + target.unlink() + print(f"GEMINI.md was empty after removal - deleted {target.resolve()}") + _uninstall_gemini_hook(project_dir or Path(".")) + + _CURSOR_RULE_PATH = Path(".cursor") / "rules" / "graphify.mdc" _CURSOR_RULE = """\ --- @@ -522,6 +624,8 @@ def main() -> None: print(" hook install install post-commit/post-checkout git hooks (all platforms)") print(" hook uninstall remove git hooks") print(" hook status check if git hooks are installed") + print(" gemini install write GEMINI.md section + BeforeTool hook (Gemini CLI)") + print(" gemini uninstall remove GEMINI.md section + BeforeTool hook") print(" cursor install write .cursor/rules/graphify.mdc (Cursor)") print(" cursor uninstall remove .cursor/rules/graphify.mdc") print(" claude install write graphify section to CLAUDE.md + PreToolUse hook (Claude Code)") @@ -567,6 +671,15 @@ def main() -> None: else: print("Usage: graphify claude [install|uninstall]", file=sys.stderr) sys.exit(1) + elif cmd == "gemini": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + gemini_install() + elif subcmd == "uninstall": + gemini_uninstall() + else: + print("Usage: graphify gemini [install|uninstall]", file=sys.stderr) + sys.exit(1) elif cmd == "cursor": subcmd = sys.argv[2] if len(sys.argv) > 2 else "" if subcmd == "install": diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index eb3409526..73eff7f33 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -608,6 +608,8 @@ Graph complete. Outputs in PATH_TO_DIR/graphify-out/ obsidian/ - Obsidian vault (only if --obsidian was given) ``` +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. Then paste these sections from GRAPH_REPORT.md directly into the chat: diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 12f17c480..d14a90bf3 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -664,6 +664,8 @@ Graph complete. Outputs in PATH_TO_DIR/graphify-out/ obsidian/ - Obsidian vault (only if --obsidian was given) ``` +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. Then paste these sections from GRAPH_REPORT.md directly into the chat: diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 42647e66c..b36399db2 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -661,6 +661,8 @@ Graph complete. Outputs in PATH_TO_DIR/graphify-out/ obsidian/ - Obsidian vault (only if --obsidian was given) ``` +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. Then paste these sections from GRAPH_REPORT.md directly into the chat: diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 1f07cca71..ad4318403 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -660,6 +660,8 @@ Graph complete. Outputs in PATH_TO_DIR/graphify-out/ obsidian/ - Obsidian vault (only if --obsidian was given) ``` +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. Then paste these sections from GRAPH_REPORT.md directly into the chat: diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index c0cfffb9e..2711dcd18 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -639,6 +639,8 @@ Graph complete. Outputs in PATH_TO_DIR/graphify-out/ obsidian/ - Obsidian vault (only if --obsidian was given) ``` +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. Then paste these sections from GRAPH_REPORT.md directly into the chat: diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index e0955c61d..1f174cb80 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -651,6 +651,8 @@ Graph complete. Outputs in PATH_TO_DIR/graphify-out/ obsidian/ - Obsidian vault (only if --obsidian was given) ``` +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. Then paste these sections from GRAPH_REPORT.md directly into the chat: diff --git a/graphify/skill.md b/graphify/skill.md index 84815de7e..72d0f2da4 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -664,6 +664,8 @@ Graph complete. Outputs in PATH_TO_DIR/graphify-out/ obsidian/ - Obsidian vault (only if --obsidian was given) ``` +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. Then paste these sections from GRAPH_REPORT.md directly into the chat: diff --git a/pyproject.toml b/pyproject.toml index 93f20cf21..a18d0873b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.22" +version = "0.3.23" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } diff --git a/tests/test_install.py b/tests/test_install.py index 34d1761e2..f8353ac4d 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -263,3 +263,58 @@ def test_cursor_uninstall_noop_if_not_installed(tmp_path): """cursor uninstall does nothing if rule was never written.""" from graphify.__main__ import _cursor_uninstall _cursor_uninstall(tmp_path) # should not raise + + +# ── Gemini CLI ──────────────────────────────────────────────────────────────── + +def test_gemini_install_writes_gemini_md(tmp_path): + from graphify.__main__ import gemini_install + gemini_install(tmp_path) + md = tmp_path / "GEMINI.md" + assert md.exists() + assert "graphify-out/GRAPH_REPORT.md" in md.read_text() + +def test_gemini_install_writes_hook(tmp_path): + import json as _json + from graphify.__main__ import gemini_install + gemini_install(tmp_path) + settings = _json.loads((tmp_path / ".gemini" / "settings.json").read_text()) + hooks = settings["hooks"]["BeforeTool"] + assert any("graphify" in str(h) for h in hooks) + +def test_gemini_install_idempotent(tmp_path): + from graphify.__main__ import gemini_install + gemini_install(tmp_path) + gemini_install(tmp_path) + md = tmp_path / "GEMINI.md" + assert md.read_text().count("## graphify") == 1 + +def test_gemini_install_merges_existing_gemini_md(tmp_path): + from graphify.__main__ import gemini_install + (tmp_path / "GEMINI.md").write_text("# My project rules\n") + gemini_install(tmp_path) + content = (tmp_path / "GEMINI.md").read_text() + assert "# My project rules" in content + assert "graphify-out/GRAPH_REPORT.md" in content + +def test_gemini_uninstall_removes_section(tmp_path): + from graphify.__main__ import gemini_install, gemini_uninstall + gemini_install(tmp_path) + gemini_uninstall(tmp_path) + md = tmp_path / "GEMINI.md" + assert not md.exists() + +def test_gemini_uninstall_removes_hook(tmp_path): + import json as _json + from graphify.__main__ import gemini_install, gemini_uninstall + gemini_install(tmp_path) + gemini_uninstall(tmp_path) + settings_path = tmp_path / ".gemini" / "settings.json" + if settings_path.exists(): + settings = _json.loads(settings_path.read_text()) + hooks = settings.get("hooks", {}).get("BeforeTool", []) + assert not any("graphify" in str(h) for h in hooks) + +def test_gemini_uninstall_noop_if_not_installed(tmp_path): + from graphify.__main__ import gemini_uninstall + gemini_uninstall(tmp_path) # should not raise From 687fbc810a2a7aecde254b01a0860f5e8caeb3c4 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 17:44:59 +0100 Subject: [PATCH 31/59] Update README: add Cursor and Gemini CLI to all platform sections --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e5a13d886..4733adaa8 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Every relationship is tagged `EXTRACTED` (found directly in source), `INFERRED` ## Install -**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), or [Trae](https://trae.ai) +**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [Cursor](https://cursor.com), [Gemini CLI](https://github.com/google-gemini/gemini-cli), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), or [Trae](https://trae.ai) ```bash pip install graphifyy && graphify install @@ -67,6 +67,8 @@ pip install graphifyy && graphify install | Factory Droid | `graphify install --platform droid` | | Trae | `graphify install --platform trae` | | Trae CN | `graphify install --platform trae-cn` | +| Gemini CLI | `graphify install --platform gemini` | +| Cursor | `graphify cursor install` | Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw uses sequential extraction (parallel agent support is still early on that platform). Trae uses the Agent tool for parallel subagent dispatch and does **not** support PreToolUse hooks — AGENTS.md is the always-on mechanism. @@ -92,6 +94,7 @@ After building a graph, run this once in your project: | Trae | `graphify trae install` | | Trae CN | `graphify trae-cn install` | | Cursor | `graphify cursor install` | +| Gemini CLI | `graphify gemini install` | **Claude Code** does two things: writes a `CLAUDE.md` section telling Claude to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs a **PreToolUse hook** (`settings.json`) that fires before every Glob and Grep call. If a knowledge graph exists, Claude sees: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ — so Claude navigates via the graph instead of grepping through every file. @@ -99,6 +102,10 @@ After building a graph, run this once in your project: **OpenCode** writes to `AGENTS.md` and also installs a **`tool.execute.before` plugin** (`.opencode/plugins/graphify.js` + `opencode.json` registration) that fires before bash tool calls and injects the graph reminder into tool output when the graph exists. +**Cursor** writes `.cursor/rules/graphify.mdc` with `alwaysApply: true` — Cursor includes it in every conversation automatically, no hook needed. + +**Gemini CLI** writes a `GEMINI.md` section and installs a `BeforeTool` hook in `.gemini/settings.json` that fires before file-read tool calls — same always-on mechanism as Claude Code. + **OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support tool hooks, so AGENTS.md is the always-on mechanism. Uninstall with the matching uninstall command (e.g. `graphify claude uninstall`). From 54c6d04b7544b59605bb0c01d00907195560e287 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 17:46:34 +0100 Subject: [PATCH 32/59] Switch star history to star-history.com for faster updates --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4733adaa8..8f5b8b357 100644 --- a/README.md +++ b/README.md @@ -285,7 +285,7 @@ graphify is the graph layer. We are building [Penpax](https://safishamsi.github. ## Star history -[![Star History Chart](https://starchart.cc/safishamsi/graphify.svg)](https://starchart.cc/safishamsi/graphify) +[![Star History Chart](https://api.star-history.com/svg?repos=safishamsi/graphify&type=Date)](https://star-history.com/#safishamsi/graphify&Date)
Contributing From ee43236c164196746bfb1f10eec74d264ba5a982 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 17:59:16 +0100 Subject: [PATCH 33/59] =?UTF-8?q?Fix=20codex/opencode=20install=20idempote?= =?UTF-8?q?ncy=20=E2=80=94=20always=20run=20hook=20step=20(#153)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ graphify/__main__.py | 11 +++++------ pyproject.toml | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 147960c3f..914eb1fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.24 (2026-04-09) + +- Fix: `graphify codex install` (and opencode) no longer exits early when `AGENTS.md` already has the graphify section — partial installs with a missing `.codex/hooks.json` can now recover on re-run (#153) + ## 0.3.23 (2026-04-09) - Add: Gemini CLI support — `graphify gemini install` writes a `GEMINI.md` section and a `BeforeTool` hook in `.gemini/settings.json` that fires before file-read tool calls (#105) diff --git a/graphify/__main__.py b/graphify/__main__.py index c2c27d4ae..604fa5248 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -450,13 +450,12 @@ def _agents_install(project_dir: Path, platform: str) -> None: content = target.read_text(encoding="utf-8") if _AGENTS_MD_MARKER in content: print(f"graphify already configured in AGENTS.md") - return - new_content = content.rstrip() + "\n\n" + _AGENTS_MD_SECTION + else: + target.write_text(content.rstrip() + "\n\n" + _AGENTS_MD_SECTION, encoding="utf-8") + print(f"graphify section written to {target.resolve()}") else: - new_content = _AGENTS_MD_SECTION - - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") + target.write_text(_AGENTS_MD_SECTION, encoding="utf-8") + print(f"graphify section written to {target.resolve()}") if platform == "codex": _install_codex_hook(project_dir or Path(".")) diff --git a/pyproject.toml b/pyproject.toml index a18d0873b..0d6568fcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.23" +version = "0.3.24" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From e440539305e170bdcc98162105c08871a6b100dc Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 22:45:52 +0100 Subject: [PATCH 34/59] Add Trendshift badge to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8f5b8b357..db48e2527 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) [![Downloads](https://img.shields.io/pypi/dm/graphifyy)](https://pypi.org/project/graphifyy/) +[![Trendshift](https://trendshift.io/api/badge/repositories/25296)](https://trendshift.io/repositories/25296) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) **An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. From cf39034744610fdf840f8683a2c1d8e61bc7ab63 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 9 Apr 2026 22:46:47 +0100 Subject: [PATCH 35/59] Remove Trendshift badge (API not available) --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index db48e2527..8f5b8b357 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) [![Downloads](https://img.shields.io/pypi/dm/graphifyy)](https://pypi.org/project/graphifyy/) -[![Trendshift](https://trendshift.io/api/badge/repositories/25296)](https://trendshift.io/repositories/25296) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) **An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. From 1cbcee539f8c7aad94426de2b8bd19ddca3f324d Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 00:56:22 +0100 Subject: [PATCH 36/59] release 0.3.25: Aider + Copilot CLI, directed graphs, frontmatter cache, graphifyignore parent discovery, MCP fixes Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 12 + README.md | 11 +- graphify/__main__.py | 48 +- graphify/analyze.py | 2 + graphify/build.py | 16 +- graphify/cache.py | 20 +- graphify/cluster.py | 5 + graphify/detect.py | 31 +- graphify/serve.py | 7 +- graphify/skill-aider.md | 1137 ++++++++++++++++++++++++++++++++++ graphify/skill-copilot.md | 1219 +++++++++++++++++++++++++++++++++++++ graphify/skill.md | 3 + pyproject.toml | 4 +- tests/test_cache.py | 54 +- tests/test_detect.py | 53 ++ 15 files changed, 2596 insertions(+), 26 deletions(-) create mode 100644 graphify/skill-aider.md create mode 100644 graphify/skill-copilot.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 914eb1fcc..c6b3d36db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.25 (2026-04-09) + +- Fix: `graphify install --platform gemini` now routes to `gemini_install()` instead of erroring — `gemini` was missing from `_PLATFORM_CONFIG` (#171) +- Fix: `graphify install --platform cursor` now routes to `_cursor_install()` the same way (#171) +- Fix: `serve.py` `validate_graph_path` now passes `base=Path(graph_path).resolve().parent` so MCP server works when graph is outside cwd (#170) +- Fix: MCP `call_tool()` handler now wraps dispatch in try/except — exceptions in tool handlers return graceful error strings instead of crashing the stdio loop (#163) +- Fix: `_load_graphifyignore` now walks parent directories up to the `.git` boundary, matching `.gitignore` discovery behavior — subdirectory scans now inherit root ignore patterns (#168) +- Add: Aider platform support — `graphify install --platform aider` copies skill to `~/.aider/graphify/SKILL.md`; `graphify aider install/uninstall` writes AGENTS.md rules (#74) +- Add: GitHub Copilot CLI platform support — `graphify install --platform copilot` copies skill to `~/.copilot/skills/graphify/SKILL.md`; `graphify copilot install/uninstall` for skill management (#134) +- Add: `--directed` flag — `build_from_json()` and `build()` now accept `directed=True` to produce a `DiGraph` preserving edge direction (source→target); `cluster()` converts to undirected internally for Leiden; `graph_diff` edge key handles directed graphs correctly (#125) +- Add: Frontmatter-aware cache for Markdown files — `.md` files hash only the body below YAML frontmatter, so metadata-only changes (reviewed, status, tags) no longer invalidate the cache (#131) + ## 0.3.24 (2026-04-09) - Fix: `graphify codex install` (and opencode) no longer exits early when `AGENTS.md` already has the graphify section — partial installs with a missing `.codex/hooks.json` can now recover on re-run (#153) diff --git a/README.md b/README.md index 8f5b8b357..224fa2606 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Downloads](https://img.shields.io/pypi/dm/graphifyy)](https://pypi.org/project/graphifyy/) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) -**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. +**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, Aider, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 20 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia). @@ -47,7 +47,7 @@ Every relationship is tagged `EXTRACTED` (found directly in source), `INFERRED` ## Install -**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [Cursor](https://cursor.com), [Gemini CLI](https://github.com/google-gemini/gemini-cli), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), or [Trae](https://trae.ai) +**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [Cursor](https://cursor.com), [Gemini CLI](https://github.com/google-gemini/gemini-cli), [GitHub Copilot CLI](https://docs.github.com/en/copilot/how-tos/copilot-cli), [Aider](https://aider.chat), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), or [Trae](https://trae.ai) ```bash pip install graphifyy && graphify install @@ -63,6 +63,8 @@ pip install graphifyy && graphify install | Claude Code (Windows) | `graphify install` (auto-detected) or `graphify install --platform windows` | | Codex | `graphify install --platform codex` | | OpenCode | `graphify install --platform opencode` | +| GitHub Copilot CLI | `graphify install --platform copilot` | +| Aider | `graphify install --platform aider` | | OpenClaw | `graphify install --platform claw` | | Factory Droid | `graphify install --platform droid` | | Trae | `graphify install --platform trae` | @@ -70,7 +72,7 @@ pip install graphifyy && graphify install | Gemini CLI | `graphify install --platform gemini` | | Cursor | `graphify cursor install` | -Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw uses sequential extraction (parallel agent support is still early on that platform). Trae uses the Agent tool for parallel subagent dispatch and does **not** support PreToolUse hooks — AGENTS.md is the always-on mechanism. +Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw and Aider use sequential extraction (parallel agent support is still early on those platforms). Trae uses the Agent tool for parallel subagent dispatch and does **not** support PreToolUse hooks — AGENTS.md is the always-on mechanism. Then open your AI coding assistant and type: @@ -89,6 +91,8 @@ After building a graph, run this once in your project: | Claude Code | `graphify claude install` | | Codex | `graphify codex install` | | OpenCode | `graphify opencode install` | +| GitHub Copilot CLI | `graphify copilot install` | +| Aider | `graphify aider install` | | OpenClaw | `graphify claw install` | | Factory Droid | `graphify droid install` | | Trae | `graphify trae install` | @@ -179,6 +183,7 @@ When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"` /graphify ./raw # run on a specific folder /graphify ./raw --mode deep # more aggressive INFERRED edge extraction /graphify ./raw --update # re-extract only changed files, merge into existing graph +/graphify ./raw --directed # build directed graph (preserves edge direction: source→target) /graphify ./raw --cluster-only # rerun clustering on existing graph, no re-extraction /graphify ./raw --no-viz # skip HTML, just produce report + JSON /graphify ./raw --obsidian # also generate Obsidian vault (opt-in) diff --git a/graphify/__main__.py b/graphify/__main__.py index 604fa5248..56a2ae646 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -62,6 +62,16 @@ def _check_skill_version(skill_dst: Path) -> None: "skill_dst": Path(".config") / "opencode" / "skills" / "graphify" / "SKILL.md", "claude_md": False, }, + "aider": { + "skill_file": "skill-aider.md", + "skill_dst": Path(".aider") / "graphify" / "SKILL.md", + "claude_md": False, + }, + "copilot": { + "skill_file": "skill-copilot.md", + "skill_dst": Path(".copilot") / "skills" / "graphify" / "SKILL.md", + "claude_md": False, + }, "claw": { "skill_file": "skill-claw.md", "skill_dst": Path(".claw") / "skills" / "graphify" / "SKILL.md", @@ -91,9 +101,15 @@ def _check_skill_version(skill_dst: Path) -> None: def install(platform: str = "claude") -> None: + if platform == "gemini": + gemini_install() + return + if platform == "cursor": + _cursor_install() + return if platform not in _PLATFORM_CONFIG: print( - f"error: unknown platform '{platform}'. Choose from: {', '.join(_PLATFORM_CONFIG)}", + f"error: unknown platform '{platform}'. Choose from: {', '.join(_PLATFORM_CONFIG)}, gemini, cursor", file=sys.stderr, ) sys.exit(1) @@ -608,7 +624,7 @@ def main() -> None: print("Usage: graphify ") print() print("Commands:") - print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|claw|droid|trae|trae-cn)") + print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|aider|claw|droid|trae|trae-cn|gemini|cursor)") print(" query \"\" BFS traversal of graph.json for a question") print(" --dfs use depth-first instead of breadth-first") print(" --budget N cap output at N tokens (default 2000)") @@ -633,6 +649,10 @@ def main() -> None: print(" codex uninstall remove graphify section from AGENTS.md") print(" opencode install write graphify section to AGENTS.md + tool.execute.before plugin (OpenCode)") print(" opencode uninstall remove graphify section from AGENTS.md + plugin") + print(" aider install write graphify section to AGENTS.md (Aider)") + print(" aider uninstall remove graphify section from AGENTS.md") + print(" copilot install copy graphify skill to ~/.copilot/skills (GitHub Copilot CLI)") + print(" copilot uninstall remove graphify skill from ~/.copilot/skills") print(" claw install write graphify section to AGENTS.md (OpenClaw)") print(" claw uninstall remove graphify section from AGENTS.md") print(" droid install write graphify section to AGENTS.md (Factory Droid)") @@ -688,7 +708,29 @@ def main() -> None: else: print("Usage: graphify cursor [install|uninstall]", file=sys.stderr) sys.exit(1) - elif cmd in ("codex", "opencode", "claw", "droid", "trae", "trae-cn"): + elif cmd == "copilot": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + install(platform="copilot") + elif subcmd == "uninstall": + skill_dst = Path.home() / _PLATFORM_CONFIG["copilot"]["skill_dst"] + removed = [] + if skill_dst.exists(): + skill_dst.unlink() + removed.append(f"skill removed: {skill_dst}") + version_file = skill_dst.parent / ".graphify_version" + if version_file.exists(): + version_file.unlink() + for d in (skill_dst.parent, skill_dst.parent.parent, skill_dst.parent.parent.parent): + try: + d.rmdir() + except OSError: + break + print("; ".join(removed) if removed else "nothing to remove") + else: + print("Usage: graphify copilot [install|uninstall]", file=sys.stderr) + sys.exit(1) + elif cmd in ("aider", "codex", "opencode", "claw", "droid", "trae", "trae-cn"): subcmd = sys.argv[2] if len(sys.argv) > 2 else "" if subcmd == "install": _agents_install(Path("."), cmd) diff --git a/graphify/analyze.py b/graphify/analyze.py index 50f85acaf..28a8b3e80 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -469,6 +469,8 @@ def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict: ] def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple: + if G.is_directed(): + return (u, v, data.get("relation", "")) return (min(u, v), max(u, v), data.get("relation", "")) old_edge_keys = { diff --git a/graphify/build.py b/graphify/build.py index 1bcb51b17..3c3d80ca6 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -26,13 +26,18 @@ from .validate import validate_extraction -def build_from_json(extraction: dict) -> nx.Graph: +def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: + """Build a NetworkX graph from an extraction dict. + + directed=True produces a DiGraph that preserves edge direction (source→target). + directed=False (default) produces an undirected Graph for backward compatibility. + """ errors = validate_extraction(extraction) # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors. real_errors = [e for e in errors if "does not match any node id" not in e] if real_errors: print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr) - G = nx.Graph() + G: nx.Graph = nx.DiGraph() if directed else nx.Graph() for node in extraction.get("nodes", []): G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) node_set = set(G.nodes()) @@ -52,9 +57,12 @@ def build_from_json(extraction: dict) -> nx.Graph: return G -def build(extractions: list[dict]) -> nx.Graph: +def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph: """Merge multiple extraction results into one graph. + directed=True produces a DiGraph that preserves edge direction (source→target). + directed=False (default) produces an undirected Graph for backward compatibility. + Extractions are merged in order. For nodes with the same ID, the last extraction's attributes win (NetworkX add_node overwrites). Pass AST results before semantic results so semantic labels take precedence, or @@ -67,4 +75,4 @@ def build(extractions: list[dict]) -> nx.Graph: combined["hyperedges"].extend(ext.get("hyperedges", [])) combined["input_tokens"] += ext.get("input_tokens", 0) combined["output_tokens"] += ext.get("output_tokens", 0) - return build_from_json(combined) + return build_from_json(combined, directed=directed) diff --git a/graphify/cache.py b/graphify/cache.py index f198e4161..7f73db069 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -7,11 +7,27 @@ from pathlib import Path +def _body_content(content: bytes) -> bytes: + """Strip YAML frontmatter from Markdown content, returning only the body.""" + text = content.decode(errors="replace") + if text.startswith("---"): + end = text.find("\n---", 3) + if end != -1: + return text[end + 4:].encode() + return content + + def file_hash(path: Path) -> str: - """SHA256 of file contents + resolved path. Prevents cache collisions on identical content.""" + """SHA256 of file contents + resolved path. Prevents cache collisions on identical content. + + For Markdown files (.md), only the body below the YAML frontmatter is hashed, + so metadata-only changes (e.g. reviewed, status, tags) do not invalidate the cache. + """ p = Path(path) + raw = p.read_bytes() + content = _body_content(raw) if p.suffix.lower() == ".md" else raw h = hashlib.sha256() - h.update(p.read_bytes()) + h.update(content) h.update(b"\x00") h.update(str(p.resolve()).encode()) return h.hexdigest() diff --git a/graphify/cluster.py b/graphify/cluster.py index 363d555c8..9227c0d13 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -62,9 +62,14 @@ def cluster(G: nx.Graph) -> dict[int, list[str]]: Community IDs are stable across runs: 0 = largest community after splitting. Oversized communities (> 25% of graph nodes, min 10) are split by running a second Leiden pass on the subgraph. + + Accepts directed or undirected graphs. DiGraphs are converted to undirected + internally since Louvain/Leiden require undirected input. """ if G.number_of_nodes() == 0: return {} + if G.is_directed(): + G = G.to_undirected() if G.number_of_edges() == 0: return {i: [n] for i, n in enumerate(sorted(G.nodes))} diff --git a/graphify/detect.py b/graphify/detect.py index 9a5f16e08..bb630527a 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -250,21 +250,34 @@ def _is_noise_dir(part: str) -> bool: def _load_graphifyignore(root: Path) -> list[str]: - """Read .graphifyignore from root and return a list of patterns. + """Read .graphifyignore from root **and ancestor directories**, returning patterns. + + Walks upward from *root* towards the filesystem root, collecting patterns + from every ``.graphifyignore`` encountered (like ``.gitignore`` discovery). + The search stops at the filesystem root or at a ``.git`` directory boundary + so it doesn't leak outside the repository. Lines starting with # are comments. Blank lines are ignored. Patterns follow gitignore semantics: glob matched against the path relative to root. A leading slash anchors to root. A trailing slash matches directories only (we match both dir and file for simplicity). """ - ignore_file = root / ".graphifyignore" - if not ignore_file.exists(): - return [] - patterns = [] - for line in ignore_file.read_text(errors="ignore").splitlines(): - line = line.strip() - if line and not line.startswith("#"): - patterns.append(line) + patterns: list[str] = [] + current = root.resolve() + while True: + ignore_file = current / ".graphifyignore" + if ignore_file.exists(): + for line in ignore_file.read_text(errors="ignore").splitlines(): + line = line.strip() + if line and not line.startswith("#"): + patterns.append(line) + # Stop climbing once we've processed the git repo root + if (current / ".git").exists(): + break + parent = current.parent + if parent == current: + break # filesystem root + current = parent return patterns diff --git a/graphify/serve.py b/graphify/serve.py index 79bd37fc7..9f42c2c05 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -10,7 +10,7 @@ def _load_graph(graph_path: str) -> nx.Graph: try: - safe = validate_graph_path(graph_path) + safe = validate_graph_path(graph_path, base=Path(graph_path).resolve().parent) data = json.loads(safe.read_text()) try: return json_graph.node_link_graph(data, edges="links") @@ -309,7 +309,10 @@ async def call_tool(name: str, arguments: dict) -> list[types.TextContent]: handler = _handlers.get(name) if not handler: return [types.TextContent(type="text", text=f"Unknown tool: {name}")] - return [types.TextContent(type="text", text=handler(arguments))] + try: + return [types.TextContent(type="text", text=handler(arguments))] + except Exception as exc: + return [types.TextContent(type="text", text=f"Error executing {name}: {exc}")] import asyncio diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md new file mode 100644 index 000000000..cc3aa446d --- /dev/null +++ b/graphify/skill-aider.md @@ -0,0 +1,1137 @@ +--- +name: graphify +description: any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that Claude alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes → one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles pipx, venv, system installs) +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac +else + PYTHON="python3" +fi +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +mkdir -p graphify-out +# Write interpreter path for all subsequent steps +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat .graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat .graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > .graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files +``` + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 3 - no need to ask anything. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (Claude, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +> **Aider platform:** Multi-agent support is still early on Aider. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. + +Print: `"Semantic extraction: N files (sequential — Aider)"` + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Sequential extraction (Aider)** + +Process each file one at a time. For each file: + +1. Read the file contents +2. Extract nodes, edges, and hyperedges applying the same rules: + - EXTRACTED: relationship explicit in source (import, call, citation) + - INFERRED: reasonable inference (shared structure, implied dependency) + - AMBIGUOUS: uncertain — flag it, do not omit + - Code files: semantic edges AST cannot find. Do not re-extract imports. + - Doc/paper files: named concepts, entities, citations, and rationale nodes (WHY decisions were made → `rationale_for` edges) + - Image files: use vision — understand what the image IS, not just OCR + - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges + - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. + - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. + - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 +3. Accumulate results across all files + +Schema for each file's output: +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +After processing all files, write the accumulated result to `.graphify_semantic_new.json`. + +**Step B3 - Cache and merge** + +For the accumulated result: + +If more than half the chunks failed, stop and tell the user. + +Save new results to cache: +```bash +$(cat .graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `.graphify_semantic.json`: +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat .graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('.graphify_ast.json').read_text()) +sem = json.loads(Path('.graphify_semantic.json').read_text()) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') + +to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) +print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') +print() +print('Open graphify-out/obsidian/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +if G.number_of_nodes() > 5000: + print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat .graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('.graphify_incremental.json').write_text(json.dumps(result)) +if new_total == 0: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + +Then: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load existing graph +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +# Load new extraction +new_extraction = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +# Merge: new nodes/edges into existing graph +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None +new_extract = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` +Clean up after: `rm -f .graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat .graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + d = G.edges[u, v] + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat .graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat .graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + edge = G.edges[nid, path[i+1]] + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat .graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat .graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + edge = G.edges[nid, neighbor] + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat .graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat .graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md new file mode 100644 index 000000000..72d0f2da4 --- /dev/null +++ b/graphify/skill-copilot.md @@ -0,0 +1,1219 @@ +--- +name: graphify +description: any input (code, docs, papers, images) → knowledge graph → clustered communities → HTML + JSON + audit report +trigger: /graphify +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that Claude alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes → one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles pipx, venv, system installs) +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac +else + PYTHON="python3" +fi +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +# Write interpreter path for all subsequent steps (persists across invocations) +mkdir -p graphify-out +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > graphify-out/.graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files +``` + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 3 - no need to ask anything. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (Claude, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** + +Before dispatching subagents, print a timing estimate: +- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` +- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) +- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) +- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Dispatch ALL subagents in a single message** + +Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. + +Concrete example for 3 chunks: +``` +[Agent tool call 1: files 1-15] +[Agent tool call 2: files 16-30] +[Agent tool call 3: files 31-45] +``` +All three in one message. Not three separate messages. + +Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE): + +``` +You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. +Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble. + +Files (chunk CHUNK_NUM of TOTAL_CHUNKS): +FILE_LIST + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2") +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain - flag for review, do not omit + +Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). + Do not re-extract imports - AST already has those. +Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Image files: use vision to understand what the image IS - do not just OCR. + UI screenshot: layout patterns, design decisions, key elements, purpose. + Chart: metric, trend/insight, data source. + Tweet/post: claim as node, author, concepts mentioned. + Diagram: components and connections. + Research figure: what it demonstrates, method, result. + Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. + +DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps, + shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. + +Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples: +- Two functions that both validate user input but never call each other +- A class in code and a concept in a paper that describe the same algorithm +- Two error types that handle the same failure mode differently +Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things. + +Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples: +- All classes that implement a common protocol or interface +- All functions in an authentication flow (even if they don't all call each other) +- All concepts from a paper section that form one coherent idea +Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk. + +If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, + contributor onto every node from that file. + +confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: +- EXTRACTED edges: confidence_score = 1.0 always +- INFERRED edges: reason about each edge individually. + Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. + Reasonable inference with some uncertainty: 0.6-0.7. + Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. +- AMBIGUOUS edges: 0.1-0.3 + +Output exactly this JSON (no other text): +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +``` + +**Step B3 - Collect, cache, and merge** + +Wait for all subagents. For each result: +- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort + +If more than half the chunks failed, stop and tell the user. + +Save new results to cache: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `graphify-out/.graphify_semantic.json`: +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +- If `--obsidian-dir ` was also given, use that path as the vault directory. Otherwise default to `graphify-out/obsidian`. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +obsidian_dir = 'OBSIDIAN_DIR' # replace with --obsidian-dir value, or 'graphify-out/obsidian' if not given + +n = to_obsidian(G, communities, obsidian_dir, community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in {obsidian_dir}/') + +to_canvas(G, communities, f'{obsidian_dir}/graph.canvas', community_labels=labels or None) +print(f'Canvas: {obsidian_dir}/graph.canvas - open in Obsidian for structured community layout') +print() +print(f'Open {obsidian_dir}/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +if G.number_of_nodes() > 5000: + print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## Interpreter guard for subcommands + +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: + +```bash +if [ ! -f graphify-out/.graphify_python ]; then + GRAPHIFY_BIN=$(which graphify 2>/dev/null) + if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac + else + PYTHON="python3" + fi + mkdir -p graphify-out + "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +fi +``` + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) +if new_total == 0: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + +Then: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load existing graph +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +# Load new extraction +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +# Prune nodes from deleted files +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text()) +deleted = set(incremental.get('deleted_files', [])) +if deleted: + to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted] + G_existing.remove_nodes_from(to_remove) + print(f'Pruned {len(to_remove)} ghost nodes from {len(deleted)} deleted file(s)') + +# Merge: new nodes/edges into existing graph +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` +Clean up after: `rm -f graphify-out/.graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + d = G.edges[u, v] + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + edge = G.edges[nid, path[i+1]] + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat graphify-out/.graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + edge = G.edges[nid, neighbor] + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat graphify-out/.graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/graphify/skill.md b/graphify/skill.md index 72d0f2da4..591ed4be5 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -15,6 +15,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify # full pipeline on specific path /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -340,6 +341,8 @@ print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(s ### Step 4 - Build graph, cluster, analyze, generate outputs +**Before starting:** note whether `--directed` was given. If so, pass `directed=True` to `build_from_json()` in the code block below. This builds a `DiGraph` that preserves edge direction (source→target) instead of the default undirected `Graph`. + ```bash mkdir -p graphify-out $(cat graphify-out/.graphify_python) -c " diff --git a/pyproject.toml b/pyproject.toml index 0d6568fcb..6dafb5670 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.24" +version = "0.3.25" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } @@ -57,4 +57,4 @@ where = ["."] include = ["graphify*"] [tool.setuptools.package-data] -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md"] +graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-aider.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md"] diff --git a/tests/test_cache.py b/tests/test_cache.py index 3375ed722..f3f584123 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,7 +1,7 @@ """Tests for graphify/cache.py.""" import pytest from pathlib import Path -from graphify.cache import file_hash, cache_dir, load_cached, save_cached, cached_files, clear_cache +from graphify.cache import file_hash, cache_dir, load_cached, save_cached, cached_files, clear_cache, _body_content @pytest.fixture @@ -72,3 +72,55 @@ def test_clear_cache(tmp_file, cache_root): assert len(list((cache_root / "graphify-out" / "cache").glob("*.json"))) > 0 clear_cache(cache_root) assert len(list((cache_root / "graphify-out" / "cache").glob("*.json"))) == 0 + + +def test_md_frontmatter_only_change_same_hash(tmp_path): + """Changing only frontmatter fields in a .md file does not change the hash.""" + f = tmp_path / "doc.md" + f.write_text("---\nreviewed: 2026-01-01\n---\n\n# Title\n\nBody text.") + h1 = file_hash(f) + f.write_text("---\nreviewed: 2026-04-09\n---\n\n# Title\n\nBody text.") + h2 = file_hash(f) + assert h1 == h2 + + +def test_md_body_change_different_hash(tmp_path): + """Changing the body of a .md file produces a different hash.""" + f = tmp_path / "doc.md" + f.write_text("---\nreviewed: 2026-01-01\n---\n\n# Title\n\nOriginal body.") + h1 = file_hash(f) + f.write_text("---\nreviewed: 2026-01-01\n---\n\n# Title\n\nChanged body.") + h2 = file_hash(f) + assert h1 != h2 + + +def test_md_no_frontmatter_hashed_normally(tmp_path): + """A .md file with no frontmatter is hashed by its full content.""" + f = tmp_path / "doc.md" + f.write_text("# Just a heading\n\nNo frontmatter here.") + h1 = file_hash(f) + f.write_text("# Just a heading\n\nDifferent content.") + h2 = file_hash(f) + assert h1 != h2 + + +def test_non_md_file_hashed_fully(tmp_path): + """Non-.md files are still hashed by their full content.""" + f = tmp_path / "script.py" + f.write_text("# comment\nx = 1") + h1 = file_hash(f) + f.write_text("# changed comment\nx = 1") + h2 = file_hash(f) + assert h1 != h2 + + +def test_body_content_strips_frontmatter(): + """_body_content correctly strips YAML frontmatter.""" + content = b"---\ntitle: Test\n---\n\nActual body." + assert _body_content(content) == b"\n\nActual body." + + +def test_body_content_no_frontmatter(): + """_body_content returns content unchanged when no frontmatter present.""" + content = b"No frontmatter here." + assert _body_content(content) == content diff --git a/tests/test_detect.py b/tests/test_detect.py index 520d02d3d..f743a6dc6 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -138,6 +138,59 @@ def test_detect_follows_symlinked_file(tmp_path): assert any("link.py" in f for f in code) +def test_graphifyignore_discovered_from_parent(tmp_path): + """A .graphifyignore in a parent directory applies to subdirectory scans.""" + (tmp_path / ".graphifyignore").write_text("vendor/\n") + sub = tmp_path / "packages" / "mylib" + sub.mkdir(parents=True) + (sub / "main.py").write_text("x = 1") + vendor = sub / "vendor" + vendor.mkdir() + (vendor / "dep.py").write_text("y = 2") + + result = detect(sub) + code_files = result["files"]["code"] + assert any("main.py" in f for f in code_files) + assert not any("vendor" in f for f in code_files) + assert result["graphifyignore_patterns"] >= 1 + + +def test_graphifyignore_stops_at_git_boundary(tmp_path): + """Upward search stops at the git repo root (.git directory).""" + (tmp_path / ".graphifyignore").write_text("main.py\n") + repo = tmp_path / "repo" + repo.mkdir() + (repo / ".git").mkdir() + sub = repo / "sub" + sub.mkdir() + (sub / "main.py").write_text("x = 1") + + result = detect(sub) + code_files = result["files"]["code"] + assert any("main.py" in f for f in code_files) + assert result["graphifyignore_patterns"] == 0 + + +def test_graphifyignore_at_git_root_is_included(tmp_path): + """A .graphifyignore at the git repo root is included when scanning a subdir.""" + repo = tmp_path / "repo" + repo.mkdir() + (repo / ".git").mkdir() + (repo / ".graphifyignore").write_text("vendor/\n") + sub = repo / "packages" / "mylib" + sub.mkdir(parents=True) + (sub / "main.py").write_text("x = 1") + vendor = sub / "vendor" + vendor.mkdir() + (vendor / "dep.py").write_text("y = 2") + + result = detect(sub) + code_files = result["files"]["code"] + assert any("main.py" in f for f in code_files) + assert not any("vendor" in f for f in code_files) + assert result["graphifyignore_patterns"] == 1 + + def test_detect_handles_circular_symlinks(tmp_path): sub = tmp_path / "a" sub.mkdir() From 863100cb060bf0bf7cbbd3e656f121a20511f0eb Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 01:08:15 +0100 Subject: [PATCH 37/59] fix MCP server path validation security issue (0.3.26) --- CHANGELOG.md | 4 ++++ graphify/serve.py | 9 +++++++-- pyproject.toml | 2 +- tests/test_serve.py | 5 +---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6b3d36db..afa67ce27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.26 (2026-04-10) + +- Fix: MCP server no longer uses a circular path validation when loading a graph outside cwd — now validates the path exists and ends in `.json` instead of checking containment within its own parent directory (security fix) + ## 0.3.25 (2026-04-09) - Fix: `graphify install --platform gemini` now routes to `gemini_install()` instead of erroring — `gemini` was missing from `_PLATFORM_CONFIG` (#171) diff --git a/graphify/serve.py b/graphify/serve.py index 9f42c2c05..81c9353ab 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -5,12 +5,17 @@ from pathlib import Path import networkx as nx from networkx.readwrite import json_graph -from graphify.security import validate_graph_path, sanitize_label +from graphify.security import sanitize_label def _load_graph(graph_path: str) -> nx.Graph: try: - safe = validate_graph_path(graph_path, base=Path(graph_path).resolve().parent) + resolved = Path(graph_path).resolve() + if resolved.suffix != ".json": + raise ValueError(f"Graph path must be a .json file, got: {graph_path!r}") + if not resolved.exists(): + raise FileNotFoundError(f"Graph file not found: {resolved}") + safe = resolved data = json.loads(safe.read_text()) try: return json_graph.node_link_graph(data, edges="links") diff --git a/pyproject.toml b/pyproject.toml index 6dafb5670..71e90a152 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.25" +version = "0.3.26" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } diff --git a/tests/test_serve.py b/tests/test_serve.py index ed5217653..6457ac501 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -138,14 +138,11 @@ def test_subgraph_to_text_edge_included(): # --- _load_graph --- def test_load_graph_roundtrip(tmp_path): - from unittest.mock import patch G = _make_graph() data = json_graph.node_link_data(G, edges="links") p = tmp_path / "graph.json" p.write_text(json.dumps(data)) - # validate_graph_path is tested separately; here we test parse correctness - with patch("graphify.serve.validate_graph_path", return_value=p): - G2 = _load_graph(str(p)) + G2 = _load_graph(str(p)) assert G2.number_of_nodes() == G.number_of_nodes() assert G2.number_of_edges() == G.number_of_edges() From a0a196e1afa2bd203361c744f2aff7c1fe6197ef Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 01:09:35 +0100 Subject: [PATCH 38/59] update README to list all platforms including Aider, Copilot CLI, Cursor, Gemini in CLI section --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 224fa2606..0dba49089 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,9 @@ After building a graph, run this once in your project: **Gemini CLI** writes a `GEMINI.md` section and installs a `BeforeTool` hook in `.gemini/settings.json` that fires before file-read tool calls — same always-on mechanism as Claude Code. -**OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support tool hooks, so AGENTS.md is the always-on mechanism. +**Aider and OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support tool hooks, so AGENTS.md is the always-on mechanism. + +**GitHub Copilot CLI** copies the skill to `~/.copilot/skills/graphify/SKILL.md`. Run `graphify copilot install` to set it up. Uninstall with the matching uninstall command (e.g. `graphify claude uninstall`). @@ -218,6 +220,14 @@ graphify claude install # CLAUDE.md + PreToolUse hook (Claude Code) graphify claude uninstall graphify codex install # AGENTS.md (Codex) graphify opencode install # AGENTS.md + tool.execute.before plugin (OpenCode) +graphify cursor install # .cursor/rules/graphify.mdc (Cursor) +graphify cursor uninstall +graphify gemini install # GEMINI.md + BeforeTool hook (Gemini CLI) +graphify gemini uninstall +graphify copilot install # skill file (GitHub Copilot CLI) +graphify copilot uninstall +graphify aider install # AGENTS.md (Aider) +graphify aider uninstall graphify claw install # AGENTS.md (OpenClaw) graphify droid install # AGENTS.md (Factory Droid) graphify trae install # AGENTS.md (Trae) From 55964bc0b1c7f06a047fe2d7b82f542913e3cb0c Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 01:28:19 +0100 Subject: [PATCH 39/59] fix gemini install missing skill file copy (0.3.27) --- CHANGELOG.md | 4 ++++ graphify/__main__.py | 26 ++++++++++++++++++++++++-- pyproject.toml | 2 +- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afa67ce27..166955e93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.27 (2026-04-10) + +- Fix: graphify install --platform gemini now also copies the skill file to ~/.gemini/skills/graphify/SKILL.md so the /graphify trigger works in Gemini CLI (#174) + ## 0.3.26 (2026-04-10) - Fix: MCP server no longer uses a circular path validation when loading a graph outside cwd — now validates the path exists and ends in `.json` instead of checking containment within its own parent directory (security fix) diff --git a/graphify/__main__.py b/graphify/__main__.py index 56a2ae646..6a86661db 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -205,7 +205,15 @@ def install(platform: str = "claude") -> None: def gemini_install(project_dir: Path | None = None) -> None: - """Write the graphify section to GEMINI.md and install BeforeTool hook.""" + """Copy skill file to ~/.gemini/skills/graphify/, write GEMINI.md section, and install BeforeTool hook.""" + # Copy skill file to ~/.gemini/skills/graphify/SKILL.md + skill_src = Path(__file__).parent / "skill.md" + skill_dst = Path.home() / ".gemini" / "skills" / "graphify" / "SKILL.md" + skill_dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(skill_src, skill_dst) + (skill_dst.parent / ".graphify_version").write_text(__version__, encoding="utf-8") + print(f" skill installed -> {skill_dst}") + target = (project_dir or Path(".")) / "GEMINI.md" if target.exists(): @@ -259,7 +267,21 @@ def _uninstall_gemini_hook(project_dir: Path) -> None: def gemini_uninstall(project_dir: Path | None = None) -> None: - """Remove the graphify section from GEMINI.md and uninstall hook.""" + """Remove the graphify section from GEMINI.md, uninstall hook, and remove skill file.""" + # Remove skill file + skill_dst = Path.home() / ".gemini" / "skills" / "graphify" / "SKILL.md" + if skill_dst.exists(): + skill_dst.unlink() + print(f" skill removed -> {skill_dst}") + version_file = skill_dst.parent / ".graphify_version" + if version_file.exists(): + version_file.unlink() + for d in (skill_dst.parent, skill_dst.parent.parent): + try: + d.rmdir() + except OSError: + break + target = (project_dir or Path(".")) / "GEMINI.md" if not target.exists(): print("No GEMINI.md found in current directory - nothing to do") diff --git a/pyproject.toml b/pyproject.toml index 71e90a152..eac661e87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.26" +version = "0.3.27" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From af3a3d215771f8d7d159df0605aef040b91ad12f Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 01:29:05 +0100 Subject: [PATCH 40/59] update README Gemini CLI description to mention skill file copy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0dba49089..c2112edfd 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ After building a graph, run this once in your project: **Cursor** writes `.cursor/rules/graphify.mdc` with `alwaysApply: true` — Cursor includes it in every conversation automatically, no hook needed. -**Gemini CLI** writes a `GEMINI.md` section and installs a `BeforeTool` hook in `.gemini/settings.json` that fires before file-read tool calls — same always-on mechanism as Claude Code. +**Gemini CLI** copies the skill to `~/.gemini/skills/graphify/SKILL.md`, writes a `GEMINI.md` section, and installs a `BeforeTool` hook in `.gemini/settings.json` that fires before file-read tool calls — same always-on mechanism as Claude Code. **Aider and OpenClaw, Factory Droid, Trae** write the same rules to `AGENTS.md` in your project root. These platforms don't support tool hooks, so AGENTS.md is the always-on mechanism. From f7ee752eb1fa34608364c5001b573e8005c4cdbb Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 09:49:50 +0100 Subject: [PATCH 41/59] switch downloads badge to pepy.tech to avoid shields.io rate limit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c2112edfd..4570fb06d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) -[![Downloads](https://img.shields.io/pypi/dm/graphifyy)](https://pypi.org/project/graphifyy/) +[![Downloads](https://static.pepy.tech/badge/graphifyy/month)](https://pepy.tech/project/graphifyy) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) **An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, Aider, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. From 210243fa58577319a11f4944b3b7aea0e1d6038e Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 10:07:11 +0100 Subject: [PATCH 42/59] fix hook reinstall, CRLF labels, skill-windows missing commands (0.3.28) --- CHANGELOG.md | 6 ++++++ graphify/__main__.py | 21 ++++++--------------- graphify/export.py | 4 ++-- graphify/extract.py | 2 +- graphify/skill-windows.md | 3 +++ pyproject.toml | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 166955e93..c7aeddc12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.28 (2026-04-10) + +- Fix: hook installers (Claude Code, Codex, Gemini CLI) now always remove and reinstall the hook on re-run — users upgrading from old versions no longer get stuck with a broken hook format (#182) +- Fix: rationale node labels no longer contain bare `\r` characters on Windows/WSL CRLF files — breaks Obsidian export was silently producing invalid filenames (#176) +- Fix: `skill-windows.md` now includes `--wiki`, `--obsidian-dir`, and `--directed` which were missing vs the main skill (#177) + ## 0.3.27 (2026-04-10) - Fix: graphify install --platform gemini now also copies the skill file to ~/.gemini/skills/graphify/SKILL.md so the /graphify trigger works in Gemini CLI (#174) diff --git a/graphify/__main__.py b/graphify/__main__.py index 6a86661db..a8a56c07b 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -241,10 +241,8 @@ def _install_gemini_hook(project_dir: Path) -> None: except json.JSONDecodeError: settings = {} before_tool = settings.setdefault("hooks", {}).setdefault("BeforeTool", []) - if any("graphify" in str(h) for h in before_tool): - print(" .gemini/settings.json -> hook already registered (no change)") - return - before_tool.append(_GEMINI_HOOK) + settings["hooks"]["BeforeTool"] = [h for h in before_tool if "graphify" not in str(h)] + settings["hooks"]["BeforeTool"].append(_GEMINI_HOOK) settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") print(" .gemini/settings.json -> BeforeTool hook registered") @@ -455,11 +453,8 @@ def _install_codex_hook(project_dir: Path) -> None: existing = {} pre_tool = existing.setdefault("hooks", {}).setdefault("PreToolUse", []) - if any("graphify" in str(h) for h in pre_tool): - print(f" .codex/hooks.json -> hook already registered (no change)") - return - - pre_tool.extend(_CODEX_HOOK["hooks"]["PreToolUse"]) + existing["hooks"]["PreToolUse"] = [h for h in pre_tool if "graphify" not in str(h)] + existing["hooks"]["PreToolUse"].extend(_CODEX_HOOK["hooks"]["PreToolUse"]) hooks_path.write_text(json.dumps(existing, indent=2), encoding="utf-8") print(f" .codex/hooks.json -> PreToolUse hook registered") @@ -578,12 +573,8 @@ def _install_claude_hook(project_dir: Path) -> None: hooks = settings.setdefault("hooks", {}) pre_tool = hooks.setdefault("PreToolUse", []) - # Check if already installed - if any(h.get("matcher") == "Glob|Grep" and "graphify" in str(h) for h in pre_tool): - print(f" .claude/settings.json -> hook already registered (no change)") - return - - pre_tool.append(_SETTINGS_HOOK) + hooks["PreToolUse"] = [h for h in pre_tool if not (h.get("matcher") == "Glob|Grep" and "graphify" in str(h))] + hooks["PreToolUse"].append(_SETTINGS_HOOK) settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") print(f" .claude/settings.json -> PreToolUse hook registered") diff --git a/graphify/export.py b/graphify/export.py index 498e514ae..e58df1764 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -463,7 +463,7 @@ def to_obsidian( # Map node_id → safe filename so wikilinks stay consistent. # Deduplicate: if two nodes produce the same filename, append a numeric suffix. def safe_name(label: str) -> str: - return re.sub(r'[\\/*?:"<>|#^[\]]', "", label).strip() or "unnamed" + return re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip() or "unnamed" node_filename: dict[str, str] = {} seen_names: dict[str, int] = {} @@ -699,7 +699,7 @@ def to_canvas( CANVAS_COLORS = ["1", "2", "3", "4", "5", "6"] # red, orange, yellow, green, cyan, purple def safe_name(label: str) -> str: - return re.sub(r'[\\/*?:"<>|#^[\]]', "", label).strip() or "unnamed" + return re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip() or "unnamed" # Build node_filenames if not provided (same dedup logic as to_obsidian) if node_filenames is None: diff --git a/graphify/extract.py b/graphify/extract.py index c767e07f4..65e62c646 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1021,7 +1021,7 @@ def _get_docstring(body_node) -> tuple[str, int] | None: return None def _add_rationale(text: str, line: int, parent_nid: str) -> None: - label = text[:80].replace("\n", " ").strip() + label = text[:80].replace("\r\n", " ").replace("\r", " ").replace("\n", " ").strip() rid = _make_id(stem, "rationale", str(line)) if rid not in seen_ids: seen_ids.add(rid) diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 1f174cb80..26984312c 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -15,6 +15,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify # full pipeline on specific path /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files +/graphify --directed # build directed graph (preserves edge direction: source→target) /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -22,6 +23,8 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --graphml # export graph.graphml (Gephi, yEd) /graphify --neo4j # generate graphify-out/cypher.txt for Neo4j /graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) +/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) /graphify --mcp # start MCP stdio server for agent access /graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) /graphify add # fetch URL, save to ./raw, update graph diff --git a/pyproject.toml b/pyproject.toml index eac661e87..2fc9ed258 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.27" +version = "0.3.28" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From 79acb7e46a11365752298b4b89b21021dcfcc7ac Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 15:40:18 +0100 Subject: [PATCH 43/59] Add video/audio corpus support with yt-dlp download and Whisper transcription Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 8 ++ README.md | 2 +- graphify/detect.py | 8 +- graphify/ingest.py | 6 ++ graphify/skill-aider.md | 49 ++++++++- graphify/skill-claw.md | 49 ++++++++- graphify/skill-codex.md | 48 ++++++++- graphify/skill-copilot.md | 49 ++++++++- graphify/skill-droid.md | 49 ++++++++- graphify/skill-opencode.md | 49 ++++++++- graphify/skill-trae.md | 49 ++++++++- graphify/skill-windows.md | 48 ++++++++- graphify/skill.md | 53 +++++++++- graphify/transcribe.py | 202 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 5 +- tests/test_detect.py | 37 +++++++ tests/test_transcribe.py | 168 ++++++++++++++++++++++++++++++ 17 files changed, 865 insertions(+), 14 deletions(-) create mode 100644 graphify/transcribe.py create mode 100644 tests/test_transcribe.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c7aeddc12..c1e253c1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.3.29 (2026-04-10) + +- Add: video and audio corpus support — drop `.mp4`, `.mp3`, `.wav`, `.mov`, `.webm`, `.m4a`, `.ogg`, `.mkv`, `.avi`, `.m4v` files into any corpus and graphify transcribes them with faster-whisper before extraction +- Add: YouTube and URL video download — pass a YouTube link (or any video URL) to `/graphify add ` and yt-dlp downloads audio-only, which is then transcribed and added to the corpus automatically +- Add: domain-aware Whisper prompts — god nodes from non-video files are used to build a one-sentence domain hint for Whisper via a cheap Haiku call, improving transcript accuracy on technical content +- Add: `graphify-out/transcripts/` cache — transcripts are cached by filename so re-runs skip already-transcribed files; URLs cached by hash +- Requires: `pip install 'graphifyy[video]'` for faster-whisper + yt-dlp + ## 0.3.28 (2026-04-10) - Fix: hook installers (Claude Code, Codex, Gemini CLI) now always remove and reinstall the hook on re-run — users upgrading from old versions no longer get stuck with a broken hook format (#182) diff --git a/README.md b/README.md index 4570fb06d..0a0e2de45 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ **An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, Aider, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. -Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, even images in other languages - graphify uses Claude vision to extract concepts and relationships from all of it and connects them into one graph. 20 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia). +Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, images in other languages, or video and audio files - graphify extracts concepts and relationships from all of it and connects them into one graph. Videos are transcribed with Whisper using a domain-aware prompt derived from your corpus. 20 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia). > Andrej Karpathy keeps a `/raw` folder where he drops papers, tweets, screenshots, and notes. graphify is the answer to that problem - 71.5x fewer tokens per query vs reading the raw files, persistent across sessions, honest about what it found vs guessed. diff --git a/graphify/detect.py b/graphify/detect.py index bb630527a..e9dc701f0 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -13,6 +13,7 @@ class FileType(str, Enum): DOCUMENT = "document" PAPER = "paper" IMAGE = "image" + VIDEO = "video" _MANIFEST_PATH = "graphify-out/manifest.json" @@ -22,6 +23,7 @@ class FileType(str, Enum): PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} OFFICE_EXTENSIONS = {'.docx', '.xlsx'} +VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'} CORPUS_WARN_THRESHOLD = 50_000 # words - below this, warn "you may not need a graph" CORPUS_UPPER_THRESHOLD = 500_000 # words - above this, warn about token cost @@ -95,6 +97,8 @@ def classify_file(path: Path) -> FileType | None: return FileType.DOCUMENT if ext in OFFICE_EXTENSIONS: return FileType.DOCUMENT + if ext in VIDEO_EXTENSIONS: + return FileType.VIDEO return None @@ -318,6 +322,7 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: FileType.DOCUMENT: [], FileType.PAPER: [], FileType.IMAGE: [], + FileType.VIDEO: [], } total_words = 0 @@ -388,7 +393,8 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]") continue files[ftype].append(str(p)) - total_words += count_words(p) + if ftype != FileType.VIDEO: + total_words += count_words(p) total_files = sum(len(v) for v in files.values()) needs_graph = total_words >= CORPUS_WARN_THRESHOLD diff --git a/graphify/ingest.py b/graphify/ingest.py index 0d4767b6d..4e74c71cc 100644 --- a/graphify/ingest.py +++ b/graphify/ingest.py @@ -207,6 +207,12 @@ def ingest(url: str, target_dir: Path, author: str | None = None, contributor: s print(f"Downloaded image: {out.name}") return out + if url_type == "youtube": + from graphify.transcribe import download_audio + out = download_audio(url, target_dir) + print(f"Downloaded audio: {out.name}") + return out + if url_type == "tweet": content, filename = _fetch_tweet(url, author, contributor) elif url_type == "arxiv": diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index cc3aa446d..d61244831 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -98,13 +98,60 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +# Try to load god nodes from a previous partial run or pass [] if not yet available +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. ### Step 3 - Extract entities and relationships diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 73eff7f33..539abbaaa 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -98,13 +98,60 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +# Try to load god nodes from a previous partial run or pass [] if not yet available +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. ### Step 3 - Extract entities and relationships diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index d14a90bf3..7f1d76b71 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -97,13 +97,59 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. ### Step 3 - Extract entities and relationships diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 72d0f2da4..ef3cefef3 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -100,13 +100,60 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +# Try to load god nodes from a previous partial run or pass [] if not yet available +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. ### Step 3 - Extract entities and relationships diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index b36399db2..5395a5ab7 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -98,13 +98,60 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +# Try to load god nodes from a previous partial run or pass [] if not yet available +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. ### Step 3 - Extract entities and relationships diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index ad4318403..6f352ead0 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -98,13 +98,60 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +# Try to load god nodes from a previous partial run or pass [] if not yet available +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. ### Step 3 - Extract entities and relationships diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 2711dcd18..ec2c56686 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -97,13 +97,60 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +# Try to load god nodes from a previous partial run or pass [] if not yet available +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. ### Step 3 - Extract entities and relationships diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 26984312c..41daccd82 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -90,13 +90,59 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command (PowerShell):** + +```powershell +& (Get-Content graphify-out\.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" | Out-File -FilePath graphify-out\.graphify_transcripts.json -Encoding utf8 +``` + +After transcription: +- Read the transcript paths from `graphify-out\.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `$env:GRAPHIFY_WHISPER_MODEL = ""` before running the command above. ### Step 3 - Extract entities and relationships diff --git a/graphify/skill.md b/graphify/skill.md index 591ed4be5..1fb84be74 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -16,6 +16,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) +/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy /graphify --cluster-only # rerun clustering on existing graph /graphify --no-viz # skip visualization, just report + JSON /graphify --html # (HTML is generated by default - this flag is a no-op) @@ -101,13 +102,60 @@ Corpus: X files · ~Y words docs: N files (.md .txt ...) papers: N files (.pdf ...) images: N files + video: N files (.mp4 .mp3 ...) ``` +Omit any category with 0 files from the summary. + Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." - If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 3 - no need to ask anything. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. + +**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. + +**Transcription command:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json +from pathlib import Path +from graphify.transcribe import build_whisper_prompt, transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) + +# Try to load god nodes from a previous partial run or pass [] if not yet available +try: + analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) + god_nodes = analysis.get('god_nodes', []) +except Exception: + god_nodes = [] + +prompt = build_whisper_prompt(god_nodes) +print(f'Whisper prompt: {prompt}') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. ### Step 3 - Extract entities and relationships @@ -1152,8 +1200,9 @@ except RuntimeError as e: Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. Supported URL types (auto-detected): +- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) - Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` +- arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run - Any webpage → converted to markdown via html2text diff --git a/graphify/transcribe.py b/graphify/transcribe.py new file mode 100644 index 000000000..5701ac56a --- /dev/null +++ b/graphify/transcribe.py @@ -0,0 +1,202 @@ +# Video transcription using faster-whisper +# Converts video/audio files to text transcripts for graph extraction +from __future__ import annotations + +import os +from pathlib import Path + + +VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'} +URL_PREFIXES = ('http://', 'https://', 'www.') + +_DEFAULT_MODEL = "base" +_TRANSCRIPTS_DIR = "graphify-out/transcripts" +_FALLBACK_PROMPT = "Use proper punctuation and paragraph breaks." + + +def _model_name() -> str: + return os.environ.get("GRAPHIFY_WHISPER_MODEL", _DEFAULT_MODEL) + + +def _get_whisper(): + try: + from faster_whisper import WhisperModel + return WhisperModel + except ImportError as exc: + raise ImportError( + "Video transcription requires faster-whisper. " + "Run: pip install 'graphifyy[video]'" + ) from exc + + +def _get_yt_dlp(): + try: + import yt_dlp + return yt_dlp + except ImportError as exc: + raise ImportError( + "YouTube/URL download requires yt-dlp. " + "Run: pip install 'graphifyy[video]'" + ) from exc + + +def is_url(path: str) -> bool: + """Return True if the string looks like a URL rather than a file path.""" + return any(path.startswith(p) for p in URL_PREFIXES) + + +def download_audio(url: str, output_dir: Path) -> Path: + """Download audio-only stream from a URL using yt-dlp. + + Returns the path to the downloaded audio file (.m4a or .opus). + Uses cached file if already downloaded. + """ + yt_dlp = _get_yt_dlp() + output_dir.mkdir(parents=True, exist_ok=True) + + # yt-dlp uses %(title)s which can be long/weird — use a stable name based on URL hash + import hashlib + url_hash = hashlib.sha1(url.encode()).hexdigest()[:12] + out_template = str(output_dir / f"yt_{url_hash}.%(ext)s") + + # Check for already-downloaded file + for ext in ('.m4a', '.opus', '.mp3', '.ogg', '.wav', '.webm'): + candidate = output_dir / f"yt_{url_hash}{ext}" + if candidate.exists(): + print(f" cached audio: {candidate.name}") + return candidate + + ydl_opts = { + 'format': 'bestaudio[ext=m4a]/bestaudio/best', + 'outtmpl': out_template, + 'quiet': True, + 'no_warnings': True, + 'noplaylist': True, + 'postprocessors': [], # no ffmpeg needed — use native audio + } + + print(f" downloading audio: {url[:80]} ...", flush=True) + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(url, download=True) + ext = info.get('ext', 'm4a') + downloaded = output_dir / f"yt_{url_hash}.{ext}" + if not downloaded.exists(): + # yt-dlp may have picked a different extension + for p in output_dir.glob(f"yt_{url_hash}.*"): + downloaded = p + break + return downloaded + + +def build_whisper_prompt(god_nodes: list[dict]) -> str: + """Build a domain hint for Whisper from god nodes extracted from the corpus. + + Takes the top god nodes (most connected concepts) already extracted from + non-video files and asks the LLM to summarise them into a one-sentence + speech-to-text hint. Falls back to a generic prompt if no nodes available. + """ + if not god_nodes: + return _FALLBACK_PROMPT + + # Use env override if set + override = os.environ.get("GRAPHIFY_WHISPER_PROMPT") + if override: + return override + + labels = [n.get("label", "") for n in god_nodes[:10] if n.get("label")] + if not labels: + return _FALLBACK_PROMPT + + try: + import anthropic + client = anthropic.Anthropic() + msg = client.messages.create( + model="claude-haiku-4-5-20251001", + max_tokens=60, + messages=[{ + "role": "user", + "content": ( + f"These are the key concepts from a document corpus: {', '.join(labels)}. " + "Write a single short sentence (under 20 words) that describes the domain " + "for a speech-to-text model. Start with 'Technical' or the domain name. " + "No explanation, just the sentence." + ), + }], + ) + prompt = msg.content[0].text.strip().strip('"') + return prompt + " Use proper punctuation and paragraph breaks." + except Exception: + # If LLM call fails for any reason, fall back gracefully + topics = ", ".join(labels[:5]) + return f"Technical discussion about {topics}. Use proper punctuation and paragraph breaks." + + +def transcribe( + video_path: Path | str, + output_dir: Path | None = None, + initial_prompt: str | None = None, + force: bool = False, +) -> Path: + """Transcribe a video/audio file or URL to a .txt transcript. + + If video_path is a URL, audio is downloaded first via yt-dlp. + Returns the path to the saved transcript file. + Uses cached transcript if it exists unless force=True. + + initial_prompt: domain hint for Whisper (built from corpus god nodes). + force: re-transcribe even if transcript already exists. + """ + out_dir = Path(output_dir) if output_dir else Path(_TRANSCRIPTS_DIR) + out_dir.mkdir(parents=True, exist_ok=True) + + if is_url(str(video_path)): + audio_path = download_audio(str(video_path), out_dir / "downloads") + else: + audio_path = Path(video_path) + + transcript_path = out_dir / (audio_path.stem + ".txt") + if transcript_path.exists() and not force: + return transcript_path + + WhisperModel = _get_whisper() + model_name = _model_name() + prompt = initial_prompt or _FALLBACK_PROMPT + + print(f" transcribing {audio_path.name} (model={model_name}) ...", flush=True) + model = WhisperModel(model_name, device="cpu", compute_type="int8") + segments, info = model.transcribe( + str(audio_path), + beam_size=5, + initial_prompt=prompt, + ) + + lines = [segment.text.strip() for segment in segments if segment.text.strip()] + transcript = "\n".join(lines) + + transcript_path.write_text(transcript, encoding="utf-8") + lang = info.language if hasattr(info, "language") else "unknown" + print(f" transcript saved -> {transcript_path} (lang={lang}, {len(lines)} segments)") + return transcript_path + + +def transcribe_all( + video_files: list[str], + output_dir: Path | None = None, + initial_prompt: str | None = None, +) -> list[str]: + """Transcribe a list of video/audio files or URLs, return paths to transcript .txt files. + + Already-transcribed files are returned from cache instantly. + initial_prompt is shared across all files — built once from corpus god nodes. + """ + if not video_files: + return [] + + transcript_paths = [] + for vf in video_files: + try: + t = transcribe(vf, output_dir, initial_prompt=initial_prompt) + transcript_paths.append(str(t)) + except Exception as exc: + print(f" warning: could not transcribe {vf}: {exc}") + return transcript_paths diff --git a/pyproject.toml b/pyproject.toml index 2fc9ed258..3653ba9ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.28" +version = "0.3.29" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } @@ -47,7 +47,8 @@ pdf = ["pypdf", "html2text"] watch = ["watchdog"] leiden = ["graspologic"] office = ["python-docx", "openpyxl"] -all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic", "python-docx", "openpyxl"] +video = ["faster-whisper", "yt-dlp"] +all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic", "python-docx", "openpyxl", "faster-whisper", "yt-dlp"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/test_detect.py b/tests/test_detect.py index f743a6dc6..ed43fea2b 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -199,3 +199,40 @@ def test_detect_handles_circular_symlinks(tmp_path): result = detect(tmp_path, follow_symlinks=True) assert any("main.py" in f for f in result["files"]["code"]) + + +def test_classify_video_extensions(): + """Video and audio file extensions should classify as VIDEO.""" + from graphify.detect import FileType + assert classify_file(Path("lecture.mp4")) == FileType.VIDEO + assert classify_file(Path("podcast.mp3")) == FileType.VIDEO + assert classify_file(Path("talk.mov")) == FileType.VIDEO + assert classify_file(Path("recording.wav")) == FileType.VIDEO + assert classify_file(Path("webinar.webm")) == FileType.VIDEO + assert classify_file(Path("audio.m4a")) == FileType.VIDEO + + +def test_detect_includes_video_key(tmp_path): + """detect() result always includes a 'video' key even with no video files.""" + (tmp_path / "main.py").write_text("x = 1") + result = detect(tmp_path) + assert "video" in result["files"] + + +def test_detect_finds_video_files(tmp_path): + """detect() correctly counts video files and does not add them to word count.""" + (tmp_path / "lecture.mp4").write_bytes(b"fake video data") + (tmp_path / "notes.md").write_text("# Notes\nSome content here.") + result = detect(tmp_path) + assert len(result["files"]["video"]) == 1 + assert any("lecture.mp4" in f for f in result["files"]["video"]) + # total_words should not include video files (they have no readable text) + assert result["total_words"] >= 0 # won't crash + + +def test_detect_video_not_in_words(tmp_path): + """Video files do not contribute to total_words.""" + (tmp_path / "clip.mp4").write_bytes(b"\x00" * 100) + result = detect(tmp_path) + # Only video file present — total_words should be 0 + assert result["total_words"] == 0 diff --git a/tests/test_transcribe.py b/tests/test_transcribe.py new file mode 100644 index 000000000..c1a002b26 --- /dev/null +++ b/tests/test_transcribe.py @@ -0,0 +1,168 @@ +"""Tests for graphify.transcribe — video/audio transcription support.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from graphify.transcribe import ( + VIDEO_EXTENSIONS, + build_whisper_prompt, + transcribe, + transcribe_all, +) + + +# --------------------------------------------------------------------------- +# VIDEO_EXTENSIONS +# --------------------------------------------------------------------------- + +def test_video_extensions_set(): + assert ".mp4" in VIDEO_EXTENSIONS + assert ".mp3" in VIDEO_EXTENSIONS + assert ".wav" in VIDEO_EXTENSIONS + assert ".mov" in VIDEO_EXTENSIONS + assert ".py" not in VIDEO_EXTENSIONS + + +# --------------------------------------------------------------------------- +# build_whisper_prompt +# --------------------------------------------------------------------------- + +def test_build_whisper_prompt_no_nodes(): + """Empty god_nodes returns fallback prompt.""" + prompt = build_whisper_prompt([]) + assert "punctuation" in prompt.lower() or len(prompt) > 0 + + +def test_build_whisper_prompt_env_override(monkeypatch): + """GRAPHIFY_WHISPER_PROMPT env var short-circuits LLM call.""" + monkeypatch.setenv("GRAPHIFY_WHISPER_PROMPT", "Custom domain hint.") + prompt = build_whisper_prompt([{"label": "Python"}, {"label": "FastAPI"}]) + assert prompt == "Custom domain hint." + + +def test_build_whisper_prompt_llm_success(): + """Successful LLM call returns generated prompt with punctuation suffix.""" + god_nodes = [{"label": "neural networks"}, {"label": "transformers"}, {"label": "attention"}] + + fake_response = MagicMock() + fake_response.content = [MagicMock(text="Machine learning and deep learning research")] + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("GRAPHIFY_WHISPER_PROMPT", None) + with patch("anthropic.Anthropic") as MockClient: + MockClient.return_value.messages.create.return_value = fake_response + prompt = build_whisper_prompt(god_nodes) + + assert "Machine learning" in prompt + assert "punctuation" in prompt.lower() + + +def test_build_whisper_prompt_llm_failure_fallback(): + """If LLM call raises, falls back to topic-based prompt.""" + god_nodes = [{"label": "kubernetes"}, {"label": "docker"}, {"label": "helm"}] + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("GRAPHIFY_WHISPER_PROMPT", None) + with patch("anthropic.Anthropic", side_effect=Exception("API error")): + prompt = build_whisper_prompt(god_nodes) + + assert "kubernetes" in prompt.lower() or "docker" in prompt.lower() + assert "punctuation" in prompt.lower() + + +def test_build_whisper_prompt_nodes_without_labels(): + """Nodes missing 'label' keys are safely skipped.""" + god_nodes = [{"id": "1"}, {"id": "2", "label": ""}] + prompt = build_whisper_prompt(god_nodes) + assert len(prompt) > 0 + + +# --------------------------------------------------------------------------- +# transcribe +# --------------------------------------------------------------------------- + +def test_transcribe_uses_cache(tmp_path): + """If transcript already exists, transcribe() returns cached path without running Whisper.""" + video = tmp_path / "lecture.mp4" + video.write_bytes(b"fake") + out_dir = tmp_path / "transcripts" + out_dir.mkdir() + cached = out_dir / "lecture.txt" + cached.write_text("Cached transcript content.") + + result = transcribe(video, output_dir=out_dir) + assert result == cached + + +def test_transcribe_force_reruns(tmp_path): + """force=True re-transcribes even when cache exists.""" + video = tmp_path / "talk.mp4" + video.write_bytes(b"fake") + out_dir = tmp_path / "transcripts" + out_dir.mkdir() + (out_dir / "talk.txt").write_text("Old transcript.") + + fake_segment = MagicMock() + fake_segment.text = "New transcript segment." + fake_info = MagicMock() + fake_info.language = "en" + + fake_model = MagicMock() + fake_model.transcribe.return_value = ([fake_segment], fake_info) + + with patch("graphify.transcribe._get_whisper", return_value=lambda *a, **kw: fake_model): + result = transcribe(video, output_dir=out_dir, force=True) + + assert result.read_text() == "New transcript segment." + + +def test_transcribe_missing_faster_whisper(tmp_path): + """ImportError propagates when faster_whisper is not installed.""" + video = tmp_path / "clip.mp4" + video.write_bytes(b"fake") + + with patch("graphify.transcribe._get_whisper", side_effect=ImportError("faster-whisper not installed")): + with pytest.raises(ImportError): + transcribe(video, output_dir=tmp_path / "out") + + +# --------------------------------------------------------------------------- +# transcribe_all +# --------------------------------------------------------------------------- + +def test_transcribe_all_empty(): + """Empty input returns empty list without error.""" + assert transcribe_all([]) == [] + + +def test_transcribe_all_uses_cache(tmp_path): + """transcribe_all() returns cached paths for already-transcribed files.""" + video = tmp_path / "lecture.mp4" + video.write_bytes(b"fake") + out_dir = tmp_path / "transcripts" + out_dir.mkdir() + cached = out_dir / "lecture.txt" + cached.write_text("Cached.") + + results = transcribe_all([str(video)], output_dir=out_dir) + assert len(results) == 1 + assert str(cached) in results[0] + + +def test_transcribe_all_skips_failed(tmp_path): + """transcribe_all() warns and skips files that fail to transcribe.""" + video = tmp_path / "broken.mp4" + video.write_bytes(b"fake") + + def raise_import(*args, **kwargs): + raise ImportError("faster_whisper not installed") + + with patch("graphify.transcribe.transcribe", side_effect=RuntimeError("boom")): + results = transcribe_all([str(video)], output_dir=tmp_path / "out") + + assert results == [] From f758911b108d15ddebbc2a8fe5432138d8226ae9 Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 15:42:35 +0100 Subject: [PATCH 44/59] Update README and descriptions to mention video/audio and YouTube support Co-Authored-By: Claude Sonnet 4.6 --- README.md | 6 ++++-- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0a0e2de45..229b02010 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Same syntax as `.gitignore`. Patterns match against file paths relative to the f ## How it works -graphify runs in two passes. First, a deterministic AST pass extracts structure from code files (classes, functions, imports, call graphs, docstrings, rationale comments) with no LLM needed. Second, Claude subagents run in parallel over docs, papers, and images to extract concepts, relationships, and design rationale. The results are merged into a NetworkX graph, clustered with Leiden community detection, and exported as interactive HTML, queryable JSON, and a plain-language audit report. +graphify runs in three passes. First, a deterministic AST pass extracts structure from code files (classes, functions, imports, call graphs, docstrings, rationale comments) with no LLM needed. Second, video and audio files are transcribed locally with faster-whisper using a domain-aware prompt derived from corpus god nodes — transcripts are cached so re-runs are instant. Third, Claude subagents run in parallel over docs, papers, images, and transcripts to extract concepts, relationships, and design rationale. The results are merged into a NetworkX graph, clustered with Leiden community detection, and exported as interactive HTML, queryable JSON, and a plain-language audit report. **Clustering is graph-topology-based — no embeddings.** Leiden finds communities by edge density. The semantic similarity edges that Claude extracts (`semantically_similar_to`, marked INFERRED) are already in the graph, so they influence community detection directly. The graph structure is the similarity signal — no separate embedding step or vector database needed. @@ -251,6 +251,8 @@ Works with any mix of file types: | Office | `.docx .xlsx` | Converted to markdown then extracted via Claude (requires `pip install graphifyy[office]`) | | Papers | `.pdf` | Citation mining + concept extraction | | Images | `.png .jpg .webp .gif` | Claude vision - screenshots, diagrams, any language | +| Video / Audio | `.mp4 .mov .mkv .webm .avi .m4v .mp3 .wav .m4a .ogg` | Transcribed locally with faster-whisper, transcript fed into Claude extraction (requires `pip install graphifyy[video]`) | +| YouTube / URLs | any video URL | Audio downloaded via yt-dlp, then same Whisper pipeline (requires `pip install graphifyy[video]`) | ## What you get @@ -288,7 +290,7 @@ Token reduction scales with corpus size. 6 files fits in a context window anyway ## Privacy -graphify sends file contents to your AI coding assistant's underlying model API for semantic extraction of docs, papers, and images — Anthropic (Claude Code), OpenAI (Codex), or whichever provider your platform uses. Code files are processed locally via tree-sitter AST — no file contents leave your machine for code. No telemetry, usage tracking, or analytics of any kind. The only network calls are to your platform's model API during extraction, using your own API key. +graphify sends file contents to your AI coding assistant's underlying model API for semantic extraction of docs, papers, and images — Anthropic (Claude Code), OpenAI (Codex), or whichever provider your platform uses. Code files are processed locally via tree-sitter AST — no file contents leave your machine for code. Video and audio files are transcribed locally with faster-whisper — audio never leaves your machine. No telemetry, usage tracking, or analytics of any kind. The only network calls are to your platform's model API during extraction, using your own API key. ## Tech stack diff --git a/pyproject.toml b/pyproject.toml index 3653ba9ff..ab69b5f49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" version = "0.3.29" -description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, or images into a queryable knowledge graph" +description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } keywords = ["claude", "claude-code", "codex", "opencode", "cursor", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] From a2872cafac1bef45b03078f4264f181942c436a9 Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 15:44:44 +0100 Subject: [PATCH 45/59] Add YouTube URL example to README usage, update repo about with pip install Co-Authored-By: Claude Sonnet 4.6 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 229b02010..d7b156285 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,7 @@ When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"` /graphify add https://arxiv.org/abs/1706.03762 # fetch a paper, save, update graph /graphify add https://x.com/karpathy/status/... # fetch a tweet +/graphify add https://www.youtube.com/watch?v=... # download audio, transcribe, add to graph /graphify add https://... --author "Name" # tag the original author /graphify add https://... --contributor "Name" # tag who added it to the corpus From 2c21bc04882495e3165b86c5e0e9433a029894ad Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 15:48:54 +0100 Subject: [PATCH 46/59] Fix CI: mock lazy anthropic import via sys.modules instead of module attribute Co-Authored-By: Claude Sonnet 4.6 --- tests/test_transcribe.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_transcribe.py b/tests/test_transcribe.py index c1a002b26..0157b7ea8 100644 --- a/tests/test_transcribe.py +++ b/tests/test_transcribe.py @@ -6,6 +6,8 @@ from pathlib import Path from unittest.mock import MagicMock, patch +import sys + import pytest from graphify.transcribe import ( @@ -52,10 +54,12 @@ def test_build_whisper_prompt_llm_success(): fake_response = MagicMock() fake_response.content = [MagicMock(text="Machine learning and deep learning research")] + mock_anthropic = MagicMock() + mock_anthropic.Anthropic.return_value.messages.create.return_value = fake_response + with patch.dict(os.environ, {}, clear=False): os.environ.pop("GRAPHIFY_WHISPER_PROMPT", None) - with patch("anthropic.Anthropic") as MockClient: - MockClient.return_value.messages.create.return_value = fake_response + with patch.dict(sys.modules, {"anthropic": mock_anthropic}): prompt = build_whisper_prompt(god_nodes) assert "Machine learning" in prompt @@ -66,9 +70,12 @@ def test_build_whisper_prompt_llm_failure_fallback(): """If LLM call raises, falls back to topic-based prompt.""" god_nodes = [{"label": "kubernetes"}, {"label": "docker"}, {"label": "helm"}] + mock_anthropic = MagicMock() + mock_anthropic.Anthropic.return_value.messages.create.side_effect = Exception("API error") + with patch.dict(os.environ, {}, clear=False): os.environ.pop("GRAPHIFY_WHISPER_PROMPT", None) - with patch("anthropic.Anthropic", side_effect=Exception("API error")): + with patch.dict(sys.modules, {"anthropic": mock_anthropic}): prompt = build_whisper_prompt(god_nodes) assert "kubernetes" in prompt.lower() or "docker" in prompt.lower() From 699e9960ce7b88076db33a4da3adbd53fb410c7c Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 15:55:22 +0100 Subject: [PATCH 47/59] Remove Anthropic API call from transcribe.py - agent generates Whisper prompt itself Co-Authored-By: Claude Sonnet 4.6 --- graphify/skill-aider.md | 30 +++++++++++++++--------------- graphify/skill-claw.md | 30 +++++++++++++++--------------- graphify/skill-codex.md | 29 +++++++++++++++-------------- graphify/skill-copilot.md | 30 +++++++++++++++--------------- graphify/skill-droid.md | 30 +++++++++++++++--------------- graphify/skill-opencode.md | 30 +++++++++++++++--------------- graphify/skill-trae.md | 30 +++++++++++++++--------------- graphify/skill-windows.md | 29 +++++++++++++++-------------- graphify/skill.md | 31 ++++++++++++++++--------------- graphify/transcribe.py | 32 ++++++-------------------------- tests/test_transcribe.py | 36 ++++-------------------------------- 11 files changed, 146 insertions(+), 191 deletions(-) diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index d61244831..520aea6bb 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -115,30 +115,30 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command:** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -# Try to load god nodes from a previous partial run or pass [] if not yet available -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 539abbaaa..9b653752f 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -115,30 +115,30 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command:** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -# Try to load god nodes from a previous partial run or pass [] if not yet available -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 7f1d76b71..94d41584d 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -114,29 +114,30 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command:** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index ef3cefef3..981224738 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -117,30 +117,30 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command:** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -# Try to load god nodes from a previous partial run or pass [] if not yet available -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 5395a5ab7..49197ffec 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -115,30 +115,30 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command:** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -# Try to load god nodes from a previous partial run or pass [] if not yet available -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 6f352ead0..7e86c6f31 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -115,30 +115,30 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command:** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -# Try to load god nodes from a previous partial run or pass [] if not yet available -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index ec2c56686..fdbb3ebd2 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -114,30 +114,30 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command:** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -# Try to load god nodes from a previous partial run or pass [] if not yet available -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 41daccd82..9d7bc6bba 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -107,29 +107,30 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command (PowerShell):** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `$env:GRAPHIFY_WHISPER_PROMPT` before running the transcription command. + +**Step 2 - Transcribe (PowerShell):** ```powershell & (Get-Content graphify-out\.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/skill.md b/graphify/skill.md index 1fb84be74..5a6d137f3 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -119,30 +119,31 @@ Skip this step entirely if `detect` returned zero `video` files. Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. -**Strategy:** Run non-video semantic extraction first (Step 3B) to get god nodes, use those to build a domain hint for Whisper, then transcribe. This keeps the prompt relevant without guessing the corpus topic from filenames. +**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. -**However**, if the corpus has *only* video files and no other docs/code, skip the god-node step and transcribe with the generic fallback prompt immediately. +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` -**Transcription command:** +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `WHISPER_PROMPT` to use in the next command. + +**Step 2 - Transcribe:** ```bash +GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed $(cat graphify-out/.graphify_python) -c " -import json +import json, os from pathlib import Path -from graphify.transcribe import build_whisper_prompt, transcribe_all +from graphify.transcribe import transcribe_all detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) video_files = detect.get('files', {}).get('video', []) - -# Try to load god nodes from a previous partial run or pass [] if not yet available -try: - analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - god_nodes = analysis.get('god_nodes', []) -except Exception: - god_nodes = [] - -prompt = build_whisper_prompt(god_nodes) -print(f'Whisper prompt: {prompt}') +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') transcript_paths = transcribe_all(video_files, initial_prompt=prompt) print(json.dumps(transcript_paths)) diff --git a/graphify/transcribe.py b/graphify/transcribe.py index 5701ac56a..70000757a 100644 --- a/graphify/transcribe.py +++ b/graphify/transcribe.py @@ -91,14 +91,14 @@ def download_audio(url: str, output_dir: Path) -> Path: def build_whisper_prompt(god_nodes: list[dict]) -> str: """Build a domain hint for Whisper from god nodes extracted from the corpus. - Takes the top god nodes (most connected concepts) already extracted from - non-video files and asks the LLM to summarise them into a one-sentence - speech-to-text hint. Falls back to a generic prompt if no nodes available. + Formats the top god node labels into a topic string for Whisper. + The coding agent (Claude Code, Codex, etc.) generates the actual one-sentence + domain hint from these labels and passes it via GRAPHIFY_WHISPER_PROMPT or + as initial_prompt — no separate API call needed here. """ if not god_nodes: return _FALLBACK_PROMPT - # Use env override if set override = os.environ.get("GRAPHIFY_WHISPER_PROMPT") if override: return override @@ -107,28 +107,8 @@ def build_whisper_prompt(god_nodes: list[dict]) -> str: if not labels: return _FALLBACK_PROMPT - try: - import anthropic - client = anthropic.Anthropic() - msg = client.messages.create( - model="claude-haiku-4-5-20251001", - max_tokens=60, - messages=[{ - "role": "user", - "content": ( - f"These are the key concepts from a document corpus: {', '.join(labels)}. " - "Write a single short sentence (under 20 words) that describes the domain " - "for a speech-to-text model. Start with 'Technical' or the domain name. " - "No explanation, just the sentence." - ), - }], - ) - prompt = msg.content[0].text.strip().strip('"') - return prompt + " Use proper punctuation and paragraph breaks." - except Exception: - # If LLM call fails for any reason, fall back gracefully - topics = ", ".join(labels[:5]) - return f"Technical discussion about {topics}. Use proper punctuation and paragraph breaks." + topics = ", ".join(labels[:5]) + return f"Technical discussion about {topics}. Use proper punctuation and paragraph breaks." def transcribe( diff --git a/tests/test_transcribe.py b/tests/test_transcribe.py index 0157b7ea8..8e35f2aaf 100644 --- a/tests/test_transcribe.py +++ b/tests/test_transcribe.py @@ -1,13 +1,10 @@ """Tests for graphify.transcribe — video/audio transcription support.""" from __future__ import annotations -import json import os from pathlib import Path from unittest.mock import MagicMock, patch -import sys - import pytest from graphify.transcribe import ( @@ -47,38 +44,13 @@ def test_build_whisper_prompt_env_override(monkeypatch): assert prompt == "Custom domain hint." -def test_build_whisper_prompt_llm_success(): - """Successful LLM call returns generated prompt with punctuation suffix.""" +def test_build_whisper_prompt_returns_topic_string(): + """Returns a topic-based prompt from god node labels — no LLM call.""" god_nodes = [{"label": "neural networks"}, {"label": "transformers"}, {"label": "attention"}] - - fake_response = MagicMock() - fake_response.content = [MagicMock(text="Machine learning and deep learning research")] - - mock_anthropic = MagicMock() - mock_anthropic.Anthropic.return_value.messages.create.return_value = fake_response - with patch.dict(os.environ, {}, clear=False): os.environ.pop("GRAPHIFY_WHISPER_PROMPT", None) - with patch.dict(sys.modules, {"anthropic": mock_anthropic}): - prompt = build_whisper_prompt(god_nodes) - - assert "Machine learning" in prompt - assert "punctuation" in prompt.lower() - - -def test_build_whisper_prompt_llm_failure_fallback(): - """If LLM call raises, falls back to topic-based prompt.""" - god_nodes = [{"label": "kubernetes"}, {"label": "docker"}, {"label": "helm"}] - - mock_anthropic = MagicMock() - mock_anthropic.Anthropic.return_value.messages.create.side_effect = Exception("API error") - - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("GRAPHIFY_WHISPER_PROMPT", None) - with patch.dict(sys.modules, {"anthropic": mock_anthropic}): - prompt = build_whisper_prompt(god_nodes) - - assert "kubernetes" in prompt.lower() or "docker" in prompt.lower() + prompt = build_whisper_prompt(god_nodes) + assert "neural networks" in prompt.lower() or "transformers" in prompt.lower() assert "punctuation" in prompt.lower() From 68bb2bd3dc16c5e4654f586faedcb40435966660 Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 15:59:43 +0100 Subject: [PATCH 48/59] Bump to 0.4.0, set v4 as working branch, update CI badge and skill URL Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 9 +++++++++ README.md | 4 ++-- pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1e253c1f..63e6baf17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.4.0 (2026-04-10) + +- Branch: v4 — video and audio corpus support +- Add: drop `.mp4`, `.mp3`, `.wav`, `.mov`, `.webm`, `.m4a`, `.ogg`, `.mkv`, `.avi`, `.m4v` files into any corpus and graphify transcribes them locally with faster-whisper before extraction +- Add: YouTube and URL download via yt-dlp — `/graphify add https://youtube.com/...` downloads audio-only and feeds it through the same Whisper pipeline +- Add: domain-aware Whisper prompts — the coding agent reads god nodes from the corpus and writes a one-sentence domain hint for Whisper itself, no separate API call +- Add: `graphify-out/transcripts/` cache — transcripts cached by filename; YouTube URLs cached by hash so re-runs skip already-transcribed files +- Requires: `pip install 'graphifyy[video]'` for faster-whisper and yt-dlp + ## 0.3.29 (2026-04-10) - Add: video and audio corpus support — drop `.mp4`, `.mp3`, `.wav`, `.mov`, `.webm`, `.m4a`, `.ogg`, `.mkv`, `.avi`, `.m4v` files into any corpus and graphify transcribes them with faster-whisper before extraction diff --git a/README.md b/README.md index d7b156285..f8a621931 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [English](README.md) | [简体中文](README.zh-CN.md) | [日本語](README.ja-JP.md) | [한국어](README.ko-KR.md) -[![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) +[![CI](https://github.com/safishamsi/graphify/actions/workflows/ci.yml/badge.svg?branch=v4)](https://github.com/safishamsi/graphify/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) [![Downloads](https://static.pepy.tech/badge/graphifyy/month)](https://pepy.tech/project/graphifyy) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) @@ -165,7 +165,7 @@ That gives the assistant structured graph access for repeated queries such as ```bash mkdir -p ~/.claude/skills/graphify -curl -fsSL https://raw.githubusercontent.com/safishamsi/graphify/v3/graphify/skill.md \ +curl -fsSL https://raw.githubusercontent.com/safishamsi/graphify/v4/graphify/skill.md \ > ~/.claude/skills/graphify/SKILL.md ``` diff --git a/pyproject.toml b/pyproject.toml index ab69b5f49..d32a96e23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.3.29" +version = "0.4.0" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From 3ce9bdabf4fe5710ee99c8b6c8b1e131338f97db Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 16:01:14 +0100 Subject: [PATCH 49/59] Add v4 to CI branch triggers Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd113ca36..cd34b55ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: ["v1", "v2", "v3", "main"] + branches: ["v1", "v2", "v3", "v4", "main"] pull_request: - branches: ["v1", "v2", "v3", "main"] + branches: ["v1", "v2", "v3", "v4", "main"] jobs: test: From fa388c8556fe22e8dd9444ca5a9be0ddc955e229 Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 16:07:51 +0100 Subject: [PATCH 50/59] Security: fix YAML injection in ingest.py, injection in export.py, bound collision loop Co-Authored-By: Claude Sonnet 4.6 --- graphify/export.py | 12 ++++++++---- graphify/ingest.py | 22 +++++++++++----------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/graphify/export.py b/graphify/export.py index e58df1764..d35edafc2 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -396,10 +396,14 @@ def to_html( n = len(communities.get(cid, [])) legend_data.append({"cid": cid, "color": color, "label": lbl, "count": n}) - nodes_json = json.dumps(vis_nodes) - edges_json = json.dumps(vis_edges) - legend_json = json.dumps(legend_data) - hyperedges_json = json.dumps(getattr(G, "graph", {}).get("hyperedges", [])) + # Escape sequences so embedded JSON cannot break out of the script tag + def _js_safe(obj) -> str: + return json.dumps(obj).replace(" tuple now = datetime.now(timezone.utc).isoformat() content = f"""--- -source_url: {url} +source_url: "{_yaml_str(url)}" type: tweet -author: {tweet_author} +author: "{_yaml_str(tweet_author)}" captured_at: {now} -contributor: {contributor or author or 'unknown'} +contributor: "{_yaml_str(contributor or author or 'unknown')}" --- # Tweet by @{tweet_author} @@ -109,11 +109,11 @@ def _fetch_webpage(url: str, author: str | None, contributor: str | None) -> tup markdown = _html_to_markdown(html, url) now = datetime.now(timezone.utc).isoformat() content = f"""--- -source_url: {url} +source_url: "{_yaml_str(url)}" type: webpage title: "{_yaml_str(title)}" captured_at: {now} -contributor: {contributor or author or 'unknown'} +contributor: "{_yaml_str(contributor or author or 'unknown')}" --- # {title} @@ -149,13 +149,13 @@ def _fetch_arxiv(url: str, author: str | None, contributor: str | None) -> tuple now = datetime.now(timezone.utc).isoformat() content = f"""--- -source_url: {url} -arxiv_id: {arxiv_id.group(1) if arxiv_id else ''} +source_url: "{_yaml_str(url)}" +arxiv_id: "{_yaml_str(arxiv_id.group(1) if arxiv_id else '')}" type: paper -title: "{title}" -paper_authors: "{paper_authors}" +title: "{_yaml_str(title)}" +paper_authors: "{_yaml_str(paper_authors)}" captured_at: {now} -contributor: {contributor or author or 'unknown'} +contributor: "{_yaml_str(contributor or author or 'unknown')}" --- # {title} @@ -225,7 +225,7 @@ def ingest(url: str, target_dir: Path, author: str | None = None, contributor: s out_path = target_dir / filename # Avoid overwriting - append counter if needed counter = 1 - while out_path.exists(): + while out_path.exists() and counter < 1000: stem = Path(filename).stem out_path = target_dir / f"{stem}_{counter}.md" counter += 1 From 209224d08c2f68b28a03978355dfbb2402fbe707 Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 16:22:53 +0100 Subject: [PATCH 51/59] Add dedicated video and audio corpus section to README Co-Authored-By: Claude Sonnet 4.6 --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index f8a621931..443b80641 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,31 @@ Works with any mix of file types: | Video / Audio | `.mp4 .mov .mkv .webm .avi .m4v .mp3 .wav .m4a .ogg` | Transcribed locally with faster-whisper, transcript fed into Claude extraction (requires `pip install graphifyy[video]`) | | YouTube / URLs | any video URL | Audio downloaded via yt-dlp, then same Whisper pipeline (requires `pip install graphifyy[video]`) | +## Video and audio corpus + +Drop video or audio files into your corpus folder alongside your code and docs — graphify picks them up automatically: + +```bash +pip install 'graphifyy[video]' # one-time setup +/graphify ./my-corpus # transcribes any video/audio files it finds +``` + +Add a YouTube video (or any public video URL) directly: + +```bash +/graphify add https://www.youtube.com/watch?v=... +``` + +yt-dlp downloads audio-only (fast, small), Whisper transcribes it locally, and the transcript is fed into the same extraction pipeline as your other docs. Transcripts are cached in `graphify-out/transcripts/` so re-runs skip already-transcribed files. + +For better accuracy on technical content, use a larger model: + +```bash +/graphify ./my-corpus --whisper-model medium +``` + +Audio never leaves your machine. All transcription runs locally. + ## What you get **God nodes** - highest-degree concepts (what everything connects through) From 0e453083de8f860877809b03de0525589120c7a5 Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 16:24:13 +0100 Subject: [PATCH 52/59] Remove YouTube URL references from README, use generic placeholder Co-Authored-By: Claude Sonnet 4.6 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 443b80641..77e2857b1 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"` /graphify add https://arxiv.org/abs/1706.03762 # fetch a paper, save, update graph /graphify add https://x.com/karpathy/status/... # fetch a tweet -/graphify add https://www.youtube.com/watch?v=... # download audio, transcribe, add to graph +/graphify add # download audio, transcribe, add to graph /graphify add https://... --author "Name" # tag the original author /graphify add https://... --contributor "Name" # tag who added it to the corpus @@ -267,7 +267,7 @@ pip install 'graphifyy[video]' # one-time setup Add a YouTube video (or any public video URL) directly: ```bash -/graphify add https://www.youtube.com/watch?v=... +/graphify add ``` yt-dlp downloads audio-only (fast, small), Whisper transcribes it locally, and the transcript is fed into the same extraction pipeline as your other docs. Transcripts are cached in `graphify-out/transcripts/` so re-runs skip already-transcribed files. From 271ee0aff69f786a8afbe711463f84f4886f7e6a Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 18:41:48 +0100 Subject: [PATCH 53/59] Fix #188: collect_files() now respects .graphifyignore; fix #195: skill.md requires general-purpose subagent type for extraction dispatch Co-Authored-By: Claude Sonnet 4.6 --- graphify/extract.py | 12 ++++++++++-- graphify/skill-codex.md | 6 ++++-- graphify/skill-copilot.md | 6 ++++-- graphify/skill-droid.md | 6 ++++-- graphify/skill-opencode.md | 6 ++++-- graphify/skill-trae.md | 6 ++++-- graphify/skill-windows.md | 6 ++++-- graphify/skill.md | 14 +++++++++----- 8 files changed, 43 insertions(+), 19 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 65e62c646..feaa399ff 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2668,7 +2668,7 @@ def extract(paths: list[Path]) -> dict: } -def collect_files(target: Path, *, follow_symlinks: bool = False) -> list[Path]: +def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | None = None) -> list[Path]: if target.is_file(): return [target] _EXTENSIONS = { @@ -2678,12 +2678,20 @@ def collect_files(target: Path, *, follow_symlinks: bool = False) -> list[Path]: ".lua", ".toc", ".zig", ".ps1", ".m", ".mm", } + from graphify.detect import _load_graphifyignore, _is_ignored + ignore_root = root if root is not None else target + patterns = _load_graphifyignore(ignore_root) + + def _ignored(p: Path) -> bool: + return bool(patterns and _is_ignored(p, ignore_root, patterns)) + if not follow_symlinks: results: list[Path] = [] for ext in sorted(_EXTENSIONS): results.extend( p for p in target.rglob(f"*{ext}") if not any(part.startswith(".") for part in p.parts) + and not _ignored(p) ) return sorted(results) # Walk with symlink following + cycle detection @@ -2701,7 +2709,7 @@ def collect_files(target: Path, *, follow_symlinks: bool = False) -> list[Path]: continue for fname in filenames: p = dp / fname - if p.suffix in _EXTENSIONS and not fname.startswith("."): + if p.suffix in _EXTENSIONS and not fname.startswith(".") and not _ignored(p): results.append(p) return sorted(results) diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 94d41584d..c75a407ec 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -305,10 +305,12 @@ Output exactly this JSON (no other text): **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed, stop and tell the user. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Save new results to cache: ```bash diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 981224738..1bd26f0aa 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -301,10 +301,12 @@ Output exactly this JSON (no other text): **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed, stop and tell the user. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Save new results to cache: ```bash diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 49197ffec..979972017 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -302,10 +302,12 @@ Output exactly this JSON (no other text): **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed, stop and tell the user. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Save new results to cache: ```bash diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 7e86c6f31..d2200f640 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -301,10 +301,12 @@ Output exactly this JSON (no other text): **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed, stop and tell the user. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Save new results to cache: ```bash diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index fdbb3ebd2..89b34c619 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -298,10 +298,12 @@ Accumulate nodes/edges/hyperedges across all results and write to `.graphify_sem **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed, stop and tell the user. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Save new results to cache: ```bash diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 9d7bc6bba..2016f8d2b 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -291,10 +291,12 @@ Output exactly this JSON (no other text): **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed, stop and tell the user. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Save new results to cache: ```powershell diff --git a/graphify/skill.md b/graphify/skill.md index 5a6d137f3..c9fdb8540 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -238,11 +238,13 @@ Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-2 Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose. +**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs. + Concrete example for 3 chunks: ``` -[Agent tool call 1: files 1-15] -[Agent tool call 2: files 16-30] -[Agent tool call 3: files 31-45] +[Agent tool call 1: files 1-15, subagent_type="general-purpose"] +[Agent tool call 2: files 16-30, subagent_type="general-purpose"] +[Agent tool call 3: files 31-45, subagent_type="general-purpose"] ``` All three in one message. Not three separate messages. @@ -304,10 +306,12 @@ Output exactly this JSON (no other text): **Step B3 - Collect, cache, and merge** Wait for all subagents. For each result: -- If a subagent returned valid JSON with `nodes` and `edges`, include it and save each file's nodes/edges to the cache +- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal +- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache +- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. - If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort -If more than half the chunks failed, stop and tell the user. +If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. Save new results to cache: ```bash From 625ca0e2f38bb0804f7818bdbe00cd5450983d7a Mon Sep 17 00:00:00 2001 From: Safi Date: Fri, 10 Apr 2026 18:43:18 +0100 Subject: [PATCH 54/59] Bump to 0.4.1 Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 6 ++++++ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63e6baf17..7c6181d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.4.1 (2026-04-10) + +- Fix: `collect_files()` in `extract.py` now respects `.graphifyignore` — previously ignored patterns, causing thousands of unwanted files (e.g. `node_modules/`) to be scanned (#188) +- Fix: skill.md Step B2 now explicitly requires `subagent_type="general-purpose"` — using `Explore` type silently dropped extraction results since it is read-only and cannot write chunk files (#195) +- Fix: Step B3 now warns when chunk files are missing from disk instead of silently skipping them + ## 0.4.0 (2026-04-10) - Branch: v4 — video and audio corpus support diff --git a/pyproject.toml b/pyproject.toml index d32a96e23..8f7c3e277 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.4.0" +version = "0.4.1" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From 3c503400cb7b0baef5acc1d8795112459dac43e3 Mon Sep 17 00:00:00 2001 From: Safi Date: Sat, 11 Apr 2026 20:53:08 +0100 Subject: [PATCH 55/59] fix bugs #211, #216, #217, #222 and bump to 0.4.2 - extract.py: use str(path) for node IDs to prevent same-basename collision (#211) - build.py: normalize from/to edge keys before KeyError (#216) - export.py: guard ZeroDivisionError when graph has no edges (#217) - hooks.py: remove stale CODE_EXTS filter, rebuild on any changed file (#222) Co-Authored-By: Claude Sonnet 4.6 --- graphify/build.py | 6 ++++++ graphify/export.py | 4 ++-- graphify/extract.py | 18 +++++++++--------- graphify/hooks.py | 10 ++-------- pyproject.toml | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/graphify/build.py b/graphify/build.py index 3c3d80ca6..4cc30f3a0 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -42,6 +42,12 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) node_set = set(G.nodes()) for edge in extraction.get("edges", []): + if "source" not in edge and "from" in edge: + edge["source"] = edge["from"] + if "target" not in edge and "to" in edge: + edge["target"] = edge["to"] + if "source" not in edge or "target" not in edge: + continue src, tgt = edge["source"], edge["target"] if src not in node_set or tgt not in node_set: continue # skip edges to external/stdlib nodes - expected, not an error diff --git a/graphify/export.py b/graphify/export.py index d35edafc2..0f54319d3 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -346,7 +346,7 @@ def to_html( node_community = _node_community_map(communities) degree = dict(G.degree()) - max_deg = max(degree.values()) if degree else 1 + max_deg = max(degree.values(), default=1) or 1 # Build nodes list for vis.js vis_nodes = [] @@ -957,7 +957,7 @@ def to_svg( pos = nx.spring_layout(G, seed=42, k=2.0 / (G.number_of_nodes() ** 0.5 + 1)) degree = dict(G.degree()) - max_deg = max(degree.values()) if degree else 1 + max_deg = max(degree.values(), default=1) or 1 node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in G.nodes()] node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in G.nodes()] diff --git a/graphify/extract.py b/graphify/extract.py index feaa399ff..dd8be4a2b 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -673,7 +673,7 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "weight": weight, }) - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) def walk(node, parent_class_nid: str | None = None) -> None: @@ -1004,7 +1004,7 @@ def _extract_python_rationale(path: Path, result: dict) -> None: nodes = result["nodes"] edges = result["edges"] seen_ids = {n["id"] for n in nodes} - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) def _get_docstring(body_node) -> tuple[str, int] | None: if not body_node: @@ -1200,7 +1200,7 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "weight": weight, }) - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) def _func_name_from_signature(sig_node) -> str | None: @@ -1415,7 +1415,7 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "weight": weight, }) - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) def walk(node) -> None: @@ -1603,7 +1603,7 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "weight": weight, }) - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) def walk(node, parent_impl_nid: str | None = None) -> None: @@ -1761,7 +1761,7 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "confidence": confidence, "source_file": str_path, "source_location": f"L{line}", "weight": weight}) - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) def _extract_import(node) -> None: @@ -1916,7 +1916,7 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "confidence": confidence, "source_file": str_path, "source_location": f"L{line}", "weight": weight}) - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) _PS_SKIP = frozenset({ @@ -2205,7 +2205,7 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "confidence": confidence, "source_file": str_path, "source_location": f"L{line}", "weight": weight}) - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) def _read(node) -> str: @@ -2403,7 +2403,7 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "confidence": confidence, "source_file": str_path, "source_location": f"L{line}", "weight": weight}) - file_nid = _make_id(stem) + file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) _IMPORT_KEYWORDS = frozenset({"alias", "import", "require", "use"}) diff --git a/graphify/hooks.py b/graphify/hooks.py index 92320272b..39fdf89d3 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -46,19 +46,13 @@ import os, sys from pathlib import Path -CODE_EXTS = { - '.py', '.ts', '.js', '.go', '.rs', '.java', '.cpp', '.c', '.rb', '.swift', - '.kt', '.cs', '.scala', '.php', '.cc', '.cxx', '.hpp', '.h', '.kts', -} - changed_raw = os.environ.get('GRAPHIFY_CHANGED', '') changed = [Path(f.strip()) for f in changed_raw.strip().splitlines() if f.strip()] -code_changed = [f for f in changed if f.suffix.lower() in CODE_EXTS and f.exists()] -if not code_changed: +if not changed: sys.exit(0) -print(f'[graphify hook] {len(code_changed)} code file(s) changed - rebuilding graph...') +print(f'[graphify hook] {len(changed)} file(s) changed - rebuilding graph...') try: from graphify.watch import _rebuild_code diff --git a/pyproject.toml b/pyproject.toml index 8f7c3e277..69e1b3b58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.4.1" +version = "0.4.2" description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, OpenClaw, Factory Droid, Trae) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From 62aac4f868b9b6566dfaf9770a08db7d909a5cb8 Mon Sep 17 00:00:00 2001 From: Safi Date: Sat, 11 Apr 2026 21:07:39 +0100 Subject: [PATCH 56/59] cherry-pick PRs #212, #220, #204, #221 into 0.4.2 - build/validate: accept NetworkX <=3.1 "links" key alongside "edges" (#212) - __main__: skip version check during install/uninstall, deduplicate paths (#220) - all file IO: explicit encoding="utf-8" to prevent crashes on Windows CJK locales (#204) - hooks: add newline="\n" on write to prevent CRLF shebang breakage on Windows (#204) - export: strip trailing .md from safe_name so "CLAUDE.md" doesn't become "CLAUDE.md.md" (#221) - report: add Community Hubs navigation block so Obsidian vault stays connected (#221) Co-Authored-By: Claude Sonnet 4.6 --- graphify/__main__.py | 10 ++++++---- graphify/benchmark.py | 2 +- graphify/build.py | 3 +++ graphify/cache.py | 4 ++-- graphify/detect.py | 10 +++++----- graphify/export.py | 15 ++++++++++----- graphify/hooks.py | 12 ++++++------ graphify/report.py | 20 ++++++++++++++++++++ graphify/serve.py | 2 +- graphify/validate.py | 9 +++++---- graphify/watch.py | 4 ++-- 11 files changed, 61 insertions(+), 30 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index a8a56c07b..912113bb1 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -628,10 +628,12 @@ def claude_uninstall(project_dir: Path | None = None) -> None: def main() -> None: - # Check all known skill install locations for a stale version stamp - for cfg in _PLATFORM_CONFIG.values(): - skill_dst = Path.home() / cfg["skill_dst"] - _check_skill_version(skill_dst) + # Check all known skill install locations for a stale version stamp. + # Skip during install/uninstall (hook writes trigger a fresh check anyway). + # Deduplicate paths so platforms sharing the same install dir don't warn twice. + if not any(arg in ("install", "uninstall") for arg in sys.argv): + for skill_dst in {Path.home() / cfg["skill_dst"] for cfg in _PLATFORM_CONFIG.values()}: + _check_skill_version(skill_dst) if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"): print("Usage: graphify ") diff --git a/graphify/benchmark.py b/graphify/benchmark.py index a71e10e7e..dc420564a 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -75,7 +75,7 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ - data = json.loads(Path(graph_path).read_text()) + data = json.loads(Path(graph_path).read_text(encoding="utf-8")) try: G = json_graph.node_link_graph(data, edges="links") except TypeError: diff --git a/graphify/build.py b/graphify/build.py index 4cc30f3a0..4d3a0b987 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -32,6 +32,9 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. """ + # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. + if "edges" not in extraction and "links" in extraction: + extraction = dict(extraction, edges=extraction["links"]) errors = validate_extraction(extraction) # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors. real_errors = [e for e in errors if "does not match any node id" not in e] diff --git a/graphify/cache.py b/graphify/cache.py index 7f73db069..54d5b8e66 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -55,7 +55,7 @@ def load_cached(path: Path, root: Path = Path(".")) -> dict | None: if not entry.exists(): return None try: - return json.loads(entry.read_text()) + return json.loads(entry.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): return None @@ -70,7 +70,7 @@ def save_cached(path: Path, result: dict, root: Path = Path(".")) -> None: entry = cache_dir(root) / f"{h}.json" tmp = entry.with_suffix(".tmp") try: - tmp.write_text(json.dumps(result)) + tmp.write_text(json.dumps(result), encoding="utf-8") os.replace(tmp, entry) except Exception: tmp.unlink(missing_ok=True) diff --git a/graphify/detect.py b/graphify/detect.py index e9dc701f0..c13196d8d 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -69,7 +69,7 @@ def _looks_like_paper(path: Path) -> bool: """Heuristic: does this text file read like an academic paper?""" try: # Only scan first 3000 chars for speed - text = path.read_text(errors="ignore")[:3000] + text = path.read_text(encoding="utf-8", errors="ignore")[:3000] hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text)) return hits >= _PAPER_SIGNAL_THRESHOLD except Exception: @@ -226,7 +226,7 @@ def count_words(path: Path) -> int: return len(docx_to_markdown(path).split()) if ext == ".xlsx": return len(xlsx_to_markdown(path).split()) - return len(path.read_text(errors="ignore").split()) + return len(path.read_text(encoding="utf-8", errors="ignore").split()) except Exception: return 0 @@ -271,7 +271,7 @@ def _load_graphifyignore(root: Path) -> list[str]: while True: ignore_file = current / ".graphifyignore" if ignore_file.exists(): - for line in ignore_file.read_text(errors="ignore").splitlines(): + for line in ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines(): line = line.strip() if line and not line.startswith("#"): patterns.append(line) @@ -427,7 +427,7 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict[str, float]: """Load the file modification time manifest from a previous run.""" try: - return json.loads(Path(manifest_path).read_text()) + return json.loads(Path(manifest_path).read_text(encoding="utf-8")) except Exception: return {} @@ -442,7 +442,7 @@ def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PA except OSError: pass # file deleted between detect() and manifest write - skip it Path(manifest_path).parent.mkdir(parents=True, exist_ok=True) - Path(manifest_path).write_text(json.dumps(manifest, indent=2)) + Path(manifest_path).write_text(json.dumps(manifest, indent=2), encoding="utf-8") def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict: diff --git a/graphify/export.py b/graphify/export.py index 0f54319d3..7ed922b70 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -295,7 +295,7 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> conf = link.get("confidence", "EXTRACTED") link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - with open(output_path, "w") as f: + with open(output_path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) @@ -322,7 +322,7 @@ def to_cypher(G: nx.Graph, output_path: str) -> None: f"MATCH (a {{id: '{u_esc}'}}), (b {{id: '{v_esc}'}}) " f"MERGE (a)-[:{rel} {{confidence: '{conf}'}}]->(b);" ) - with open(output_path, "w") as f: + with open(output_path, "w", encoding="utf-8") as f: f.write("\n".join(lines)) @@ -467,7 +467,10 @@ def to_obsidian( # Map node_id → safe filename so wikilinks stay consistent. # Deduplicate: if two nodes produce the same filename, append a numeric suffix. def safe_name(label: str) -> str: - return re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip() or "unnamed" + cleaned = re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip() + # Strip trailing .md/.mdx/.markdown so "CLAUDE.md" doesn't become "CLAUDE.md.md" + cleaned = re.sub(r"\.(md|mdx|markdown)$", "", cleaned, flags=re.IGNORECASE) + return cleaned or "unnamed" node_filename: dict[str, str] = {} seen_names: dict[str, int] = {} @@ -681,7 +684,7 @@ def _community_reach(node_id: str) -> int: for cid, label in sorted((community_labels or {}).items()) ] } - (obsidian_dir / "graph.json").write_text(json.dumps(graph_config, indent=2)) + (obsidian_dir / "graph.json").write_text(json.dumps(graph_config, indent=2), encoding="utf-8") return G.number_of_nodes() + community_notes_written @@ -703,7 +706,9 @@ def to_canvas( CANVAS_COLORS = ["1", "2", "3", "4", "5", "6"] # red, orange, yellow, green, cyan, purple def safe_name(label: str) -> str: - return re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip() or "unnamed" + cleaned = re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip() + cleaned = re.sub(r"\.(md|mdx|markdown)$", "", cleaned, flags=re.IGNORECASE) + return cleaned or "unnamed" # Build node_filenames if not provided (same dedup logic as to_obsidian) if node_filenames is None: diff --git a/graphify/hooks.py b/graphify/hooks.py index 39fdf89d3..d99a8c4a7 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -113,12 +113,12 @@ def _install_hook(hooks_dir: Path, name: str, script: str, marker: str) -> str: """Install a single git hook, appending if an existing hook is present.""" hook_path = hooks_dir / name if hook_path.exists(): - content = hook_path.read_text() + content = hook_path.read_text(encoding="utf-8") if marker in content: return f"already installed at {hook_path}" - hook_path.write_text(content.rstrip() + "\n\n" + script) + hook_path.write_text(content.rstrip() + "\n\n" + script, encoding="utf-8", newline="\n") return f"appended to existing {name} hook at {hook_path}" - hook_path.write_text("#!/bin/sh\n" + script) + hook_path.write_text("#!/bin/sh\n" + script, encoding="utf-8", newline="\n") hook_path.chmod(0o755) return f"installed at {hook_path}" @@ -128,7 +128,7 @@ def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> hook_path = hooks_dir / name if not hook_path.exists(): return f"no {name} hook found - nothing to remove." - content = hook_path.read_text() + content = hook_path.read_text(encoding="utf-8") if marker not in content: return f"graphify hook not found in {name} - nothing to remove." new_content = re.sub( @@ -140,7 +140,7 @@ def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> if not new_content or new_content in ("#!/bin/bash", "#!/bin/sh"): hook_path.unlink() return f"removed {name} hook at {hook_path}" - hook_path.write_text(new_content + "\n") + hook_path.write_text(new_content + "\n", encoding="utf-8", newline="\n") return f"graphify removed from {name} at {hook_path} (other hook content preserved)" @@ -183,7 +183,7 @@ def _check(name: str, marker: str) -> str: p = hooks_dir / name if not p.exists(): return "not installed" - return "installed" if marker in p.read_text() else "not installed (hook exists but graphify not found)" + return "installed" if marker in p.read_text(encoding="utf-8") else "not installed (hook exists but graphify not found)" commit = _check("post-commit", _HOOK_MARKER) checkout = _check("post-checkout", _CHECKOUT_MARKER) diff --git a/graphify/report.py b/graphify/report.py index 91f331cc3..180233d21 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -1,9 +1,17 @@ # generate GRAPH_REPORT.md - the human-readable audit trail from __future__ import annotations +import re from datetime import date import networkx as nx +def _safe_community_name(label: str) -> str: + """Mirrors export.safe_name so community hub filenames and report wikilinks always agree.""" + cleaned = re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip() + cleaned = re.sub(r"\.(md|mdx|markdown)$", "", cleaned, flags=re.IGNORECASE) + return cleaned or "unnamed" + + def generate( G: nx.Graph, communities: dict[int, list[str]], @@ -48,6 +56,18 @@ def generate( f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS" + (f" · INFERRED: {len(inf_edges)} edges (avg confidence: {inf_avg})" if inf_avg is not None else ""), f"- Token cost: {token_cost.get('input', 0):,} input · {token_cost.get('output', 0):,} output", + ] + + # Community hub navigation - links to _COMMUNITY_*.md files in the Obsidian vault. + # Without these, GRAPH_REPORT.md is a dead-end and the vault splits into disconnected components. + if communities: + lines += ["", "## Community Hubs (Navigation)"] + for cid in communities: + label = community_labels.get(cid, f"Community {cid}") + safe = _safe_community_name(label) + lines.append(f"- [[_COMMUNITY_{safe}|{label}]]") + + lines += [ "", "## God Nodes (most connected - your core abstractions)", ] diff --git a/graphify/serve.py b/graphify/serve.py index 81c9353ab..279b5d316 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -16,7 +16,7 @@ def _load_graph(graph_path: str) -> nx.Graph: if not resolved.exists(): raise FileNotFoundError(f"Graph file not found: {resolved}") safe = resolved - data = json.loads(safe.read_text()) + data = json.loads(safe.read_text(encoding="utf-8")) try: return json_graph.node_link_graph(data, edges="links") except TypeError: diff --git a/graphify/validate.py b/graphify/validate.py index 2c3727777..45139974e 100644 --- a/graphify/validate.py +++ b/graphify/validate.py @@ -36,14 +36,15 @@ def validate_extraction(data: dict) -> list[str]: f"'{node['file_type']}' - must be one of {sorted(VALID_FILE_TYPES)}" ) - # Edges - if "edges" not in data: + # Edges - accept "links" (NetworkX <= 3.1) as fallback for "edges" + edge_list = data.get("edges") if "edges" in data else data.get("links") + if edge_list is None: errors.append("Missing required key 'edges'") - elif not isinstance(data["edges"], list): + elif not isinstance(edge_list, list): errors.append("'edges' must be a list") else: node_ids = {n["id"] for n in data.get("nodes", []) if isinstance(n, dict) and "id" in n} - for i, edge in enumerate(data["edges"]): + for i, edge in enumerate(edge_list): if not isinstance(edge, dict): errors.append(f"Edge {i} must be an object") continue diff --git a/graphify/watch.py b/graphify/watch.py index 734de8bf0..df2871f6f 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -53,7 +53,7 @@ def _rebuild_code(watch_path: Path, *, follow_symlinks: bool = False) -> bool: report = generate(G, communities, cohesion, labels, gods, surprises, detection, {"input": 0, "output": 0}, str(watch_path), suggested_questions=questions) - (out / "GRAPH_REPORT.md").write_text(report) + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") to_json(G, communities, str(out / "graph.json")) # clear stale needs_update flag if present @@ -75,7 +75,7 @@ def _notify_only(watch_path: Path) -> None: """Write a flag file and print a notification (fallback for non-code-only corpora).""" flag = watch_path / "graphify-out" / "needs_update" flag.parent.mkdir(parents=True, exist_ok=True) - flag.write_text("1") + flag.write_text("1", encoding="utf-8") print(f"\n[graphify watch] New or changed files detected in {watch_path}") print("[graphify watch] Non-code files changed - semantic re-extraction requires LLM.") print("[graphify watch] Run `/graphify --update` in Claude Code to update the graph.") From b4d4ac8e9bbb5ef9f1d0f582bdec13e3bb824829 Mon Sep 17 00:00:00 2001 From: Safi Date: Sat, 11 Apr 2026 21:10:45 +0100 Subject: [PATCH 57/59] add 0.4.2 CHANGELOG entry --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c6181d8d..25a69bb3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.4.2 (2026-04-11) + +- Fix: same-basename files in different directories produced colliding node IDs — now uses full path (#211) +- Fix: edges using `from`/`to` keys instead of `source`/`target` were silently dropped (#216) +- Fix: empty graphs (no edges) crashed `to_html` with `ZeroDivisionError` (#217) +- Fix: post-commit hook skipped `.tsx`, `.jsx`, and other valid code extensions due to stale allowlist (#222) +- Fix: NetworkX ≤3.1 serialises edges as `links` — now accepted alongside `edges` (#212) +- Fix: version warning fired during `install`/`uninstall` and duplicated on shared paths (#220) +- Fix: all file IO now uses `encoding="utf-8"` — prevents crashes on Windows with CJK or emoji labels; hook writes use `newline="\n"` to prevent CRLF shebang breakage (#204) +- Fix: Obsidian export — node labels ending in `.md` produced `.md.md` filenames; `GRAPH_REPORT.md` now links to community hub files so vault stays in one connected component (#221) + ## 0.4.1 (2026-04-10) - Fix: `collect_files()` in `extract.py` now respects `.graphifyignore` — previously ignored patterns, causing thousands of unwanted files (e.g. `node_modules/`) to be scanned (#188) From 013313f8d62ebba82985b899bb554b8231bdd8ee Mon Sep 17 00:00:00 2001 From: Safi Date: Sat, 11 Apr 2026 21:15:32 +0100 Subject: [PATCH 58/59] add LinkedIn badge to README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 77e2857b1..250909521 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![PyPI](https://img.shields.io/pypi/v/graphifyy)](https://pypi.org/project/graphifyy/) [![Downloads](https://static.pepy.tech/badge/graphifyy/month)](https://pepy.tech/project/graphifyy) [![Sponsor](https://img.shields.io/badge/sponsor-safishamsi-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/safishamsi) +[![LinkedIn](https://img.shields.io/badge/LinkedIn-Safi%20Shamsi-0077B5?logo=linkedin)](https://www.linkedin.com/in/safi-shamsi) **An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, Aider, OpenClaw, Factory Droid, or Trae - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. @@ -320,7 +321,7 @@ graphify sends file contents to your AI coding assistant's underlying model API ## Tech stack -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantic extraction via Claude (Claude Code), GPT-4 (Codex), or whichever model your platform runs. No Neo4j required, no server, runs entirely locally. +NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantic extraction via Claude (Claude Code), GPT-4 (Codex), or whichever model your platform runs. Video transcription via faster-whisper + yt-dlp (optional, `pip install graphifyy[video]`). No Neo4j required, no server, runs entirely locally. ## What we are building next From 4dfcad30450871c070b2bf99404e7f09237fd6c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erhan=20=C3=9CRG=C3=9CN?= <405591@ogr.ktu.edu.tr> Date: Sun, 12 Apr 2026 03:09:07 +0300 Subject: [PATCH 59/59] feat(php): capture scoped_call_expression as uses_static edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PHP static method calls like StringHelper::slugify($title) are not picked up by the extractor today because the walker only handles function_call_expression and member_call_expression. This leaves helper classes, façade-style utilities and static factory methods invisible to the graph. Adds a static_call_types field to LanguageConfig and populates it in _PHP_CONFIG with scoped_call_expression. The walker resolves the class reference the same way the const handler does: scope field first, then a positional name fallback for older tree-sitter-php layouts. A new uses_static edge is emitted from the calling function or method to the class that owns the static method. Test: tests/test_languages.py covers sample_php_static_call.php and asserts both that the uses_static relation appears and that the target is the class that holds the static method. --- graphify/extract.py | 29 +++++++++++++++++++++++ tests/fixtures/sample_php_static_call.php | 29 +++++++++++++++++++++++ tests/test_languages.py | 13 ++++++++++ 3 files changed, 71 insertions(+) create mode 100644 tests/fixtures/sample_php_static_call.php diff --git a/graphify/extract.py b/graphify/extract.py index dd8be4a2b..9ca9e45cf 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -29,6 +29,7 @@ class LanguageConfig: function_types: frozenset = frozenset() import_types: frozenset = frozenset() call_types: frozenset = frozenset() + static_call_types: frozenset = frozenset() # Name extraction name_field: str = "name" @@ -536,6 +537,7 @@ def _swift_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: s function_types=frozenset({"function_definition", "method_declaration"}), import_types=frozenset({"namespace_use_clause"}), call_types=frozenset({"function_call_expression", "member_call_expression"}), + static_call_types=frozenset({"scoped_call_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"member_call_expression"}), call_accessor_field="name", @@ -849,6 +851,7 @@ def walk(node, parent_class_nid: str | None = None) -> None: label_to_nid[normalised.lower()] = n["id"] seen_call_pairs: set[tuple[str, str]] = set() + seen_static_ref_pairs: set[tuple[str, str, str]] = set() def walk_calls(node, caller_nid: str) -> None: if node.type in config.function_boundary_types: @@ -962,6 +965,32 @@ def walk_calls(node, caller_nid: str) -> None: "weight": 1.0, }) + # Static method calls: Foo::bar(), Foo::make() + if node.type in config.static_call_types: + scope_node = node.child_by_field_name("scope") + if scope_node is None: + for child in node.children: + if child.is_named and child.type in ("name", "qualified_name", "identifier"): + scope_node = child + break + if scope_node is not None: + class_name = _read_text(scope_node, source) + tgt_nid = label_to_nid.get(class_name.lower()) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid, "uses_static") + if pair not in seen_static_ref_pairs: + seen_static_ref_pairs.add(pair) + line = node.start_point[0] + 1 + edges.append({ + "source": caller_nid, + "target": tgt_nid, + "relation": "uses_static", + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": f"L{line}", + "weight": 1.0, + }) + for child in node.children: walk_calls(child, caller_nid) diff --git a/tests/fixtures/sample_php_static_call.php b/tests/fixtures/sample_php_static_call.php new file mode 100644 index 000000000..ef2be7fd2 --- /dev/null +++ b/tests/fixtures/sample_php_static_call.php @@ -0,0 +1,29 @@ +