From 48e54ecb16d7fb856718d24f632c5f1d4553c598 Mon Sep 17 00:00:00 2001 From: edudatsuj45 Date: Sun, 7 Jun 2026 15:17:11 +0800 Subject: [PATCH] fix: _is_sensitive no longer drops topic notes like token-economics-of-recall.md The generic keyword patterns (credential/secret/password/token) flagged any filename containing the keyword as a standalone word, silently dropping prose documents whose descriptive slug merely mentions the topic: token-economics-of-recall.md -> skipped as sensitive (a note ABOUT tokens) password-policy-discussion.md -> skipped as sensitive Follow-up to the #436 -> #718 -> #920 lineage: the remaining failure class is keyword-as-topic-word in multi-word descriptive filenames. Fix: split the generic keyword patterns out of _SENSITIVE_PATTERNS and only count a match when the keyword is load-bearing in the name: - the keyword ends the stem (api_token.txt, github-personal-access-token.txt, oauth_token.json) - secret stores name their contents, and the content noun is the head of the compound, which comes last; or - the stem has <= 2 words (token.txt, token_config.yaml, secret_handler.txt). A keyword buried mid-phrase in a >= 3-word slug is a topic word, not a credential store. Specific patterns (.pem/.env/id_rsa/.netrc/aws_credentials) are unchanged and still always apply. All existing contracts preserved: api_token.txt, oauth_token.json, token.txt, token_config.yaml, secret_handler.txt, passwords.py, credentials.json still flagged; tokenizer.py / tokenize.py still clean. Adds 6 regression tests including dotfile (.token), plural (tokens.txt), and multi-word-keyword (my_private_key.txt) edge cases. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/detect.py | 56 +++++++++++++++++++++++++++++++++++++------- tests/test_detect.py | 27 +++++++++++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index cb6964c91..921661d2e 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -98,23 +98,60 @@ def _zip_within_caps(path: Path) -> bool: ".ssh", ".gnupg", ".aws", ".gcloud", "secrets", ".secrets", "credentials", }) -# Files that may contain secrets - skip silently. +# Files that may contain secrets - skip silently. These patterns are specific +# (extensions, exact credential-store names) and always apply. +_SENSITIVE_PATTERNS = [ + re.compile(r'(^|[\\/])\.(env|envrc)(\.|$)', re.IGNORECASE), + re.compile(r'\.(pem|key|p12|pfx|cert|crt|der|p8)$', re.IGNORECASE), + re.compile(r'(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$'), + re.compile(r'(\.netrc|\.pgpass|\.htpasswd)$', re.IGNORECASE), + re.compile(r'(aws_credentials|gcloud_credentials|service.account)', re.IGNORECASE), +] + +# Generic keyword patterns - these only count when the keyword is LOAD-BEARING +# in the filename (see _generic_keyword_hit), because a keyword buried mid-phrase +# in a long descriptive slug names a topic, not a credential store: +# "token-economics-of-recall.md" is a note ABOUT tokens; "api_token.txt" IS one. # Uses lookarounds instead of \b so underscore-prefixed names like api_token.txt # match. Both patterns use (?![a-zA-Z]) so that the trailing-underscore behavior # is consistent: "secret_store.txt" IS flagged, "tokenizer.py" is NOT (because # "i" after "token" is alpha and blocks the match). # `token` is kept separate because its longer suffix "izer"/"ize" is the only # common false-positive; other keywords have no such well-known derivatives. -_SENSITIVE_PATTERNS = [ - re.compile(r'(^|[\\/])\.(env|envrc)(\.|$)', re.IGNORECASE), - re.compile(r'\.(pem|key|p12|pfx|cert|crt|der|p8)$', re.IGNORECASE), +_GENERIC_KEYWORD_PATTERNS = [ re.compile(r'(? bool: + """True if a generic secret keyword appears load-bearing in the filename. + + Secret-store files name their contents, and in English compounds the + content noun is the head, which comes last: "github-personal-access-token", + "api_token", "oauth_token". A keyword that is neither at the end of the + stem nor in a short (<=2 word) name is a topic word in a descriptive slug + ("token-economics-of-recall.md", "password-policy-discussion.md") and must + not cause the file to be silently dropped from the graph (#436, #718). + """ + # Stem = name up to the first dot, ignoring leading dots so dotfiles like + # ".token" keep their keyword ("" stems would never match). + stem = name.lstrip('.').split('.')[0] + for pat in _GENERIC_KEYWORD_PATTERNS: + hit = False + for m in pat.finditer(stem): + hit = True + if m.end() == len(stem): # keyword ends the stem -> names the contents + return True + if hit and len([w for w in _WORD_SPLIT.split(stem) if w]) <= 2: + return True # short name like token_config.yaml / secret_handler.txt + return False + # Signals that a .md/.txt file is actually a converted academic paper _PAPER_SIGNALS = [ re.compile(r'\barxiv\b', re.IGNORECASE), @@ -143,7 +180,10 @@ def _is_sensitive(path: Path) -> bool: return True # Stage 2: filename pattern match name = path.name - return any(p.search(name) for p in _SENSITIVE_PATTERNS) + if any(p.search(name) for p in _SENSITIVE_PATTERNS): + return True + # Stage 3: generic keywords, only when load-bearing in the name + return _generic_keyword_hit(name) def _looks_like_paper(path: Path) -> bool: diff --git a/tests/test_detect.py b/tests/test_detect.py index ee5ac1d69..395a9d4d7 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -638,6 +638,33 @@ def test_sensitive_token_config_yaml(): assert _is_sensitive(Path("token_config.yaml")) +# ── Generic keywords must be load-bearing: topic slugs are not secret stores ── +# A keyword buried mid-phrase in a >=3-word descriptive name is a note ABOUT +# the topic, not a credential file. It must not be silently dropped. + +def test_sensitive_does_not_flag_token_economics_note(): + assert not _is_sensitive(Path("token-economics-of-recall.md")) + +def test_sensitive_does_not_flag_password_policy_discussion(): + assert not _is_sensitive(Path("password-policy-discussion.md")) + +def test_sensitive_flags_keyword_at_end_of_long_name(): + # Keyword as the final word names the file's contents — still a secret store. + assert _is_sensitive(Path("github-personal-access-token.txt")) + +def test_sensitive_flags_my_private_key_txt(): + # Multi-word keyword at end of stem (end-of-stem check runs before word + # counting, so splitting private_key on "_" cannot un-flag it). + assert _is_sensitive(Path("my_private_key.txt")) + +def test_sensitive_flags_dotfile_token(): + # Leading dot stripped before stem extraction; ".token" keeps its keyword. + assert _is_sensitive(Path(".token")) + +def test_sensitive_flags_plural_tokens_txt(): + assert _is_sensitive(Path("tokens.txt")) + + # ── Issue #933: failed-chunk files must not be frozen in manifest ───────────── def test_save_manifest_skips_semantic_hash_for_files_without_cache(tmp_path):