diff --git a/graphify/detect.py b/graphify/detect.py index cb6964c91..921661d2e 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -98,23 +98,60 @@ def _zip_within_caps(path: Path) -> bool: ".ssh", ".gnupg", ".aws", ".gcloud", "secrets", ".secrets", "credentials", }) -# Files that may contain secrets - skip silently. +# Files that may contain secrets - skip silently. These patterns are specific +# (extensions, exact credential-store names) and always apply. +_SENSITIVE_PATTERNS = [ + re.compile(r'(^|[\\/])\.(env|envrc)(\.|$)', re.IGNORECASE), + re.compile(r'\.(pem|key|p12|pfx|cert|crt|der|p8)$', re.IGNORECASE), + re.compile(r'(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$'), + re.compile(r'(\.netrc|\.pgpass|\.htpasswd)$', re.IGNORECASE), + re.compile(r'(aws_credentials|gcloud_credentials|service.account)', re.IGNORECASE), +] + +# Generic keyword patterns - these only count when the keyword is LOAD-BEARING +# in the filename (see _generic_keyword_hit), because a keyword buried mid-phrase +# in a long descriptive slug names a topic, not a credential store: +# "token-economics-of-recall.md" is a note ABOUT tokens; "api_token.txt" IS one. # Uses lookarounds instead of \b so underscore-prefixed names like api_token.txt # match. Both patterns use (?![a-zA-Z]) so that the trailing-underscore behavior # is consistent: "secret_store.txt" IS flagged, "tokenizer.py" is NOT (because # "i" after "token" is alpha and blocks the match). # `token` is kept separate because its longer suffix "izer"/"ize" is the only # common false-positive; other keywords have no such well-known derivatives. -_SENSITIVE_PATTERNS = [ - re.compile(r'(^|[\\/])\.(env|envrc)(\.|$)', re.IGNORECASE), - re.compile(r'\.(pem|key|p12|pfx|cert|crt|der|p8)$', re.IGNORECASE), +_GENERIC_KEYWORD_PATTERNS = [ re.compile(r'(? bool: + """True if a generic secret keyword appears load-bearing in the filename. + + Secret-store files name their contents, and in English compounds the + content noun is the head, which comes last: "github-personal-access-token", + "api_token", "oauth_token". A keyword that is neither at the end of the + stem nor in a short (<=2 word) name is a topic word in a descriptive slug + ("token-economics-of-recall.md", "password-policy-discussion.md") and must + not cause the file to be silently dropped from the graph (#436, #718). + """ + # Stem = name up to the first dot, ignoring leading dots so dotfiles like + # ".token" keep their keyword ("" stems would never match). + stem = name.lstrip('.').split('.')[0] + for pat in _GENERIC_KEYWORD_PATTERNS: + hit = False + for m in pat.finditer(stem): + hit = True + if m.end() == len(stem): # keyword ends the stem -> names the contents + return True + if hit and len([w for w in _WORD_SPLIT.split(stem) if w]) <= 2: + return True # short name like token_config.yaml / secret_handler.txt + return False + # Signals that a .md/.txt file is actually a converted academic paper _PAPER_SIGNALS = [ re.compile(r'\barxiv\b', re.IGNORECASE), @@ -143,7 +180,10 @@ def _is_sensitive(path: Path) -> bool: return True # Stage 2: filename pattern match name = path.name - return any(p.search(name) for p in _SENSITIVE_PATTERNS) + if any(p.search(name) for p in _SENSITIVE_PATTERNS): + return True + # Stage 3: generic keywords, only when load-bearing in the name + return _generic_keyword_hit(name) def _looks_like_paper(path: Path) -> bool: diff --git a/tests/test_detect.py b/tests/test_detect.py index ee5ac1d69..395a9d4d7 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -638,6 +638,33 @@ def test_sensitive_token_config_yaml(): assert _is_sensitive(Path("token_config.yaml")) +# ── Generic keywords must be load-bearing: topic slugs are not secret stores ── +# A keyword buried mid-phrase in a >=3-word descriptive name is a note ABOUT +# the topic, not a credential file. It must not be silently dropped. + +def test_sensitive_does_not_flag_token_economics_note(): + assert not _is_sensitive(Path("token-economics-of-recall.md")) + +def test_sensitive_does_not_flag_password_policy_discussion(): + assert not _is_sensitive(Path("password-policy-discussion.md")) + +def test_sensitive_flags_keyword_at_end_of_long_name(): + # Keyword as the final word names the file's contents — still a secret store. + assert _is_sensitive(Path("github-personal-access-token.txt")) + +def test_sensitive_flags_my_private_key_txt(): + # Multi-word keyword at end of stem (end-of-stem check runs before word + # counting, so splitting private_key on "_" cannot un-flag it). + assert _is_sensitive(Path("my_private_key.txt")) + +def test_sensitive_flags_dotfile_token(): + # Leading dot stripped before stem extraction; ".token" keeps its keyword. + assert _is_sensitive(Path(".token")) + +def test_sensitive_flags_plural_tokens_txt(): + assert _is_sensitive(Path("tokens.txt")) + + # ── Issue #933: failed-chunk files must not be frozen in manifest ───────────── def test_save_manifest_skips_semantic_hash_for_files_without_cache(tmp_path):