diff --git a/.jules/bolt.md b/.jules/bolt.md index a86b7aafd..c420a0a9e 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -43,3 +43,9 @@ ## 2026-07-09 - Avoid N+1 API blocking in SBOM aggregator **Learning:** The `collect_inventories` function in `scripts/ci/sbom_inventory_aggregator.py` was fetching SBOMs from the GitHub dependency graph synchronously for every repository in the organization. For large organizations (up to 500 repos), this N+1 network/CLI bottleneck significantly stalled the aggregation workflow. **Action:** Use `concurrent.futures.ThreadPoolExecutor` to fetch SBOMs concurrently when multiple repositories are provided, bounded by a `max_workers` limit (e.g., 10) to avoid overwhelming the CLI/API, while preserving the fast serial path for single-item inputs. +## 2026-07-08 - [Log Redaction Performance Optimization] +**Learning:** O(N^2) complexity in Python string manipulation (character-by-character append) and multiple regex substitute passes significantly degrade performance when processing massive CI logs. +**Action:** When implementing parsing or string manipulation in tight linear loops, use string slicing to batch-process chunks (`output.append(text[start:cursor])`). When applying multiple unrelated regex replacements to the same string fragment, combine them using the `|` alternation operator into a single compiled pattern to prevent redundant parsing over the text. +## 2026-08-05 - [Regex Algorithmic Complexity Issue Fix] +**Learning:** An unbounded while loop processing input characters to match a pattern inside a string parser can lead to O(N^2) complexity if the parser fails after matching a very long invalid string and falls back to check the next index. This can trigger timeouts in CI jobs on adversarial input logs. +**Action:** Always add an upper bound (e.g., `MAX_KEY_LENGTH`) in linear parsers when scanning for known patterns in untrusted input. diff --git a/scripts/ci/redact_sensitive_log.py b/scripts/ci/redact_sensitive_log.py index cb89fe67b..af09f948c 100644 --- a/scripts/ci/redact_sensitive_log.py +++ b/scripts/ci/redact_sensitive_log.py @@ -9,6 +9,7 @@ from typing import Any REDACTED = "[REDACTED]" +MAX_KEY_LENGTH = 64 KEY_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-") SENSITIVE_KEY_RE = re.compile( r"(?:token|secret|password|passwd|credential|authorization|jwt|" @@ -24,11 +25,12 @@ r"[^\s\"'\\]+", re.IGNORECASE, ) -PROVIDER_TOKEN_RES = ( - re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b"), - re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"), - re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"), - re.compile(r"\bAKIA[0-9A-Z]{16}\b"), +# Optimizing: using `|` alternation operator to combine regexes into a single compiled pattern to avoid redundant passes over the string +PROVIDER_TOKEN_RE = re.compile( + r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b|" + r"\bsk-[A-Za-z0-9_-]{20,}\b|" + r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b|" + r"\bAKIA[0-9A-Z]{16}\b" ) @@ -54,7 +56,7 @@ def _consume_sensitive_assignment(text: str, start: int) -> tuple[str, int] | No key_start = cursor if cursor >= len(text) or text[cursor] not in KEY_CHARS or text[cursor].isdigit(): return None - while cursor < len(text) and text[cursor] in KEY_CHARS: + while cursor < len(text) and text[cursor] in KEY_CHARS and (cursor - key_start) < MAX_KEY_LENGTH: cursor += 1 key = text[key_start:cursor] if key_quote: @@ -99,14 +101,19 @@ def _redact_assignments(text: str) -> str: """Redact sensitive key/value assignments without backtracking regexes.""" output: list[str] = [] cursor = 0 + start = 0 while cursor < len(text): match = _consume_sensitive_assignment(text, cursor) if match is None: - output.append(text[cursor]) cursor += 1 continue + if cursor > start: + output.append(text[start:cursor]) replacement, cursor = match output.append(replacement) + start = cursor + if cursor > start: + output.append(text[start:cursor]) return "".join(output) @@ -115,8 +122,7 @@ def _redact_unstructured(text: str) -> str: cleaned = _redact_assignments(text) cleaned = BEARER_RE.sub(lambda match: f"{match.group('prefix')}{REDACTED}", cleaned) cleaned = JWT_RE.sub(REDACTED, cleaned) - for pattern in PROVIDER_TOKEN_RES: - cleaned = pattern.sub(REDACTED, cleaned) + cleaned = PROVIDER_TOKEN_RE.sub(REDACTED, cleaned) return cleaned