diff --git a/.jules/bolt.md b/.jules/bolt.md index a86b7aafd..8b6e9593f 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -43,3 +43,6 @@ ## 2026-07-09 - Avoid N+1 API blocking in SBOM aggregator **Learning:** The `collect_inventories` function in `scripts/ci/sbom_inventory_aggregator.py` was fetching SBOMs from the GitHub dependency graph synchronously for every repository in the organization. For large organizations (up to 500 repos), this N+1 network/CLI bottleneck significantly stalled the aggregation workflow. **Action:** Use `concurrent.futures.ThreadPoolExecutor` to fetch SBOMs concurrently when multiple repositories are provided, bounded by a `max_workers` limit (e.g., 10) to avoid overwhelming the CLI/API, while preserving the fast serial path for single-item inputs. +## 2026-07-28 - Fast-Path Chunking for Linear Text Scanners +**Learning:** In linear parsing functions like `_redact_assignments` that iterate char-by-char, appending a single character to a list and continuing the loop causes measurable O(N^2) overhead due to tight loop overhead and high volume of list append ops. Scanning ahead and appending non-matching text as single string slices (e.g. `output.append(text[start:cursor])`) substantially reduces this overhead and provides a fast path. Also, compiling multiple token matching regexes into a single combined regex using `|` eliminates redundant function calls for each text fragment. +**Action:** When writing linear text scanners, batch unmatched text using an inner loop and string slicing rather than appending character-by-character. Combine multiple regex replacements acting on the same text into a single regex with `|` whenever feasible. diff --git a/scripts/ci/redact_sensitive_log.py b/scripts/ci/redact_sensitive_log.py index cb89fe67b..793163d7d 100644 --- a/scripts/ci/redact_sensitive_log.py +++ b/scripts/ci/redact_sensitive_log.py @@ -24,11 +24,11 @@ r"[^\s\"'\\]+", re.IGNORECASE, ) -PROVIDER_TOKEN_RES = ( - re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b"), - re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"), - re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"), - re.compile(r"\bAKIA[0-9A-Z]{16}\b"), +PROVIDER_TOKEN_RE = re.compile( + r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|" + r"sk-[A-Za-z0-9_-]{20,}|" + r"xox[baprs]-[A-Za-z0-9-]{20,}|" + r"AKIA[0-9A-Z]{16})\b" ) @@ -99,14 +99,21 @@ def _redact_assignments(text: str) -> str: """Redact sensitive key/value assignments without backtracking regexes.""" output: list[str] = [] cursor = 0 - while cursor < len(text): + length = len(text) + while cursor < length: match = _consume_sensitive_assignment(text, cursor) if match is None: - output.append(text[cursor]) + start_cursor = cursor cursor += 1 - continue - replacement, cursor = match - output.append(replacement) + while cursor < length: + match = _consume_sensitive_assignment(text, cursor) + if match is not None: + break + cursor += 1 + output.append(text[start_cursor:cursor]) + if match is not None: + replacement, cursor = match + output.append(replacement) return "".join(output) @@ -115,8 +122,7 @@ def _redact_unstructured(text: str) -> str: cleaned = _redact_assignments(text) cleaned = BEARER_RE.sub(lambda match: f"{match.group('prefix')}{REDACTED}", cleaned) cleaned = JWT_RE.sub(REDACTED, cleaned) - for pattern in PROVIDER_TOKEN_RES: - cleaned = pattern.sub(REDACTED, cleaned) + cleaned = PROVIDER_TOKEN_RE.sub(REDACTED, cleaned) return cleaned