diff --git a/.jules/bolt.md b/.jules/bolt.md index a86b7aafd..4474121b0 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -43,3 +43,10 @@ ## 2026-07-09 - Avoid N+1 API blocking in SBOM aggregator **Learning:** The `collect_inventories` function in `scripts/ci/sbom_inventory_aggregator.py` was fetching SBOMs from the GitHub dependency graph synchronously for every repository in the organization. For large organizations (up to 500 repos), this N+1 network/CLI bottleneck significantly stalled the aggregation workflow. **Action:** Use `concurrent.futures.ThreadPoolExecutor` to fetch SBOMs concurrently when multiple repositories are provided, bounded by a `max_workers` limit (e.g., 10) to avoid overwhelming the CLI/API, while preserving the fast serial path for single-item inputs. +## 2026-08-02 - Combine Multiple Token Regexes using the OR Operator (`|`) +**Learning:** Found a performance bottleneck in `scripts/ci/redact_sensitive_log.py` where a tuple of regexes (`PROVIDER_TOKEN_RES`) was executed iteratively over the exact same block of text inside a loop (`for pattern in PROVIDER_TOKEN_RES: cleaned = pattern.sub(REDACTED, cleaned)`). This requires parsing the string multiple times, imposing linear time overhead equivalent to the number of rules. +**Action:** Always combine mutually exclusive string token replacements on the same text fragment into a single module-level compiled pattern using the regex alternation operator (`|`), yielding a single optimized regex engine pass (`PROVIDER_TOKEN_RE = re.compile(r"A|B|C|D")`). + +## 2026-08-02 - Slice-based buffer accumulation over char-by-char iteration +**Learning:** Appending characters individually to a list in a `while` loop over strings (`output.append(text[cursor])`) is extremely inefficient (O(n) per character plus Python function call overhead) in parsing and redaction scripts. +**Action:** Track index offsets and slice the underlying string into contiguous chunks of unmatched segments, appending the whole slice to the output buffer instead (`output.append(text[start:cursor])`). diff --git a/scripts/ci/redact_sensitive_log.py b/scripts/ci/redact_sensitive_log.py index cb89fe67b..f8c3205b1 100644 --- a/scripts/ci/redact_sensitive_log.py +++ b/scripts/ci/redact_sensitive_log.py @@ -24,11 +24,11 @@ r"[^\s\"'\\]+", re.IGNORECASE, ) -PROVIDER_TOKEN_RES = ( - re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b"), - re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"), - re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"), - re.compile(r"\bAKIA[0-9A-Z]{16}\b"), +PROVIDER_TOKEN_RE = re.compile( + r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b|" + r"\bsk-[A-Za-z0-9_-]{20,}\b|" + r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b|" + r"\bAKIA[0-9A-Z]{16}\b" ) @@ -99,14 +99,19 @@ def _redact_assignments(text: str) -> str: """Redact sensitive key/value assignments without backtracking regexes.""" output: list[str] = [] cursor = 0 + start = 0 while cursor < len(text): match = _consume_sensitive_assignment(text, cursor) if match is None: - output.append(text[cursor]) cursor += 1 continue + if cursor > start: + output.append(text[start:cursor]) replacement, cursor = match output.append(replacement) + start = cursor + if cursor > start: + output.append(text[start:cursor]) return "".join(output) @@ -115,8 +120,7 @@ def _redact_unstructured(text: str) -> str: cleaned = _redact_assignments(text) cleaned = BEARER_RE.sub(lambda match: f"{match.group('prefix')}{REDACTED}", cleaned) cleaned = JWT_RE.sub(REDACTED, cleaned) - for pattern in PROVIDER_TOKEN_RES: - cleaned = pattern.sub(REDACTED, cleaned) + cleaned = PROVIDER_TOKEN_RE.sub(REDACTED, cleaned) return cleaned