From f79a558c85075f9c42ab46f087605594d214e475 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:08:26 +0000 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9A=A1=20Bolt:=20Optimize=20CI=20log=20r?= =?UTF-8?q?edaction=20string=20processing=20and=20regex=20replacements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Refactor `_redact_assignments` character-by-character append to string slicing, eliminating O(N^2) memory reallocation inside the parsing loop. - Combine four `PROVIDER_TOKEN_RES` patterns into a single `PROVIDER_TOKEN_RE` compiled regex using the alternation (`|`) operator to remove redundant full-string passes. - Documentation added to `.jules/bolt.md`. --- .jules/bolt.md | 3 +++ scripts/ci/redact_sensitive_log.py | 21 +++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.jules/bolt.md b/.jules/bolt.md index a86b7aafd..a61977fb7 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -43,3 +43,6 @@ ## 2026-07-09 - Avoid N+1 API blocking in SBOM aggregator **Learning:** The `collect_inventories` function in `scripts/ci/sbom_inventory_aggregator.py` was fetching SBOMs from the GitHub dependency graph synchronously for every repository in the organization. For large organizations (up to 500 repos), this N+1 network/CLI bottleneck significantly stalled the aggregation workflow. **Action:** Use `concurrent.futures.ThreadPoolExecutor` to fetch SBOMs concurrently when multiple repositories are provided, bounded by a `max_workers` limit (e.g., 10) to avoid overwhelming the CLI/API, while preserving the fast serial path for single-item inputs. +## 2026-07-08 - [Log Redaction Performance Optimization] +**Learning:** O(N^2) complexity in Python string manipulation (character-by-character append) and multiple regex substitute passes significantly degrade performance when processing massive CI logs. +**Action:** When implementing parsing or string manipulation in tight linear loops, use string slicing to batch-process chunks (`output.append(text[start:cursor])`). When applying multiple unrelated regex replacements to the same string fragment, combine them using the `|` alternation operator into a single compiled pattern to prevent redundant parsing over the text. diff --git a/scripts/ci/redact_sensitive_log.py b/scripts/ci/redact_sensitive_log.py index cb89fe67b..84469d21a 100644 --- a/scripts/ci/redact_sensitive_log.py +++ b/scripts/ci/redact_sensitive_log.py @@ -24,11 +24,12 @@ r"[^\s\"'\\]+", re.IGNORECASE, ) -PROVIDER_TOKEN_RES = ( - re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b"), - re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"), - re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"), - re.compile(r"\bAKIA[0-9A-Z]{16}\b"), +# Optimizing: using `|` alternation operator to combine regexes into a single compiled pattern to avoid redundant passes over the string +PROVIDER_TOKEN_RE = re.compile( + r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b|" + r"\bsk-[A-Za-z0-9_-]{20,}\b|" + r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b|" + r"\bAKIA[0-9A-Z]{16}\b" ) @@ -99,14 +100,19 @@ def _redact_assignments(text: str) -> str: """Redact sensitive key/value assignments without backtracking regexes.""" output: list[str] = [] cursor = 0 + start = 0 while cursor < len(text): match = _consume_sensitive_assignment(text, cursor) if match is None: - output.append(text[cursor]) cursor += 1 continue + if cursor > start: + output.append(text[start:cursor]) replacement, cursor = match output.append(replacement) + start = cursor + if cursor > start: + output.append(text[start:cursor]) return "".join(output) @@ -115,8 +121,7 @@ def _redact_unstructured(text: str) -> str: cleaned = _redact_assignments(text) cleaned = BEARER_RE.sub(lambda match: f"{match.group('prefix')}{REDACTED}", cleaned) cleaned = JWT_RE.sub(REDACTED, cleaned) - for pattern in PROVIDER_TOKEN_RES: - cleaned = pattern.sub(REDACTED, cleaned) + cleaned = PROVIDER_TOKEN_RE.sub(REDACTED, cleaned) return cleaned From 26e92d3eb40404e82b9c2229505d16c47b329875 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:35:40 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9A=A1=20Bolt:=20Fix=20algorithmic=20com?= =?UTF-8?q?plexity=20DoS=20in=20log=20redaction=20string=20processing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added a `MAX_KEY_LENGTH` boundary condition to the character-by-character scan loop in `_consume_sensitive_assignment`. - This ensures the parsing algorithm maintains linear time O(N) complexity even when processing maliciously crafted log files containing unbounded alphanumeric sequences. - Updated `.jules/bolt.md` with the new learning. --- .jules/bolt.md | 3 +++ scripts/ci/redact_sensitive_log.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.jules/bolt.md b/.jules/bolt.md index a61977fb7..c420a0a9e 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -46,3 +46,6 @@ ## 2026-07-08 - [Log Redaction Performance Optimization] **Learning:** O(N^2) complexity in Python string manipulation (character-by-character append) and multiple regex substitute passes significantly degrade performance when processing massive CI logs. **Action:** When implementing parsing or string manipulation in tight linear loops, use string slicing to batch-process chunks (`output.append(text[start:cursor])`). When applying multiple unrelated regex replacements to the same string fragment, combine them using the `|` alternation operator into a single compiled pattern to prevent redundant parsing over the text. +## 2026-08-05 - [Regex Algorithmic Complexity Issue Fix] +**Learning:** An unbounded while loop processing input characters to match a pattern inside a string parser can lead to O(N^2) complexity if the parser fails after matching a very long invalid string and falls back to check the next index. This can trigger timeouts in CI jobs on adversarial input logs. +**Action:** Always add an upper bound (e.g., `MAX_KEY_LENGTH`) in linear parsers when scanning for known patterns in untrusted input. diff --git a/scripts/ci/redact_sensitive_log.py b/scripts/ci/redact_sensitive_log.py index 84469d21a..af09f948c 100644 --- a/scripts/ci/redact_sensitive_log.py +++ b/scripts/ci/redact_sensitive_log.py @@ -9,6 +9,7 @@ from typing import Any REDACTED = "[REDACTED]" +MAX_KEY_LENGTH = 64 KEY_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-") SENSITIVE_KEY_RE = re.compile( r"(?:token|secret|password|passwd|credential|authorization|jwt|" @@ -55,7 +56,7 @@ def _consume_sensitive_assignment(text: str, start: int) -> tuple[str, int] | No key_start = cursor if cursor >= len(text) or text[cursor] not in KEY_CHARS or text[cursor].isdigit(): return None - while cursor < len(text) and text[cursor] in KEY_CHARS: + while cursor < len(text) and text[cursor] in KEY_CHARS and (cursor - key_start) < MAX_KEY_LENGTH: cursor += 1 key = text[key_start:cursor] if key_quote: