Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .jules/bolt.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,6 @@
## 2026-07-09 - Avoid N+1 API blocking in SBOM aggregator
**Learning:** The `collect_inventories` function in `scripts/ci/sbom_inventory_aggregator.py` was fetching SBOMs from the GitHub dependency graph synchronously for every repository in the organization. For large organizations (up to 500 repos), this N+1 network/CLI bottleneck significantly stalled the aggregation workflow.
**Action:** Use `concurrent.futures.ThreadPoolExecutor` to fetch SBOMs concurrently when multiple repositories are provided, bounded by a `max_workers` limit (e.g., 10) to avoid overwhelming the CLI/API, while preserving the fast serial path for single-item inputs.
## 2024-07-25 - Avoid O(N^2) String Append in Parsing Loops
**Learning:** Found an O(N^2) bottleneck in `scripts/ci/redact_sensitive_log.py`'s `_redact_assignments` function where non-matching text chunks were being appended character-by-character (`output.append(text[cursor])`) inside a `while cursor < len(text)` loop. When combined with a `_consume_sensitive_assignment` function that only advanced by 1 character on failures, processing 1 million characters took over 400 seconds.
**Action:** When parsing large text strings, modify matchers to return the next valid fast-forward cursor instead of jumping by 1. Use string slicing (`output.append(text[last_append:cursor])`) to batch-append non-matching blocks in O(1) steps instead of O(N^2) character iteration.
31 changes: 20 additions & 11 deletions scripts/ci/redact_sensitive_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def _redact_json(value: Any) -> Any:
return value


def _consume_sensitive_assignment(text: str, start: int) -> tuple[str, int] | None:
def _consume_sensitive_assignment(text: str, start: int) -> tuple[str, int] | tuple[None, int]:
"""Return a redacted key/value assignment parsed in linear time."""
cursor = start
key_quote = ""
Expand All @@ -53,25 +53,25 @@ def _consume_sensitive_assignment(text: str, start: int) -> tuple[str, int] | No
cursor += 1
key_start = cursor
if cursor >= len(text) or text[cursor] not in KEY_CHARS or text[cursor].isdigit():
return None
return None, start + 1
while cursor < len(text) and text[cursor] in KEY_CHARS:
cursor += 1
key = text[key_start:cursor]
if key_quote:
if cursor >= len(text) or text[cursor] != key_quote:
return None
return None, start + 1
cursor += 1
if not SENSITIVE_KEY_RE.search(key):
return None
return None, cursor
while cursor < len(text) and text[cursor].isspace():
cursor += 1
if cursor >= len(text) or text[cursor] not in ":=":
return None
return None, cursor
cursor += 1
while cursor < len(text) and text[cursor].isspace():
cursor += 1
if cursor >= len(text):
return None
return None, cursor

value_start = cursor
if text[cursor] in "\"'":
Expand All @@ -91,22 +91,31 @@ def _consume_sensitive_assignment(text: str, start: int) -> tuple[str, int] | No
while cursor < len(text) and not text[cursor].isspace() and text[cursor] not in ",}":
cursor += 1
if cursor == value_start:
return None
return None, start + 1
return text[start:value_start] + REDACTED, cursor


def _redact_assignments(text: str) -> str:
"""Redact sensitive key/value assignments without backtracking regexes."""
output: list[str] = []
cursor = 0
last_append = 0
while cursor < len(text):
match = _consume_sensitive_assignment(text, cursor)
if match is None:
output.append(text[cursor])
cursor += 1
if match[0] is None:
next_cursor = match[1]
if next_cursor <= cursor:
next_cursor = cursor + 1
cursor = next_cursor
continue
replacement, cursor = match
replacement, next_cursor = match
if cursor > last_append:
output.append(text[last_append:cursor])
output.append(replacement)
cursor = next_cursor
last_append = cursor
if last_append < len(text):
output.append(text[last_append:])
return "".join(output)


Expand Down
Loading