optimize(detection-analysis-report): ~45 AIC/run savings via prompt compaction and script extraction

.github/scripts/detection_comparison.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Generate detection feature comparison chart from pre-computed metrics JSON.
+
+Usage:
+    python3 detection_comparison.py --metrics <path-to-metrics.json> --output <output.png>
+
+Input JSON keys:
+    regular_runs, detection_runs, regular_success_rate, detection_success_rate,
+    regular_failure_count, detection_failure_count, misconfigured_count
+"""
+
+import argparse
+import json
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import seaborn as sns
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Generate detection feature comparison chart")
+    parser.add_argument("--metrics", required=True, help="Path to metrics JSON file")
+    parser.add_argument("--output", required=True, help="Output PNG file path")
+    args = parser.parse_args()
+
+    with open(args.metrics) as f:
+        m = json.load(f)
+
+    regular_runs = int(m.get("regular_runs", 0))
+    detection_runs = int(m.get("detection_runs", 0))
+    regular_success_rate = float(m.get("regular_success_rate", 0.0))
+    detection_success_rate = float(m.get("detection_success_rate", 0.0))
+    regular_failure = int(m.get("regular_failure_count", 0))
+    detection_failure = int(m.get("detection_failure_count", 0))
+    misconfigured_count = int(m.get("misconfigured_count", 0))
+
+    regular_success = regular_runs - regular_failure
+    detection_success = detection_runs - detection_failure
+
+    sns.set_style("whitegrid")
+    palette = sns.color_palette("muted")
+
+    fig, ax1 = plt.subplots(figsize=(10, 6), dpi=150)
+    ax2 = ax1.twinx()
+
+    x = np.array([0, 1])
+    width = 0.5
+
+    # Stacked bars: success (bottom) + failure (top)
+    ax1.bar(
+        x, [regular_success, detection_success], width,
+        label="Success", color=palette[2], alpha=0.85,
+    )
+    ax1.bar(
+        x, [regular_failure, detection_failure], width,
+        bottom=[regular_success, detection_success],
+        label="Failure", color=palette[3], alpha=0.85,
+    )
+
+    # Success-rate line on secondary axis
+    ax2.plot(
+        x, [regular_success_rate, detection_success_rate],
+        color=palette[0], marker="o", linewidth=2, markersize=8,
+        label="Success Rate %",
+    )
+
+    # Annotation band when misconfigured workflows are present
+    if misconfigured_count > 0:
+        max_y = max(regular_runs, detection_runs, 1)
+        ax1.axhspan(
+            0, max_y * 0.12, alpha=0.12, color="red",
+            label=f"{misconfigured_count} misconfigured workflow(s)",
+        )
+
+    ax1.set_xticks(x)
+    ax1.set_xticklabels(["Regular Runs", "Detection Runs"], fontsize=12)
+    ax1.set_ylabel("Run Count", fontsize=12)
+    ax1.set_title("Detection Feature Comparison — Last 24h", fontsize=14, fontweight="bold")
+    ax2.set_ylabel("Success Rate (%)", fontsize=12)
+    ax2.set_ylim(0, 110)
+
+    handles1, labels1 = ax1.get_legend_handles_labels()
+    handles2, labels2 = ax2.get_legend_handles_labels()
+    ax1.legend(handles1 + handles2, labels1 + labels2, loc="upper right", fontsize=10)
+
+    plt.tight_layout()
+    os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
+    plt.savefig(args.output, dpi=150, bbox_inches="tight")
+    print(f"Chart saved to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/detection-analysis-report.md

@@ -64,87 +64,57 @@ Upload the chart using the `upload_asset` safe-output tool with the absolute pat
 
 ### Step 1 — Fetch Logs
 
-Use the `agentic-workflows` MCP `logs` tool to download workflow runs from the last 24 hours:
+Call `agenticworkflows logs --start-date -1d` to download workflow run directories to `/tmp/gh-aw/aw-mcp/logs`. Each run's `aw_info.json` contains `features["gh-aw-detection"]` (boolean), `status`, `total_tokens`, and `engine_id`. Use `features.gh-aw-detection` directly — do not infer detection status from `.lock.yml` files.
 
-```
-Use the agentic-workflows MCP tool `logs` with parameters:
-- start_date: "-1d"
-Output is saved to: /tmp/gh-aw/aw-mcp/logs
-```
-
-Each run directory contains `aw_info.json` with fields including `engine_id`, `workflow`, `status`, `tokens`, and feature flags. The `gh-aw-detection` feature flag is stored under the `features` key in `aw_info.json` (e.g., `features.gh-aw-detection: true`). Use this field directly — do not infer detection status by scanning `.lock.yml` files.
-
-### Step 2 — Classify Runs
+### Analyze Runs
 
-For each run, classify it as:
-- **Detection-enabled**: `features.gh-aw-detection` is `true` in the run metadata
-- **Regular**: `features.gh-aw-detection` is `false`, absent, or unset
+In a single pass over the run directories at `/tmp/gh-aw/aw-mcp/logs`, classify and aggregate all runs:
 
-Collect per-run:
-- `workflow_name`
-- `status` (success / failure / cancelled / timed_out)
-- `total_tokens` (from `aw_info.json`)
-- `engine_id`
-- `detection_enabled` (boolean)
+**Classify each run** using `features.gh-aw-detection` from `aw_info.json`:
+- **Detection-enabled**: value is `true`
+- **Regular**: value is `false`, absent, or unset
 
-### Step 3 — Identify Misconfigured Workflows
+Collect per-run: `workflow_name`, `status`, `total_tokens`, `engine_id`, `detection_enabled`.
 
-Flag a workflow as **misconfigured** when any of the following apply:
+**Flag a workflow as misconfigured** when any of the following apply:
+1. `gh-aw-detection: false` on a workflow with >3 total runs in the last 7 days
+2. Workflow name contains `audit`, `analyzer`, `report`, `detector`, `monitor`, or `inspector` but lacks `gh-aw-detection: true`
+3. Run has `gh-aw-detection: true` but detection-related steps failed
+4. Workflow alternates between detection-enabled and detection-disabled within the 24h window
 
-1. **Detection explicitly disabled on an active workflow**: `gh-aw-detection: false` on a workflow that has completed more than 3 total runs (any status) in the last 7 days.
-2. **Detection feature absent on a workflow that is a known audit, analysis, or report workflow**: The workflow name contains keywords such as `audit`, `analyzer`, `report`, `detector`, `monitor`, or `inspector`, but lacks `gh-aw-detection: true`.
-3. **Detection job failures**: The run has `gh-aw-detection: true` but its detection-related job step failed or produced errors (check agent logs for detection-related error patterns).
-4. **Inconsistent detection state**: A workflow alternates between detection-enabled and detection-disabled runs within the same 24-hour window.
+For each misconfigured workflow, record: `workflow_name`, `misconfiguration_type`, `run_count`, `example_run_id`, `recommended_fix`.
 
-For each misconfigured workflow, record:
-- `workflow_name`
-- `misconfiguration_type` (one of the four above)
-- `run_count`
-- `example_run_id`
-- `recommended_fix`
+**Save** aggregated metrics to `/tmp/gh-aw/python/data/metrics.json`:
 
-### Step 4 — Compute Metrics
-
-Aggregate metrics for the chart and report:
-
-| Metric | Regular Runs | Detection Runs |
-|--------|-------------|----------------|
-| Total runs | count | count |
-| Success rate (%) | % | % |
-| Avg tokens | mean | mean |
-| Failure count | count | count |
-| Misconfigured count | — | count |
+```json
+{
+  "regular_runs": <count>,
+  "detection_runs": <count>,
+  "regular_success_rate": <percent>,
+  "detection_success_rate": <percent>,
+  "regular_failure_count": <count>,
+  "detection_failure_count": <count>,
+  "regular_avg_tokens": <mean>,
+  "detection_avg_tokens": <mean>,
+  "misconfigured_count": <count>
+}
+```
 
 ### Step 5 — Generate Chart
 
-Write a Python script to `/tmp/gh-aw/python/detection_comparison.py` that:
-
-- Reads aggregated metrics from `/tmp/gh-aw/python/data/metrics.json`
-- Produces a grouped bar chart comparing Regular vs. Detection-enabled runs:
-  - Left y-axis: run counts (stacked success/failure bars)
-  - Right y-axis: success rate % (line with markers)
-  - X-axis labels: "Regular Runs", "Detection Runs"
-  - Title: "Detection Feature Comparison — Last 24h"
-  - A horizontal annotation band highlighting any workflows with detection misconfigurations
-- Saves the chart to `/tmp/gh-aw/python/charts/detection_comparison.png` at 150 DPI
-- Uses a clean style (seaborn `whitegrid`, muted color palette)
-
-### Step 6 — Historical Trending
-
-Append today's aggregated metrics to the cache-memory store so future runs can draw a trend line. Create the directory before writing if it does not exist:
-
 ```bash
-mkdir -p /tmp/gh-aw/cache-memory/trending/detection-metrics
+python3 .github/scripts/detection_comparison.py \
+  --metrics /tmp/gh-aw/python/data/metrics.json \
+  --output /tmp/gh-aw/python/charts/detection_comparison.png
 ```
 
-- File: `/tmp/gh-aw/cache-memory/trending/detection-metrics/history.jsonl`
-- Fields: `timestamp`, `regular_runs`, `detection_runs`, `regular_success_rate`, `detection_success_rate`, `regular_avg_tokens`, `detection_avg_tokens`, `misconfigured_count`
+Upload the chart using the `upload_asset` safe-output tool. Record the returned asset URL and embed it in the discussion body.
+
+### Step 6 — Historical Trending
 
-If history data exists (at least 7 days), generate a second trend chart:
+Use the `update-detection-trending` agent to append today's metrics to the cache-memory trending history and optionally generate a trend chart.
 
-- **Detection Adoption Over Time**: Line chart showing `regular_runs` vs. `detection_runs` per day for the last 30 days, with a shaded area for the detection success rate band.
-- Save to: `/tmp/gh-aw/python/charts/detection_trend.png`
-- Upload with `upload_asset` and include in the report.
+If `/tmp/gh-aw/python/charts/detection_trend.png` was generated, upload it with `upload_asset` and embed in the report under "View Historical Trend".
 
 ---
 
@@ -198,4 +168,30 @@ Call `noop` with a brief explanation when:
 - No workflow runs exist in the last 24 hours
 - State the evaluated window in the no-op message
 
-Example: `noop("No workflow runs found in the last 24 full hours (YYYY-MM-DDTHH:MM:SSZ to YYYY-MM-DDTHH:MM:SSZ)")`
+Example: `noop("No workflow runs found in the last 24 full hours (YYYY-MM-DDTHH:MM:SSZ to YYYY-MM-DDTHH:MM:SSZ)")`
+
+---
+
+## agent: `update-detection-trending`
+---
+model: small
+description: Appends detection metrics to cache-memory trending history and generates a trend chart when ≥7 days of data exist
+---
+Append today's detection metrics to the cache-memory trending history store.
+
+Create the directory before writing:
+
+```bash
+mkdir -p /tmp/gh-aw/cache-memory/trending/detection-metrics
+```
+
+Read `/tmp/gh-aw/python/data/metrics.json` and append one JSON line to `/tmp/gh-aw/cache-memory/trending/detection-metrics/history.jsonl`:
+
+```json
+{"timestamp": "<ISO-8601 UTC>", "regular_runs": N, "detection_runs": N, "regular_success_rate": X.X, "detection_success_rate": X.X, "regular_avg_tokens": X, "detection_avg_tokens": X, "misconfigured_count": N}
+```
+
+If the history file has ≥7 entries, generate a 30-day trend chart at `/tmp/gh-aw/python/charts/detection_trend.png`:
+- Line chart: `regular_runs` vs. `detection_runs` per day for the last 30 days
+- Shaded area band for the detection success rate
+- Style: seaborn whitegrid, 12×7 inches, 150 DPI