#217: Create find-orchestration-savings skill to identify token waste

packages/agents/content/skills/find-orchestration-savings/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: find-orchestration-savings
+description: Analyze completed orchestrated runs for token waste and resource misallocation
+user-invocable: true
+---
+
+# Find orchestration savings
+
+Analyze a completed orchestrated run to identify cost-saving opportunities while preserving or improving output quality.
+
+## Arguments
+
+- Run directory path (optional): path to the run directory to analyze. If omitted, resolved from session context (most recent run in the current project's artifact directory).
+
+## Process
+
+1. **Resolve run directory:**
+   - If a run directory path is provided, use it.
+   - If a run is active in the current session (run-index.json path known), use that.
+   - Otherwise, use `get-branch-context` to get project_slug and ticket_id, then scan `{artifacts.base_dir}/projects/{project_slug}/tickets/{ticket_id}/` for the most recent completed run (directory with latest timestamp).
+   - If no run found, report "No completed run found for this context" and exit.
+
+2. **Verify the run directory** contains `run-log.jsonl` and `run-index.json`. If either is missing, report and exit.
+
+3. **Dispatch the savings-analyzer subagent** via Task tool:
+   - Model: `haiku`
+   - Prompt: provide the run directory path and the next available sequence number for the artifact filename
+   - The subagent reads the event log, applies the analysis framework, and writes the artifact
+
+4. **Present chat summary:** After the subagent completes, read the artifact and present the top 3 findings in conversation.
+
+## Auto-trigger integration
+
+This skill is also invoked automatically during Phase 5 (summary) of orchestrated runs. The orchestrate engine dispatches the savings-analyzer subagent as a background Task while the orchestrator writes the run-summary inline. When auto-triggered:
+
+- The run directory is known from the active run context
+- The sequence number follows the run-summary artifact
+- The model is Haiku (configured via the `savings_analyzer` model key, defaulting to `haiku`)
+- Zero added latency: the analysis runs as a background Task while the orchestrator writes the run summary

packages/agents/content/skills/orchestrate/SKILL.md

@@ -69,23 +69,27 @@ Invalid model names (e.g., `gpt4`) are rejected by the Task tool at dispatch tim
 | `default`           | `sonnet` |
 | `coder`             | `opus`   |
 | `holistic_reviewer` | `opus`   |
+| `savings_analyzer`  | `haiku`  |
 
 #### Available keys
 
-| Key                              | Maps to                                          |
-| -------------------------------- | ------------------------------------------------ |
-| `default`                        | All agents unless a specific key overrides       |
-| `architect`                      | orchestrated-architect (Phase 1)                 |
-| `planner`                        | orchestrated-planner (Phase 2)                   |
-| `coder`                          | orchestrated-coder (all invocations, all phases) |
-| `reviewer`                       | orchestrated-reviewer core (Phase 4)             |
-| `aspect_code_reviewer`           | aspect-code-reviewer (Phase 4)                   |
-| `aspect_silent_failure_reviewer` | aspect-silent-failure-reviewer (Phase 4)         |
-| `aspect_test_reviewer`           | aspect-test-reviewer (Phase 4)                   |
-| `code_simplifier`                | code-simplifier (Phase 4a)                       |
-| `holistic_reviewer`              | orchestrated-reviewer holistic (Phase 4b)        |
+| Key                              | Maps to                                           |
+| -------------------------------- | ------------------------------------------------- |
+| `default`                        | All agents unless a specific key overrides        |
+| `architect`                      | orchestrated-architect (Phase 1)                  |
+| `planner`                        | orchestrated-planner (Phase 2)                    |
+| `coder`                          | orchestrated-coder (all invocations, all phases)  |
+| `reviewer`                       | orchestrated-reviewer core (Phase 4)              |
+| `aspect_code_reviewer`           | aspect-code-reviewer (Phase 4)                    |
+| `aspect_silent_failure_reviewer` | aspect-silent-failure-reviewer (Phase 4)          |
+| `aspect_test_reviewer`           | aspect-test-reviewer (Phase 4)                    |
+| `code_simplifier`                | code-simplifier (Phase 4a)                        |
+| `holistic_reviewer`              | orchestrated-reviewer holistic (Phase 4b)         |
+| `savings_analyzer`               | savings-analyzer (Phase 5, parallel with summary) |
 
 > **Note:** The `coder` engine default ensures it never falls back to the `default` key. This is intentional — the coder runs in Phases 3, 4, 4a, and 4b, and its model must be consistent across all invocations. To change the coder's model, override the `coder` key explicitly (e.g., `--models=coder:sonnet`); setting `default` alone does not affect it.
+>
+> The `savings_analyzer` engine default (`haiku`) also ignores the `default` key. This is intentional — the savings analyzer is a cost-optimization tool and should always run on the lowest-cost model. To change its model, override the `savings_analyzer` key explicitly (e.g., `--models=savings_analyzer:sonnet`); setting `default` alone does not affect it.
 
 #### Example preferences
 
@@ -100,7 +104,7 @@ orchestration:
     holistic_reviewer: opus
 ```
 
-Keys with engine defaults (`coder`, `holistic_reviewer`) ignore `default` — override them explicitly to change their models.
+Keys with engine defaults (`coder`, `holistic_reviewer`, `savings_analyzer`) ignore `default` — override them explicitly to change their models.
 
 ### Resolving MCP policy
 
@@ -138,6 +142,7 @@ Prefix the status line with a colored emoji for visual distinction:
 | `pr-review-toolkit:pr-test-analyzer`      | 🔬    |
 | `pr-review-toolkit:code-reviewer`         | 🔎    |
 | `pr-review-toolkit:code-simplifier`       | 🙃    |
+| `savings-analyzer`                        | 💰    |
 
 **Example:**
 
@@ -455,6 +460,13 @@ Use `{seq}` to continue artifact sequencing for the Phase 5 run-summary artifact
 
 ## Phase 5: Summary (always)
 
+Dispatch the savings-analyzer subagent as a background Task and immediately proceed to write the run-summary inline (do not wait for the Task to complete before continuing). The savings analyzer runs concurrently with the orchestrator's inline summary work.
+
+- `subagent_type: savings-analyzer`
+- `max_turns: 15`
+- `model: {models.savings_analyzer}` (resolved from the `savings_analyzer` key, defaults to `haiku`)
+- `prompt:` Provide the run directory path (`{run-dir}`) and the next sequence number after the run-summary (`{NN+1}` where `{NN}` is the run-summary sequence number). The subagent will write `{NN+1}_analyst_savings-analysis.md` to the run directory.
+
 Write run-summary artifact to `{run-dir}/{NN}_orchestrator_run-summary.md`:
 
 ```markdown
@@ -517,6 +529,18 @@ Include:
 
 After writing the artifact, call `register_artifact` for the run-summary artifact. Present the same summary to the user in the conversation. The conversational output should match the artifact content — do not abbreviate or omit sections.
 
+After the savings-analyzer Task completes (it runs concurrently and will finish while or after the run-summary is written), call `register_artifact` with:
+
+```
+runDir: {run-dir}
+filename: {NN+1}_analyst_savings-analysis.md
+role: analyst
+roleType: analyst
+agent: savings-analyzer
+type: savings-analysis
+phase: summary
+```
+
 Call MCP tool `complete_run` with `{ runDir: {run-dir}, status: "completed" | "failed" | "needs_manual_review", reason?: string }`. When `status` is `"failed"`, this emits a `run_failed` event (the optional `reason` field is included if provided); otherwise it emits a `run_completed` event. Either way, `completedAt` is stamped on the run-index.json header.
 
 ## Phase 6: Wrap-up (prompted, conditional)

packages/agents/content/subagents/savings-analyzer.md

@@ -0,0 +1,84 @@
+---
+name: savings-analyzer
+description: Analyze an orchestrated run for token waste, unnecessary work, and resource misallocation. Outputs a structured savings analysis artifact.
+tools: [Read, Glob, Write]
+maxTurns: 15
+---
+
+# Savings analyzer
+
+You analyze a completed orchestrated run to identify cost-saving opportunities. You read structured event data and produce a findings report. You never modify project code.
+
+## Inputs
+
+You receive:
+
+1. A run directory path containing `run-log.jsonl` and `run-index.json`
+2. The analysis framework (three questions, priority order)
+
+## Process
+
+1. **Read run-index.json** -- extract effort, thresholds, model config
+2. **Read run-log.jsonl** -- parse all events
+3. **Check artifact files** -- use Glob to list files in the run directory; note which agents produced artifacts and which did not
+4. **Analyze** -- apply the three-question framework (see below)
+5. **Write artifact** -- write `{NN}_analyst_savings-analysis.md` to the run directory
+
+## Three-question framework
+
+Analyze in priority order:
+
+### 1. Was the communication right-sized?
+
+- Look for repeated context across subagent dispatches (requires session context from the orchestrator's summary)
+- Check for phases with high duration but low artifact size (suggests verbose tool output consumption)
+
+### 2. Was the work necessary?
+
+- **Artifact-less agents:** For each `reviewer_dispatched`, check if a matching `artifact_written` exists. Flag agents dispatched but with no artifact.
+- **Low-value convergence:** If all review phases converged to `none`/`low` criticality and the holistic review also returned `none`, flag the holistic review as confirming what was already established.
+- **Disproportionate optional phases:** Compare simplifier/holistic duration or tokens against implementation. Flag when an optional phase exceeds 50% of implementation cost.
+
+### 3. Were the right resources used?
+
+- **Sequential dispatch:** Check `reviewer_dispatched` events within the same iteration. If timestamps span >5 seconds, flag as potentially sequential.
+- **Expensive re-reviews:** Compare re-review token cost or duration against initial review. Flag when re-review exceeds 100% of initial review.
+
+## Conservatism rule
+
+Tag every suggestion:
+
+- **zero-risk** -- pure waste, no quality impact
+- **low-risk** -- efficiency improvement, quality preserved
+- **tradeoff** -- could affect quality, present data and let human decide
+
+## Output format
+
+Write a markdown artifact with this structure:
+
+```
+# Savings analysis
+
+## Summary
+- Run: {runId}, effort: {effort}, status: {status}
+- Findings: N (X high-impact, Y medium, Z low)
+- Top recommendation: [one-liner]
+
+## Communication
+[findings]
+
+## Necessity
+[findings]
+
+## Resources
+[findings]
+
+## Metrics
+- Total duration: {startedAt} -> {completedAt}
+- Per-phase breakdown (duration and tokens when available)
+- Artifact write rate: {written}/{dispatched}
+```
+
+## ARTIFACT-WRITE SAFEGUARD
+
+**You MUST write your artifact file before exhausting your turn budget.** If you are approaching your turn limit, immediately write what you have. A partial analysis is better than no artifact.

packages/run-core/src/__tests__/event-folder.test.ts

@@ -662,6 +662,120 @@ describe('foldEvents', () => {
     expect(result.phases.parallelReview?.reviewers?.['unknown-reviewer']).toBeUndefined();
   });
 
+  // -- Usage metrics folding ---------------------------------------------------------------
+
+  it('folds usage metrics from reviewer_completed into ReviewerInfo', () => {
+    const header = createHeader();
+    const events: RunEvent[] = [
+      { t: '2026-01-01T00:00:00Z', event: 'run_started' },
+      { t: '2026-01-01T00:01:00Z', event: 'phase_started', phase: 'review' },
+      { t: '2026-01-01T00:01:01Z', event: 'reviewer_dispatched', reviewer: 'test-reviewer' },
+      {
+        t: '2026-01-01T00:02:00Z',
+        event: 'reviewer_completed',
+        reviewer: 'test-reviewer',
+        status: 'completed',
+        criticality: 'low',
+        tokens: 45_000,
+        toolUses: 12,
+        durationMs: 30_000,
+      },
+    ];
+    const state = foldEvents(header, events);
+    expect(state.phases.parallelReview?.reviewers?.['test-reviewer']?.usage).toEqual({
+      tokens: 45_000,
+      toolUses: 12,
+      durationMs: 30_000,
+    });
+  });
+
+  it('folds usage metrics from coder_fix_completed into ReviewIteration', () => {
+    const header = createHeader();
+    const events: RunEvent[] = [
+      { t: '2026-01-01T00:00:00Z', event: 'run_started' },
+      { t: '2026-01-01T00:01:00Z', event: 'phase_started', phase: 'review' },
+      { t: '2026-01-01T00:01:01Z', event: 'reviewer_dispatched', reviewer: 'test-reviewer' },
+      {
+        t: '2026-01-01T00:02:00Z',
+        event: 'reviewer_completed',
+        reviewer: 'test-reviewer',
+        status: 'completed',
+        criticality: 'low',
+      },
+      { t: '2026-01-01T00:03:00Z', event: 'coder_fix_started', iteration: 1 },
+      {
+        t: '2026-01-01T00:05:00Z',
+        event: 'coder_fix_completed',
+        iteration: 1,
+        tokens: 80_000,
+        toolUses: 25,
+        durationMs: 120_000,
+      },
+    ];
+    const state = foldEvents(header, events);
+    expect(state.phases.parallelReview?.iterations?.[0]?.usage).toEqual({
+      tokens: 80_000,
+      toolUses: 25,
+      durationMs: 120_000,
+    });
+  });
+
+  it('folds usage metrics from re_review_completed into ReviewIteration', () => {
+    const header = createHeader();
+    const events: RunEvent[] = [
+      { t: '2026-01-01T00:00:00Z', event: 'run_started' },
+      { t: '2026-01-01T00:01:00Z', event: 'phase_started', phase: 'review' },
+      { t: '2026-01-01T00:01:01Z', event: 'reviewer_dispatched', reviewer: 'test-reviewer' },
+      {
+        t: '2026-01-01T00:02:00Z',
+        event: 'reviewer_completed',
+        reviewer: 'test-reviewer',
+        status: 'completed',
+        criticality: 'low',
+      },
+      { t: '2026-01-01T00:03:00Z', event: 're_review_dispatched', reviewers: ['test-reviewer'] },
+      {
+        t: '2026-01-01T00:04:00Z',
+        event: 're_review_completed',
+        criticalities: { 'test-reviewer': 'none' },
+        tokens: 29_000,
+        toolUses: 8,
+        durationMs: 25_000,
+      },
+    ];
+    const state = foldEvents(header, events);
+    // re_review_dispatched creates iteration at index 1; re_review_completed targets the last iteration
+    expect(state.phases.parallelReview?.iterations?.[1]?.reReviewUsage).toEqual({
+      tokens: 29_000,
+      toolUses: 8,
+      durationMs: 25_000,
+    });
+  });
+
+  it('folds usage metrics from phase_completed into phase object', () => {
+    const header = createHeader();
+    const events: RunEvent[] = [
+      { t: '2026-01-01T00:00:00Z', event: 'run_started' },
+      { t: '2026-01-01T00:01:00Z', event: 'phase_started', phase: 'implementation' },
+      {
+        t: '2026-01-01T00:06:00Z',
+        event: 'phase_completed',
+        phase: 'implementation',
+        status: 'completed',
+        data: { qualityGates: 'passed' },
+        tokens: 150_000,
+        toolUses: 40,
+        durationMs: 300_000,
+      },
+    ];
+    const state = foldEvents(header, events);
+    expect(state.phases.implementation?.usage).toEqual({
+      tokens: 150_000,
+      toolUses: 40,
+      durationMs: 300_000,
+    });
+  });
+
   // -- Full event sequence integration test (F1) ----------------------------------------
 
   it('produces correct CanonicalRunStatus from a full event sequence', () => {