feat(phase 1.D #2): migrate chordTimeline from Essentia smoothing to librosa+Viterbi

apps/backend/JSON_SCHEMA.md

@@ -610,8 +610,11 @@ Type: `object \| null`
 | `chordDetail.chordStrength` | `float` | Mean chord detection strength. | 0-1 (approx) | Low/medium values indicate probable ambiguity on full-master chord detection. |
 | `chordDetail.progression` | `string[]` | Consecutive-duplicate-removed progression, capped at 16. | chord labels | Compact harmonic change path for arrangement planning. |
 | `chordDetail.dominantChords` | `string[]` | Top 4 most frequent chord labels. | chord labels | Candidate tonic/relative function anchors. |
-| `chordDetail.chordTimeline` | `array<{startSec, endSec, label, confidence}> \| null` | Phase 1.D #2 — chord segments with start/end times (seconds) and per-segment confidence (mean of per-frame strength values). Each segment is the result of a 5-frame median-filter smoothing of the per-frame chord labels, then merging consecutive same-label frames. Segments shorter than ~250 ms are dropped as detection noise. Capped at 64 segments. | seconds; 0-1 confidence | Use for arrangement-aware harmonic recommendations ("the bridge sits on Cm for 8 bars, then drops to Eb for 4 — recommend a chord-stab MIDI sequence that matches the timeline"). When `confidence < 0.5` on a segment, hedge — chord-detection on full-mix is noisy. |
-| `chordDetail.chordChangeCount` | `int` | Phase 1.D #2 — count of unique chord-to-chord transitions in the smoothed timeline. Flat 1-chord tracks score 0; harmonically active tracks score 16+. | count | A proxy for "how harmonically active" the track is. Cite for arrangement-density and harmonic-rhythm recommendations. |
+| `chordDetail.chordTimeline` | `array<{startSec, endSec, label, labelLong, confidence}> \| null` | Phase 1.D #2 — chord segments with start/end times (seconds), short-form `label` (`"Cm"`, `"Eb"`, `"N"` no-chord), long-form `labelLong` (`"C minor"`, `"Eb major"`, `"N"`), and per-segment confidence. State path decoded by Viterbi over the L1 mass-overlap of `librosa.feature.chroma_cqt` against a 25-state vocabulary (12 major + 12 minor triads + 1 no-chord). Confidence is mean L2 cosine similarity between the chroma and the winning state's template across the segment's frames — bounded [0, 1], where 0.5 is the noise floor and 0.7+ indicates a clean chord match. Segments shorter than 250 ms are dropped; low-confidence "N" segments (<0.4) are also dropped as transition artifacts. Capped at 64 segments. | seconds; 0-1 confidence | Use for arrangement-aware harmonic recommendations ("the bridge sits on Cm for 8 bars, then drops to Eb for 4 — recommend a chord-stab MIDI sequence that matches the timeline"). When `confidence < 0.5` on a segment, hedge — chord detection on full-mix electronic material is noisy. |
+| `chordDetail.chordTimelineSource` | `string` | Phase 1.D #2 — identifier of the engine that produced `chordTimeline`. Currently always `"librosa_viterbi"`. Future-proofs swapping engines. | categorical | Phase 2 should hedge if this is anything other than the engine the prompt is calibrated against. |
+| `chordDetail.chordTimelineAgreement` | `boolean \| null` | Phase 1.D #2 — `true` when the most-frequent non-"N" Viterbi label in `chordTimeline` matches Essentia's `dominantChords[0]` after enharmonic normalization (`D# ↔ Eb`, `F# ↔ Gb`, etc.). `null` when either source has no usable label. | boolean | Strong hedging signal. When `false`, the two chord engines disagree — describe the harmonic content as uncertain and cite both readings. When `true`, you can commit to the timeline reading. |
+| `chordDetail.chordTimeline[].labelLong` | `string` | Long-form chord label paired with the short-form `label`: `"C major"`, `"F# minor"`, `"N"`. | chord labels | Use when a recommendation reads better with human-readable harmonic citations. |
+| `chordDetail.chordChangeCount` | `int` | Phase 1.D #2 — count of unique chord-to-chord transitions in the Viterbi-decoded timeline. Flat 1-chord tracks score 0; harmonically active tracks score 16+. | count | A proxy for "how harmonically active" the track is. Cite for arrangement-density and harmonic-rhythm recommendations. |
 
 ### `segmentKey`
 

apps/backend/prompts/phase2_system.txt

@@ -171,9 +171,10 @@ REVERB DETAIL (Phase 1.D #5)
 - When `stemAnalysis.{stem}.reverbDetail` is present (Phase 1.D #5 per-stem), cite it for per-element reverb advice — e.g. "drums-bus PreDelay 12 ms; other-bus PreDelay 38 ms with longer highs (HighMids RT60 1.8 s) → Hall reverb with damping set high." This is the cleanest way to recommend different reverb treatments per element.
 
 CHORD TIMELINE (Phase 1.D #2)
-- chordDetail.chordTimeline is an array of `{startSec, endSec, label, confidence}` segments after 5-frame median-filter smoothing. Cite for arrangement-aware harmonic claims: "the build sits on Cm from 0:48 to 1:00 (confidence 0.7), then drops to Eb for 8 bars (confidence 0.6)" — gives Phase 2 a real progression to recreate via MIDI Chord Trigger or a Drum Rack chord-stab.
+- chordDetail.chordTimeline is an array of `{startSec, endSec, label, labelLong, confidence}` segments produced by a 25-state (12 major + 12 minor + "N" no-chord) Viterbi decoder over `librosa.feature.chroma_cqt`. `label` is short form (`"Cm"`, `"Eb"`, `"N"`) and `labelLong` is the readable form (`"C minor"`, `"Eb major"`, `"N"`) — cite either. Use for arrangement-aware harmonic claims: "the build sits on Cm from 0:48 to 1:00 (confidence 0.7), then drops to Eb for 8 bars (confidence 0.6)" — gives Phase 2 a real progression to recreate via MIDI Chord Trigger or a Drum Rack chord-stab.
 - chordDetail.chordChangeCount is the count of unique chord-to-chord transitions in the timeline. Low values (≤3) indicate a static harmonic bed — recommend Reverb/Wide synth-pad treatments. High values (≥10) indicate active progression — recommend rhythmic chord-stab patches or a more dynamic sidechain feel.
 - When a chordTimeline segment has `confidence < 0.5`, treat it as noisy — chord detection on a full-mix dense electronic track is hard. Hedge: "the harmony around 0:48 reads as Cm but with low confidence; alternative reading is Eb minor — keep the synth-pad chord-agnostic with sus2 voicings".
+- chordDetail.chordTimelineAgreement is true when the most-frequent non-"N" Viterbi label matches Essentia's dominantChords[0] after enharmonic normalization. When agreement is false, hedge any harmonic recommendation — cite both readings ("the timeline reads Cm but the simpler ChordsDetection summary reads C — treat the modal flavor as uncertain") rather than committing to one. When agreement is true, you can commit to the timeline reading without that caveat.
 - Always cross-reference with `key` and `keyConfidence` at the top level. If `chordTimeline` consistently lands outside the measured key, either the key detection is wrong (low keyConfidence) or the track modulates — say so explicitly rather than committing to one interpretation.
 
 VOCAL DETECTION (with Demucs ghost-stem awareness)

apps/backend/tests/test_analyze.py

@@ -2458,5 +2458,171 @@ def test_bpm_raw_original_always_set(self) -> None:
         self.assertEqual(result["bpmRawOriginal"], 66.0)
 
 
+class ChordTimelineViterbiTests(unittest.TestCase):
+    """Phase 1.D #2 — librosa+Viterbi chord-timeline migration.
+
+    These tests verify the new 25-state Viterbi engine that replaced the
+    earlier 5-frame median-filter smoothing in analyze_chords. We do not
+    re-test the Essentia layer or the median smoother — those are gone.
+    """
+
+    SAMPLE_RATE = 44_100
+    DURATION_SECONDS = 4.0
+    EXPECTED_TIMELINE_KEYS = {"startSec", "endSec", "label", "labelLong", "confidence"}
+
+    def _make_chord_triad_audio(
+        self, frequencies: tuple[float, ...], duration_s: float = DURATION_SECONDS
+    ) -> np.ndarray:
+        """Sum sine tones at the given frequencies to make a synthetic chord."""
+        n = int(self.SAMPLE_RATE * duration_s)
+        t = np.arange(n, dtype=np.float64) / self.SAMPLE_RATE
+        signal = np.zeros(n, dtype=np.float64)
+        for freq in frequencies:
+            signal += np.sin(2 * np.pi * freq * t)
+        signal /= max(1.0, len(frequencies))
+        # 50 ms fade in / out so the highpass + window edge effects don't
+        # dominate the start/end frames.
+        fade = int(0.05 * self.SAMPLE_RATE)
+        signal[:fade] *= np.linspace(0.0, 1.0, fade)
+        signal[-fade:] *= np.linspace(1.0, 0.0, fade)
+        return signal.astype(np.float32)
+
+    def test_chord_timeline_synthetic_c_major(self) -> None:
+        """A 4-second C-major-triad sine cluster produces a clean Viterbi timeline."""
+        from analyze_segments import analyze_chords
+
+        # C4 + E4 + G4 — the canonical C major triad.
+        audio = self._make_chord_triad_audio((261.63, 329.63, 392.00))
+        result = analyze_chords(audio, sample_rate=self.SAMPLE_RATE)
+        cd = result.get("chordDetail")
+        self.assertIsNotNone(cd, "chord analysis should not return None for clean audio")
+        timeline = cd.get("chordTimeline")
+        self.assertIsInstance(timeline, list)
+        self.assertGreater(len(timeline), 0, "synthetic chord should produce ≥1 segment")
+
+        # Shape & invariants for every segment.
+        previous_end = -1.0
+        for seg in timeline:
+            self.assertEqual(set(seg.keys()), self.EXPECTED_TIMELINE_KEYS)
+            self.assertGreaterEqual(seg["startSec"], 0.0)
+            self.assertGreaterEqual(seg["endSec"], seg["startSec"])
+            self.assertGreaterEqual(seg["confidence"], 0.0)
+            self.assertLessEqual(seg["confidence"], 1.0)
+            self.assertIsInstance(seg["label"], str)
+            self.assertIsInstance(seg["labelLong"], str)
+            # Non-overlapping & ordered.
+            self.assertGreaterEqual(seg["startSec"], previous_end - 1e-6)
+            previous_end = seg["endSec"]
+
+        # Soft expectation: dominant label is C major. Don't fail on this —
+        # synthetic-fixture decoding can flicker under template ambiguity.
+        from collections import Counter as _Counter
+        label_counts = _Counter(seg["label"] for seg in timeline)
+        dominant = label_counts.most_common(1)[0][0]
+        if dominant != "C":
+            print(
+                f"[soft] expected dominant 'C', got {dominant} "
+                f"(distribution: {dict(label_counts)})",
+                file=sys.stderr,
+            )
+
+    def test_chord_timeline_white_noise_no_spurious_confident_triads(self) -> None:
+        """White noise should not produce strongly-confident triads.
+
+        The cosine-similarity confidence has a noise floor at ~0.5 for random
+        chroma against 3-active-bin templates (sqrt(3/12) ≈ 0.5), so the bar
+        is 0.65 — well above the noise floor, well below a clean chord match.
+        """
+        from analyze_segments import analyze_chords
+
+        rng = np.random.default_rng(seed=42)
+        noise = rng.standard_normal(int(self.SAMPLE_RATE * self.DURATION_SECONDS))
+        noise = (noise * 0.1).astype(np.float32)
+        result = analyze_chords(noise, sample_rate=self.SAMPLE_RATE)
+        cd = result.get("chordDetail")
+        self.assertIsNotNone(cd)
+        timeline = cd.get("chordTimeline")
+        self.assertIsInstance(timeline, list)
+        for seg in timeline:
+            allowed = seg["label"] == "N" or seg["confidence"] < 0.65
+            self.assertTrue(
+                allowed,
+                f"white noise produced confident triad: {seg!r}",
+            )
+
+    def test_chord_timeline_source_and_agreement_fields(self) -> None:
+        """chordDetail exposes chordTimelineSource and chordTimelineAgreement."""
+        from analyze_segments import analyze_chords
+
+        audio = self._make_chord_triad_audio((261.63, 329.63, 392.00))
+        result = analyze_chords(audio, sample_rate=self.SAMPLE_RATE)
+        cd = result["chordDetail"]
+        self.assertEqual(cd["chordTimelineSource"], "librosa_viterbi")
+        # agreement must be a bool or None — never a string or number.
+        agreement = cd["chordTimelineAgreement"]
+        self.assertIn(agreement, (True, False, None))
+
+    def test_chord_change_count_recomputed_from_viterbi_timeline(self) -> None:
+        """chordChangeCount counts transitions in the new Viterbi timeline."""
+        from analyze_segments import analyze_chords
+
+        audio = self._make_chord_triad_audio((261.63, 329.63, 392.00))
+        result = analyze_chords(audio, sample_rate=self.SAMPLE_RATE)
+        cd = result["chordDetail"]
+        expected = sum(
+            1 for i in range(1, len(cd["chordTimeline"]))
+            if cd["chordTimeline"][i]["label"] != cd["chordTimeline"][i - 1]["label"]
+        )
+        self.assertEqual(cd["chordChangeCount"], expected)
+
+    def test_normalize_chord_label_handles_enharmonics_and_quality(self) -> None:
+        """_normalize_chord_label_for_compare handles short/long forms and enharmonics."""
+        from analyze_segments import _normalize_chord_label_for_compare
+
+        # Short form.
+        self.assertEqual(_normalize_chord_label_for_compare("C"), "C:maj")
+        self.assertEqual(_normalize_chord_label_for_compare("Cm"), "C:min")
+        self.assertEqual(_normalize_chord_label_for_compare("Em"), "E:min")
+        self.assertEqual(_normalize_chord_label_for_compare("F#"), "Gb:maj")
+        self.assertEqual(_normalize_chord_label_for_compare("F#m"), "Gb:min")
+        # Long form.
+        self.assertEqual(_normalize_chord_label_for_compare("C major"), "C:maj")
+        self.assertEqual(_normalize_chord_label_for_compare("C minor"), "C:min")
+        self.assertEqual(_normalize_chord_label_for_compare("D# minor"), "Eb:min")
+        # Enharmonic equivalence — these MUST compare equal after normalization.
+        self.assertEqual(
+            _normalize_chord_label_for_compare("D#m"),
+            _normalize_chord_label_for_compare("Eb minor"),
+        )
+        self.assertEqual(
+            _normalize_chord_label_for_compare("A#"),
+            _normalize_chord_label_for_compare("Bb major"),
+        )
+        # N stays N.
+        self.assertEqual(_normalize_chord_label_for_compare("N"), "N")
+        # Non-string input safely returns "".
+        self.assertEqual(_normalize_chord_label_for_compare(None), "")  # type: ignore[arg-type]
+
+    def test_chord_templates_25_has_correct_shape_and_triad_masks(self) -> None:
+        """_chord_templates_25 emits 25 L1-normalized rows: 12 major + 12 minor + N."""
+        from analyze_segments import _chord_templates_25
+
+        templates = _chord_templates_25()
+        self.assertEqual(templates.shape, (25, 12))
+        # Each row sums to 1.0 (L1-normalized).
+        np.testing.assert_allclose(templates.sum(axis=1), np.ones(25), atol=1e-9)
+        # Row 0 = C major (root C, +4 E, +7 G) → pitch classes 0, 4, 7 active.
+        c_major = templates[0]
+        active = np.where(c_major > 0)[0].tolist()
+        self.assertEqual(active, [0, 4, 7])
+        # Row 12 = C minor (root C, +3 Eb, +7 G) → pitch classes 0, 3, 7 active.
+        c_minor = templates[12]
+        active = np.where(c_minor > 0)[0].tolist()
+        self.assertEqual(active, [0, 3, 7])
+        # Row 24 = N — uniform across all 12 pitch classes.
+        n_row = templates[24]
+        np.testing.assert_allclose(n_row, np.full(12, 1.0 / 12.0), atol=1e-9)
+
+
 if __name__ == "__main__":
     unittest.main()

apps/ui/src/services/backendPhase1Client.ts

@@ -7,6 +7,8 @@ import {
   BackendEstimateResponse,
   BassDetail,
   BeatsLoudness,
+  ChordDetail,
+  ChordTimelineEntry,
   DanceabilityResult,
   DynamicCharacter,
   GenreDetail,
@@ -643,7 +645,7 @@ export function parsePhase1Result(value: unknown): Phase1Result {
     segmentSpectral: Array.isArray(phase1.segmentSpectral) ? phase1.segmentSpectral as Phase1Result["segmentSpectral"] : null,
     segmentStereo: Array.isArray(phase1.segmentStereo) ? phase1.segmentStereo as Phase1Result["segmentStereo"] : null,
     segmentKey: Array.isArray(phase1.segmentKey) ? phase1.segmentKey as Phase1Result["segmentKey"] : null,
-    chordDetail: isRecord(phase1.chordDetail) ? phase1.chordDetail as Phase1Result["chordDetail"] : null,
+    chordDetail: parseOptionalChordDetail(phase1.chordDetail),
     perceptual: isRecord(phase1.perceptual) ? phase1.perceptual as unknown as Phase1Result["perceptual"] : null,
     essentiaFeatures: isRecord(phase1.essentiaFeatures) ? phase1.essentiaFeatures as Phase1Result["essentiaFeatures"] : null,
     acidDetail: parseOptionalAcidDetail(phase1.acidDetail),
@@ -824,6 +826,95 @@ function parseOptionalRhythmTimeline(value: unknown): RhythmTimeline | null {
   };
 }
 
+/**
+ * Tolerant parser for `chordDetail`. Modeled on `parseOptionalRhythmTimeline`:
+ * forwards the four Essentia fields as before, parses the new Viterbi fields
+ * (`chordTimeline`, `chordChangeCount`, `chordTimelineSource`,
+ * `chordTimelineAgreement`) defensively. One malformed `chordTimeline` entry
+ * is dropped silently rather than rejecting the whole `chordDetail` — Phase 1
+ * payloads sometimes come from disk caches written by older analyzer versions.
+ */
+function parseOptionalChordDetail(value: unknown): ChordDetail | null {
+  if (value === undefined || value === null) return null;
+  if (!isRecord(value)) return null;
+
+  const chordSequence = Array.isArray(value.chordSequence)
+    ? value.chordSequence.filter((entry): entry is string => typeof entry === "string")
+    : null;
+  const progression = Array.isArray(value.progression)
+    ? value.progression.filter((entry): entry is string => typeof entry === "string")
+    : null;
+  const dominantChords = Array.isArray(value.dominantChords)
+    ? value.dominantChords.filter((entry): entry is string => typeof entry === "string")
+    : null;
+
+  const chordStrength = toNumber(value.chordStrength);
+  const chordChangeCountRaw = toNumber(value.chordChangeCount);
+  const chordChangeCount =
+    chordChangeCountRaw === null ? null : Math.max(0, Math.round(chordChangeCountRaw));
+
+  const chordTimelineSource =
+    typeof value.chordTimelineSource === "string" ? value.chordTimelineSource : null;
+
+  const chordTimelineAgreement =
+    value.chordTimelineAgreement === true
+      ? true
+      : value.chordTimelineAgreement === false
+        ? false
+        : null;
+
+  const chordTimeline = parseOptionalChordTimeline(value.chordTimeline);
+
+  return {
+    chordSequence,
+    chordStrength,
+    progression,
+    dominantChords,
+    chordTimeline,
+    chordChangeCount,
+    chordTimelineSource,
+    chordTimelineAgreement,
+  };
+}
+
+function parseOptionalChordTimeline(value: unknown): ChordTimelineEntry[] | null {
+  if (!Array.isArray(value)) return null;
+
+  const parsed: ChordTimelineEntry[] = [];
+  for (const entry of value) {
+    if (!isRecord(entry)) continue;
+
+    const startSec = toNumber(entry.startSec);
+    const endSec = toNumber(entry.endSec);
+    const confidenceRaw = toNumber(entry.confidence);
+    const label = typeof entry.label === "string" ? entry.label : null;
+    if (
+      startSec === null ||
+      endSec === null ||
+      confidenceRaw === null ||
+      label === null ||
+      endSec < startSec
+    ) {
+      continue;
+    }
+
+    const labelLong = typeof entry.labelLong === "string" ? entry.labelLong : undefined;
+    const confidence = Math.min(1, Math.max(0, confidenceRaw));
+
+    const segment: ChordTimelineEntry = {
+      startSec,
+      endSec,
+      label,
+      confidence,
+    };
+    if (labelLong !== undefined) segment.labelLong = labelLong;
+    parsed.push(segment);
+  }
+
+  parsed.sort((a, b) => a.startSec - b.startSec);
+  return parsed;
+}
+
 function parseOptionalAcidDetail(value: unknown): AcidDetail | null {
   if (value === undefined || value === null) return null;
   if (!isRecord(value)) return null;

apps/ui/src/services/fieldAnalytics.ts

@@ -162,7 +162,9 @@ const ALL_PHASE1_FIELDS: string[] = [
   // Chords
   'chordDetail.dominantChords',
   'chordDetail.chordStrength',
-  'chordDetail.chordProgression',
+  'chordDetail.progression',
+  'chordDetail.chordTimeline',
+  'chordDetail.chordChangeCount',
   // Danceability
   'danceability.danceability',
   'danceability.dfa',