feat(chords): stem-aware chord + segment-key path (accuracy PR-B4)

apps/backend/JSON_SCHEMA.md

@@ -654,6 +654,7 @@ Type: `object \| null`
 | `chordDetail.chordTimeline` | `array<{startSec, endSec, label, labelLong, confidence}> \| null` | Phase 1.D #2 — chord segments with start/end times (seconds), short-form `label` (`"Cm"`, `"Eb"`, `"N"` no-chord), long-form `labelLong` (`"C minor"`, `"Eb major"`, `"N"`), and per-segment confidence. State path decoded by Viterbi over the L1 mass-overlap of `librosa.feature.chroma_cqt` against a 25-state vocabulary (12 major + 12 minor triads + 1 no-chord). Confidence is mean L2 cosine similarity between the chroma and the winning state's template across the segment's frames — bounded [0, 1], where 0.5 is the noise floor and 0.7+ indicates a clean chord match. Segments shorter than 250 ms are dropped; low-confidence "N" segments (<0.4) are also dropped as transition artifacts. Capped at 64 segments. | seconds; 0-1 confidence | Use for arrangement-aware harmonic recommendations ("the bridge sits on Cm for 8 bars, then drops to Eb for 4 — recommend a chord-stab MIDI sequence that matches the timeline"). When `confidence < 0.5` on a segment, hedge — chord detection on full-mix electronic material is noisy. |
 | `chordDetail.chordTimelineSource` | `string` | Phase 1.D #2 — identifier of the engine that produced `chordTimeline`. Currently always `"librosa_viterbi"`. Future-proofs swapping engines. | categorical | Phase 2 should hedge if this is anything other than the engine the prompt is calibrated against. |
 | `chordDetail.chordTimelineAgreement` | `boolean \| null` | Phase 1.D #2 — `true` when the most-frequent non-"N" Viterbi label in `chordTimeline` matches Essentia's `dominantChords[0]` after enharmonic normalization (`D# ↔ Eb`, `F# ↔ Gb`, etc.). `null` when either source has no usable label. | boolean | Strong hedging signal. When `false`, the two chord engines disagree — describe the harmonic content as uncertain and cite both readings. When `true`, you can commit to the timeline reading. |
+| `chordDetail.chordSource` | `"full_mix" \| "harmonic_stems"` | Which audio the chroma ran on (accuracy program PR-B4). `harmonic_stems` = bass-removed stem mix (`other`+`vocals`), run when Demucs stems were available; `full_mix` = whole track otherwise. | categorical | `harmonic_stems` is the higher-accuracy path (the bassline is the biggest full-mix chroma polluter). Provenance only — not an override of the chord labels. |
 | `chordDetail.chordTimeline[].labelLong` | `string` | Long-form chord label paired with the short-form `label`: `"C major"`, `"F# minor"`, `"N"`. | chord labels | Use when a recommendation reads better with human-readable harmonic citations. |
 | `chordDetail.chordChangeCount` | `int` | Phase 1.D #2 — count of unique chord-to-chord transitions in the Viterbi-decoded timeline. Flat 1-chord tracks score 0; harmonically active tracks score 16+. | count | A proxy for "how harmonically active" the track is. Cite for arrangement-density and harmonic-rhythm recommendations. |
 
@@ -666,6 +667,7 @@ Type: `array<object> \| null`
 | `segmentKey[].segmentIndex` | `int` | Segment index aligned with `structure.segments`. | integer index | Use for joining harmonic data to arrangement segments. |
 | `segmentKey[].key` | `string \| null` | Per-segment key label (`edma` profile). | categorical | Detects section-level key drift or modal pivots. |
 | `segmentKey[].keyConfidence` | `float \| null` | Per-segment key confidence. | 0-1 (approx) | Low confidence means treat segment key as tentative. |
+| `segmentKey[].source` | `"full_mix" \| "harmonic_stems"` | Which audio the per-segment key ran on (accuracy program PR-B4); same harmonic-isolation rationale as `chordDetail.chordSource`. | categorical | Provenance only. |
 
 ---
 

apps/backend/analyze.py

@@ -63,6 +63,7 @@
     _load_stem_stereo,
     analyze_crepe_pitch,
     cleanup_stems,
+    mix_stems_mono,
 )
 from separation_backend import separate_stems_backend  # noqa: E402
 from analyze_estimate import (  # noqa: E402
@@ -1792,14 +1793,21 @@ def main():
                 sample_rate=sample_rate,
             )
         )
-        result.update(analyze_segment_key(result.get("structure"), mono, sample_rate))
+        # Harmonic-source mix (bass/drums removed) for chord + key chroma when
+        # stems are available; None -> full-mix path, bit-identical to before.
+        harmonic_mono = mix_stems_mono(stems, ("other", "vocals"), sample_rate)
+        result.update(
+            analyze_segment_key(
+                result.get("structure"), mono, sample_rate, harmonic_mono=harmonic_mono
+            )
+        )
 
         # Stereo array no longer needed — release memory.
         del stereo
         gc.collect()
 
         # Chords
-        result.update(analyze_chords(mono, sample_rate))
+        result.update(analyze_chords(mono, sample_rate, harmonic_mono=harmonic_mono))
 
         # Perceptual
         result.update(analyze_perceptual(mono, sample_rate))

apps/backend/analyze_audio_io.py

@@ -47,6 +47,35 @@ def _load_stem_mono(
         return None
 
 
+def mix_stems_mono(
+    stems: dict | None,
+    stem_names: tuple[str, ...],
+    sample_rate: int = 44100,
+) -> np.ndarray | None:
+    """Sum the named Demucs stems into one mono array (harmonic-source mix).
+
+    Used by chord/key analysis to run chroma on bass-removed audio: summing
+    ``other`` + ``vocals`` keeps the harmonic content while dropping the
+    bassline and drums that pollute full-mix chroma. Returns ``None`` when no
+    named stem is loadable, so callers fall back to the full mix unchanged.
+    Stems are equal-length (same source render), but guard for safety.
+    """
+    loaded = [
+        arr
+        for name in stem_names
+        if (arr := _load_stem_mono(stems, name, sample_rate=sample_rate)) is not None
+    ]
+    if not loaded:
+        return None
+    length = min(arr.shape[0] for arr in loaded)
+    if length == 0:
+        return None
+    mixed = np.zeros(length, dtype=np.float32)
+    for arr in loaded:
+        mixed += arr[:length]
+    return mixed
+
+
 def _load_stem_stereo(
     stems: dict | None,
     stem_name: str,

apps/backend/analyze_segments.py

@@ -467,13 +467,24 @@ def analyze_segment_key(
     structure_data: dict | None,
     mono: np.ndarray,
     sample_rate: int = 44100,
+    *,
+    harmonic_mono: np.ndarray | None = None,
 ) -> dict:
-    """Compute key and confidence per segment using KeyExtractor."""
+    """Compute key and confidence per segment using KeyExtractor.
+
+    When ``harmonic_mono`` (bass-removed stem mix) is provided, per-segment
+    key runs on it instead of the full mix — same harmonic-isolation rationale
+    as ``analyze_chords``. Each entry records its ``source``. Falls back to the
+    full mix (bit-identical) when stems are unavailable.
+    """
+    source = "harmonic_stems" if harmonic_mono is not None else "full_mix"
     try:
         if structure_data is None:
             return {"segmentKey": None}
 
-        mono_arr = np.asarray(mono, dtype=np.float32)
+        mono_arr = np.asarray(
+            harmonic_mono if harmonic_mono is not None else mono, dtype=np.float32
+        )
         if mono_arr.ndim != 1 or mono_arr.size == 0:
             return {"segmentKey": None}
 
@@ -508,6 +519,7 @@ def analyze_segment_key(
                     "segmentIndex": index,
                     "key": key_value,
                     "keyConfidence": key_confidence,
+                    "source": source,
                 }
             )
 
@@ -517,11 +529,41 @@ def analyze_segment_key(
         return {"segmentKey": None}
 
 
-def analyze_chords(mono: np.ndarray, sample_rate: int = 44100) -> dict:
-    """Frame-wise HPCP analysis and chord detection via ChordsDetection."""
+def _empty_chord_detail(source: str) -> dict:
+    return {
+        "chordDetail": {
+            "chordSequence": [],
+            "chordStrength": 0.0,
+            "progression": [],
+            "dominantChords": [],
+            "chordTimeline": [],
+            "chordChangeCount": 0,
+            "chordTimelineSource": "librosa_viterbi",
+            "chordTimelineAgreement": None,
+            "chordSource": source,
+        }
+    }
+
+
+def analyze_chords(
+    mono: np.ndarray,
+    sample_rate: int = 44100,
+    *,
+    harmonic_mono: np.ndarray | None = None,
+) -> dict:
+    """Frame-wise HPCP analysis and chord detection via ChordsDetection.
+
+    When ``harmonic_mono`` is provided (a bass-removed stem mix), chroma runs
+    on it instead of the full mix — the bassline is the biggest polluter of
+    full-mix chroma, so harmonic-source isolation lifts chord accuracy on
+    dense material. Falls back to the full mix (bit-identical to before) when
+    stems are unavailable. ``chordSource`` records which path ran.
+    """
+    source = "harmonic_stems" if harmonic_mono is not None else "full_mix"
     try:
+        analysis_mono = harmonic_mono if harmonic_mono is not None else mono
         hp_filter = es.HighPass(cutoffFrequency=120, sampleRate=sample_rate)
-        mono_filtered = hp_filter(mono)
+        mono_filtered = hp_filter(analysis_mono)
 
         frame_size = 4096
         hop_size = 2048
@@ -550,36 +592,14 @@ def analyze_chords(mono: np.ndarray, sample_rate: int = 44100) -> dict:
                 continue
 
         if len(hpcp_sequence) == 0:
-            return {
-                "chordDetail": {
-                    "chordSequence": [],
-                    "chordStrength": 0.0,
-                    "progression": [],
-                    "dominantChords": [],
-                    "chordTimeline": [],
-                    "chordChangeCount": 0,
-                    "chordTimelineSource": "librosa_viterbi",
-                    "chordTimelineAgreement": None,
-                }
-            }
+            return _empty_chord_detail(source)
 
         chords, strength = chords_algo(np.asarray(hpcp_sequence, dtype=np.float32))
         chords = [str(c) for c in chords]
         strength = np.asarray(strength, dtype=np.float64)
 
         if len(chords) == 0:
-            return {
-                "chordDetail": {
-                    "chordSequence": [],
-                    "chordStrength": 0.0,
-                    "progression": [],
-                    "dominantChords": [],
-                    "chordTimeline": [],
-                    "chordChangeCount": 0,
-                    "chordTimelineSource": "librosa_viterbi",
-                    "chordTimelineAgreement": None,
-                }
-            }
+            return _empty_chord_detail(source)
 
         # Keep payload manageable.
         if len(chords) > 32:
@@ -650,6 +670,7 @@ def analyze_chords(mono: np.ndarray, sample_rate: int = 44100) -> dict:
                 "chordChangeCount": chord_change_count,
                 "chordTimelineSource": "librosa_viterbi",
                 "chordTimelineAgreement": chord_timeline_agreement,
+                "chordSource": source,
             }
         }
     except Exception as e:

apps/backend/tests/fixtures/golden/phase1_default.json

@@ -74,6 +74,7 @@
     "chordDetail": {
       "chordChangeCount": "number",
       "chordSequence": "scalarList",
+      "chordSource": "str",
       "chordStrength": "number",
       "chordTimeline": {
         "[]": {
@@ -367,7 +368,8 @@
       "[]": {
         "key": "str",
         "keyConfidence": "number",
-        "segmentIndex": "number"
+        "segmentIndex": "number",
+        "source": "str"
       }
     },
     "segmentLoudness": {

apps/backend/tests/test_stem_aware_chords.py

@@ -0,0 +1,64 @@
+import os
+import tempfile
+import unittest
+import wave
+
+import numpy as np
+
+from analyze_audio_io import mix_stems_mono
+from analyze_segments import analyze_chords, analyze_segment_key
+
+
+def _write_wav(path: str, samples: np.ndarray, sr: int = 44100) -> None:
+    pcm = np.clip(samples, -1.0, 1.0)
+    pcm16 = (pcm * 32767.0).astype("<i2")
+    with wave.open(path, "wb") as w:
+        w.setnchannels(1)
+        w.setsampwidth(2)
+        w.setframerate(sr)
+        w.writeframes(pcm16.tobytes())
+
+
+class MixStemsMonoTests(unittest.TestCase):
+    def test_sums_named_stems_and_skips_missing(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            other = os.path.join(d, "other.wav")
+            vocals = os.path.join(d, "vocals.wav")
+            _write_wav(other, np.full(44100, 0.2, dtype=np.float32))
+            _write_wav(vocals, np.full(44100, 0.1, dtype=np.float32))
+            stems = {"other": other, "vocals": vocals, "bass": os.path.join(d, "nope.wav")}
+            mixed = mix_stems_mono(stems, ("other", "vocals"))
+            self.assertIsNotNone(mixed)
+            self.assertEqual(mixed.shape[0], 44100)
+            # 0.2 + 0.1 summed.
+            self.assertAlmostEqual(float(mixed.mean()), 0.3, places=2)
+
+    def test_none_when_no_stem_loadable(self) -> None:
+        self.assertIsNone(mix_stems_mono(None, ("other", "vocals")))
+        self.assertIsNone(mix_stems_mono({"bass": "/no/such.wav"}, ("other", "vocals")))
+
+
+class ChordSourceTaggingTests(unittest.TestCase):
+    def _tone(self, freqs, seconds=2.0, sr=44100):
+        t = np.arange(int(seconds * sr)) / sr
+        sig = sum(np.sin(2 * np.pi * f * t) for f in freqs)
+        return (sig / np.max(np.abs(sig)) * 0.8).astype(np.float32)
+
+    def test_chord_source_reflects_harmonic_input(self) -> None:
+        full = self._tone([261.6, 329.6, 392.0])  # C major triad
+        harmonic = self._tone([261.6, 329.6, 392.0])
+        default = analyze_chords(full)["chordDetail"]
+        self.assertEqual(default["chordSource"], "full_mix")
+        stemmed = analyze_chords(full, harmonic_mono=harmonic)["chordDetail"]
+        self.assertEqual(stemmed["chordSource"], "harmonic_stems")
+
+    def test_segment_key_source_tagged(self) -> None:
+        mono = self._tone([261.6, 329.6, 392.0], seconds=4.0)
+        structure = {"segments": [{"start": 0.0, "end": 4.0}]}
+        out = analyze_segment_key(structure, mono, harmonic_mono=mono)["segmentKey"]
+        if out:  # segmentation may return None on trivial structure; tag when present
+            self.assertTrue(all(e["source"] == "harmonic_stems" for e in out))
+
+
+if __name__ == "__main__":
+    unittest.main()

apps/ui/src/services/backendPhase1Client.ts

@@ -944,6 +944,10 @@ function parseOptionalChordDetail(value: unknown): ChordDetail | null {
         : null;
 
   const chordTimeline = parseOptionalChordTimeline(value.chordTimeline);
+  const chordSource =
+    value.chordSource === "harmonic_stems" || value.chordSource === "full_mix"
+      ? value.chordSource
+      : undefined;
 
   return {
     chordSequence,
@@ -954,6 +958,7 @@ function parseOptionalChordDetail(value: unknown): ChordDetail | null {
     chordChangeCount,
     chordTimelineSource,
     chordTimelineAgreement,
+    chordSource,
   };
 }
 

apps/ui/src/types/measurement.ts

@@ -448,6 +448,8 @@ export interface SegmentKeyEntry {
   segmentIndex: number;
   key: string | null;
   keyConfidence?: number | null;
+  /** Which audio the per-segment key ran on (accuracy program PR-B4). */
+  source?: "full_mix" | "harmonic_stems";
 }
 
 /**
@@ -479,6 +481,12 @@ export interface ChordDetail {
   chordTimelineSource?: string | null;
   /** Phase 1.D #2: true when Viterbi's dominant matches Essentia's dominantChords[0] after enharmonic normalization. */
   chordTimelineAgreement?: boolean | null;
+  /**
+   * Which audio the chroma ran on (accuracy program PR-B4): `harmonic_stems`
+   * = bass-removed stem mix (higher chord accuracy on dense material),
+   * `full_mix` = whole track (when stems were unavailable).
+   */
+  chordSource?: "full_mix" | "harmonic_stems";
 }
 
 export interface PerceptualDetail {

apps/ui/tests/fixtures/phase1FullPayload.ts

@@ -575,7 +575,7 @@ export const phase1EnvelopeFixture = {
     },
   ],
   segmentStereo: [{ segmentIndex: 0, stereoWidth: 0.8, stereoCorrelation: 0.9 }],
-  segmentKey: [{ segmentIndex: 0, key: 'A minor', keyConfidence: 0.85 }],
+  segmentKey: [{ segmentIndex: 0, key: 'A minor', keyConfidence: 0.85, source: 'full_mix' }],
   essentiaFeatures: {
     zeroCrossingRate: 0.12,
     hfc: 0.45,
@@ -594,6 +594,7 @@ export const phase1EnvelopeFixture = {
     chordChangeCount: 1,
     chordTimelineSource: 'librosa_viterbi',
     chordTimelineAgreement: true,
+    chordSource: 'harmonic_stems',
   },
   perceptual: {
     sharpness: 1.42,