From a45558867dae527470fd38782abf927d5655c678 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 05:18:47 +0000 Subject: [PATCH] feat(chords): stem-aware chord + segment-key path (accuracy PR-B4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs chord/key chroma on a bass-removed stem mix (other+vocals) instead of the full mix when Demucs stems are available. The bassline is the single biggest polluter of full-mix chroma, so harmonic-source isolation is the highest-leverage chord-accuracy lever in the baseline audit. - mix_stems_mono(stems, ('other','vocals')) in analyze_audio_io.py sums the harmonic stems to mono; None when no stem loads. - analyze_chords / analyze_segment_key gain a harmonic_mono kwarg; when present the chroma runs on it. Additive chordDetail.chordSource + segmentKey[].source ('full_mix' | 'harmonic_stems') record which ran. - Stems absent -> full-mix path, BIT-IDENTICAL to before (golden diff adds only the two source keys; chord values unchanged; chords-only corpus clips hold at ~0.99). Validation honesty: Demucs model weights are network-blocked in this sandbox (proxy 403, same restriction as the datasets), so separation returns None here and the synthetic multi-layer clips exercise the full-mix fallback, not the harmonic path. The harmonic wiring is proven by unit tests (mix_stems_mono summing/skip-missing; chordSource + segmentKey source tagging); the end-to-end accuracy GAIN must be measured where Demucs weights are reachable (operator machine / CI with network) on real dense mixes — the synthetic bass is a clean low sine that barely pollutes chroma anyway. No accuracy improvement is claimed from this sandbox. Contract mirrors (tripwire #4): JSON_SCHEMA rows; types/measurement.ts ChordDetail.chordSource + SegmentKeyEntry.source; backendPhase1Client chord reconstructor forwards chordSource (segmentKey passes through); phase1FullPayload fixture; golden re-baselined (structural). Tests: test_stem_aware_chords. Backend 1306 OK, fundamentals gate green, frontend verify green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01T4wfz87k6kzJqkKLKZE7YL --- apps/backend/JSON_SCHEMA.md | 2 + apps/backend/analyze.py | 12 ++- apps/backend/analyze_audio_io.py | 29 +++++++ apps/backend/analyze_segments.py | 79 ++++++++++++------- .../tests/fixtures/golden/phase1_default.json | 4 +- apps/backend/tests/test_stem_aware_chords.py | 64 +++++++++++++++ apps/ui/src/services/backendPhase1Client.ts | 5 ++ apps/ui/src/types/measurement.ts | 8 ++ apps/ui/tests/fixtures/phase1FullPayload.ts | 3 +- 9 files changed, 173 insertions(+), 33 deletions(-) create mode 100644 apps/backend/tests/test_stem_aware_chords.py diff --git a/apps/backend/JSON_SCHEMA.md b/apps/backend/JSON_SCHEMA.md index b294e004..1e87206f 100644 --- a/apps/backend/JSON_SCHEMA.md +++ b/apps/backend/JSON_SCHEMA.md @@ -654,6 +654,7 @@ Type: `object \| null` | `chordDetail.chordTimeline` | `array<{startSec, endSec, label, labelLong, confidence}> \| null` | Phase 1.D #2 — chord segments with start/end times (seconds), short-form `label` (`"Cm"`, `"Eb"`, `"N"` no-chord), long-form `labelLong` (`"C minor"`, `"Eb major"`, `"N"`), and per-segment confidence. State path decoded by Viterbi over the L1 mass-overlap of `librosa.feature.chroma_cqt` against a 25-state vocabulary (12 major + 12 minor triads + 1 no-chord). Confidence is mean L2 cosine similarity between the chroma and the winning state's template across the segment's frames — bounded [0, 1], where 0.5 is the noise floor and 0.7+ indicates a clean chord match. Segments shorter than 250 ms are dropped; low-confidence "N" segments (<0.4) are also dropped as transition artifacts. Capped at 64 segments. | seconds; 0-1 confidence | Use for arrangement-aware harmonic recommendations ("the bridge sits on Cm for 8 bars, then drops to Eb for 4 — recommend a chord-stab MIDI sequence that matches the timeline"). When `confidence < 0.5` on a segment, hedge — chord detection on full-mix electronic material is noisy. | | `chordDetail.chordTimelineSource` | `string` | Phase 1.D #2 — identifier of the engine that produced `chordTimeline`. Currently always `"librosa_viterbi"`. Future-proofs swapping engines. | categorical | Phase 2 should hedge if this is anything other than the engine the prompt is calibrated against. | | `chordDetail.chordTimelineAgreement` | `boolean \| null` | Phase 1.D #2 — `true` when the most-frequent non-"N" Viterbi label in `chordTimeline` matches Essentia's `dominantChords[0]` after enharmonic normalization (`D# ↔ Eb`, `F# ↔ Gb`, etc.). `null` when either source has no usable label. | boolean | Strong hedging signal. When `false`, the two chord engines disagree — describe the harmonic content as uncertain and cite both readings. When `true`, you can commit to the timeline reading. | +| `chordDetail.chordSource` | `"full_mix" \| "harmonic_stems"` | Which audio the chroma ran on (accuracy program PR-B4). `harmonic_stems` = bass-removed stem mix (`other`+`vocals`), run when Demucs stems were available; `full_mix` = whole track otherwise. | categorical | `harmonic_stems` is the higher-accuracy path (the bassline is the biggest full-mix chroma polluter). Provenance only — not an override of the chord labels. | | `chordDetail.chordTimeline[].labelLong` | `string` | Long-form chord label paired with the short-form `label`: `"C major"`, `"F# minor"`, `"N"`. | chord labels | Use when a recommendation reads better with human-readable harmonic citations. | | `chordDetail.chordChangeCount` | `int` | Phase 1.D #2 — count of unique chord-to-chord transitions in the Viterbi-decoded timeline. Flat 1-chord tracks score 0; harmonically active tracks score 16+. | count | A proxy for "how harmonically active" the track is. Cite for arrangement-density and harmonic-rhythm recommendations. | @@ -666,6 +667,7 @@ Type: `array \| null` | `segmentKey[].segmentIndex` | `int` | Segment index aligned with `structure.segments`. | integer index | Use for joining harmonic data to arrangement segments. | | `segmentKey[].key` | `string \| null` | Per-segment key label (`edma` profile). | categorical | Detects section-level key drift or modal pivots. | | `segmentKey[].keyConfidence` | `float \| null` | Per-segment key confidence. | 0-1 (approx) | Low confidence means treat segment key as tentative. | +| `segmentKey[].source` | `"full_mix" \| "harmonic_stems"` | Which audio the per-segment key ran on (accuracy program PR-B4); same harmonic-isolation rationale as `chordDetail.chordSource`. | categorical | Provenance only. | --- diff --git a/apps/backend/analyze.py b/apps/backend/analyze.py index 66f82d01..49ffbbf3 100644 --- a/apps/backend/analyze.py +++ b/apps/backend/analyze.py @@ -63,6 +63,7 @@ _load_stem_stereo, analyze_crepe_pitch, cleanup_stems, + mix_stems_mono, ) from separation_backend import separate_stems_backend # noqa: E402 from analyze_estimate import ( # noqa: E402 @@ -1792,14 +1793,21 @@ def main(): sample_rate=sample_rate, ) ) - result.update(analyze_segment_key(result.get("structure"), mono, sample_rate)) + # Harmonic-source mix (bass/drums removed) for chord + key chroma when + # stems are available; None -> full-mix path, bit-identical to before. + harmonic_mono = mix_stems_mono(stems, ("other", "vocals"), sample_rate) + result.update( + analyze_segment_key( + result.get("structure"), mono, sample_rate, harmonic_mono=harmonic_mono + ) + ) # Stereo array no longer needed — release memory. del stereo gc.collect() # Chords - result.update(analyze_chords(mono, sample_rate)) + result.update(analyze_chords(mono, sample_rate, harmonic_mono=harmonic_mono)) # Perceptual result.update(analyze_perceptual(mono, sample_rate)) diff --git a/apps/backend/analyze_audio_io.py b/apps/backend/analyze_audio_io.py index 17e49c6d..0e78c9c8 100644 --- a/apps/backend/analyze_audio_io.py +++ b/apps/backend/analyze_audio_io.py @@ -47,6 +47,35 @@ def _load_stem_mono( return None +def mix_stems_mono( + stems: dict | None, + stem_names: tuple[str, ...], + sample_rate: int = 44100, +) -> np.ndarray | None: + """Sum the named Demucs stems into one mono array (harmonic-source mix). + + Used by chord/key analysis to run chroma on bass-removed audio: summing + ``other`` + ``vocals`` keeps the harmonic content while dropping the + bassline and drums that pollute full-mix chroma. Returns ``None`` when no + named stem is loadable, so callers fall back to the full mix unchanged. + Stems are equal-length (same source render), but guard for safety. + """ + loaded = [ + arr + for name in stem_names + if (arr := _load_stem_mono(stems, name, sample_rate=sample_rate)) is not None + ] + if not loaded: + return None + length = min(arr.shape[0] for arr in loaded) + if length == 0: + return None + mixed = np.zeros(length, dtype=np.float32) + for arr in loaded: + mixed += arr[:length] + return mixed + + def _load_stem_stereo( stems: dict | None, stem_name: str, diff --git a/apps/backend/analyze_segments.py b/apps/backend/analyze_segments.py index 84c0b591..e46d0dce 100644 --- a/apps/backend/analyze_segments.py +++ b/apps/backend/analyze_segments.py @@ -467,13 +467,24 @@ def analyze_segment_key( structure_data: dict | None, mono: np.ndarray, sample_rate: int = 44100, + *, + harmonic_mono: np.ndarray | None = None, ) -> dict: - """Compute key and confidence per segment using KeyExtractor.""" + """Compute key and confidence per segment using KeyExtractor. + + When ``harmonic_mono`` (bass-removed stem mix) is provided, per-segment + key runs on it instead of the full mix — same harmonic-isolation rationale + as ``analyze_chords``. Each entry records its ``source``. Falls back to the + full mix (bit-identical) when stems are unavailable. + """ + source = "harmonic_stems" if harmonic_mono is not None else "full_mix" try: if structure_data is None: return {"segmentKey": None} - mono_arr = np.asarray(mono, dtype=np.float32) + mono_arr = np.asarray( + harmonic_mono if harmonic_mono is not None else mono, dtype=np.float32 + ) if mono_arr.ndim != 1 or mono_arr.size == 0: return {"segmentKey": None} @@ -508,6 +519,7 @@ def analyze_segment_key( "segmentIndex": index, "key": key_value, "keyConfidence": key_confidence, + "source": source, } ) @@ -517,11 +529,41 @@ def analyze_segment_key( return {"segmentKey": None} -def analyze_chords(mono: np.ndarray, sample_rate: int = 44100) -> dict: - """Frame-wise HPCP analysis and chord detection via ChordsDetection.""" +def _empty_chord_detail(source: str) -> dict: + return { + "chordDetail": { + "chordSequence": [], + "chordStrength": 0.0, + "progression": [], + "dominantChords": [], + "chordTimeline": [], + "chordChangeCount": 0, + "chordTimelineSource": "librosa_viterbi", + "chordTimelineAgreement": None, + "chordSource": source, + } + } + + +def analyze_chords( + mono: np.ndarray, + sample_rate: int = 44100, + *, + harmonic_mono: np.ndarray | None = None, +) -> dict: + """Frame-wise HPCP analysis and chord detection via ChordsDetection. + + When ``harmonic_mono`` is provided (a bass-removed stem mix), chroma runs + on it instead of the full mix — the bassline is the biggest polluter of + full-mix chroma, so harmonic-source isolation lifts chord accuracy on + dense material. Falls back to the full mix (bit-identical to before) when + stems are unavailable. ``chordSource`` records which path ran. + """ + source = "harmonic_stems" if harmonic_mono is not None else "full_mix" try: + analysis_mono = harmonic_mono if harmonic_mono is not None else mono hp_filter = es.HighPass(cutoffFrequency=120, sampleRate=sample_rate) - mono_filtered = hp_filter(mono) + mono_filtered = hp_filter(analysis_mono) frame_size = 4096 hop_size = 2048 @@ -550,36 +592,14 @@ def analyze_chords(mono: np.ndarray, sample_rate: int = 44100) -> dict: continue if len(hpcp_sequence) == 0: - return { - "chordDetail": { - "chordSequence": [], - "chordStrength": 0.0, - "progression": [], - "dominantChords": [], - "chordTimeline": [], - "chordChangeCount": 0, - "chordTimelineSource": "librosa_viterbi", - "chordTimelineAgreement": None, - } - } + return _empty_chord_detail(source) chords, strength = chords_algo(np.asarray(hpcp_sequence, dtype=np.float32)) chords = [str(c) for c in chords] strength = np.asarray(strength, dtype=np.float64) if len(chords) == 0: - return { - "chordDetail": { - "chordSequence": [], - "chordStrength": 0.0, - "progression": [], - "dominantChords": [], - "chordTimeline": [], - "chordChangeCount": 0, - "chordTimelineSource": "librosa_viterbi", - "chordTimelineAgreement": None, - } - } + return _empty_chord_detail(source) # Keep payload manageable. if len(chords) > 32: @@ -650,6 +670,7 @@ def analyze_chords(mono: np.ndarray, sample_rate: int = 44100) -> dict: "chordChangeCount": chord_change_count, "chordTimelineSource": "librosa_viterbi", "chordTimelineAgreement": chord_timeline_agreement, + "chordSource": source, } } except Exception as e: diff --git a/apps/backend/tests/fixtures/golden/phase1_default.json b/apps/backend/tests/fixtures/golden/phase1_default.json index b31c866d..702f1da9 100644 --- a/apps/backend/tests/fixtures/golden/phase1_default.json +++ b/apps/backend/tests/fixtures/golden/phase1_default.json @@ -74,6 +74,7 @@ "chordDetail": { "chordChangeCount": "number", "chordSequence": "scalarList", + "chordSource": "str", "chordStrength": "number", "chordTimeline": { "[]": { @@ -367,7 +368,8 @@ "[]": { "key": "str", "keyConfidence": "number", - "segmentIndex": "number" + "segmentIndex": "number", + "source": "str" } }, "segmentLoudness": { diff --git a/apps/backend/tests/test_stem_aware_chords.py b/apps/backend/tests/test_stem_aware_chords.py new file mode 100644 index 00000000..cc208cb6 --- /dev/null +++ b/apps/backend/tests/test_stem_aware_chords.py @@ -0,0 +1,64 @@ +import os +import tempfile +import unittest +import wave + +import numpy as np + +from analyze_audio_io import mix_stems_mono +from analyze_segments import analyze_chords, analyze_segment_key + + +def _write_wav(path: str, samples: np.ndarray, sr: int = 44100) -> None: + pcm = np.clip(samples, -1.0, 1.0) + pcm16 = (pcm * 32767.0).astype(" None: + with tempfile.TemporaryDirectory() as d: + other = os.path.join(d, "other.wav") + vocals = os.path.join(d, "vocals.wav") + _write_wav(other, np.full(44100, 0.2, dtype=np.float32)) + _write_wav(vocals, np.full(44100, 0.1, dtype=np.float32)) + stems = {"other": other, "vocals": vocals, "bass": os.path.join(d, "nope.wav")} + mixed = mix_stems_mono(stems, ("other", "vocals")) + self.assertIsNotNone(mixed) + self.assertEqual(mixed.shape[0], 44100) + # 0.2 + 0.1 summed. + self.assertAlmostEqual(float(mixed.mean()), 0.3, places=2) + + def test_none_when_no_stem_loadable(self) -> None: + self.assertIsNone(mix_stems_mono(None, ("other", "vocals"))) + self.assertIsNone(mix_stems_mono({"bass": "/no/such.wav"}, ("other", "vocals"))) + + +class ChordSourceTaggingTests(unittest.TestCase): + def _tone(self, freqs, seconds=2.0, sr=44100): + t = np.arange(int(seconds * sr)) / sr + sig = sum(np.sin(2 * np.pi * f * t) for f in freqs) + return (sig / np.max(np.abs(sig)) * 0.8).astype(np.float32) + + def test_chord_source_reflects_harmonic_input(self) -> None: + full = self._tone([261.6, 329.6, 392.0]) # C major triad + harmonic = self._tone([261.6, 329.6, 392.0]) + default = analyze_chords(full)["chordDetail"] + self.assertEqual(default["chordSource"], "full_mix") + stemmed = analyze_chords(full, harmonic_mono=harmonic)["chordDetail"] + self.assertEqual(stemmed["chordSource"], "harmonic_stems") + + def test_segment_key_source_tagged(self) -> None: + mono = self._tone([261.6, 329.6, 392.0], seconds=4.0) + structure = {"segments": [{"start": 0.0, "end": 4.0}]} + out = analyze_segment_key(structure, mono, harmonic_mono=mono)["segmentKey"] + if out: # segmentation may return None on trivial structure; tag when present + self.assertTrue(all(e["source"] == "harmonic_stems" for e in out)) + + +if __name__ == "__main__": + unittest.main() diff --git a/apps/ui/src/services/backendPhase1Client.ts b/apps/ui/src/services/backendPhase1Client.ts index 3c37adac..3a6bab1d 100644 --- a/apps/ui/src/services/backendPhase1Client.ts +++ b/apps/ui/src/services/backendPhase1Client.ts @@ -944,6 +944,10 @@ function parseOptionalChordDetail(value: unknown): ChordDetail | null { : null; const chordTimeline = parseOptionalChordTimeline(value.chordTimeline); + const chordSource = + value.chordSource === "harmonic_stems" || value.chordSource === "full_mix" + ? value.chordSource + : undefined; return { chordSequence, @@ -954,6 +958,7 @@ function parseOptionalChordDetail(value: unknown): ChordDetail | null { chordChangeCount, chordTimelineSource, chordTimelineAgreement, + chordSource, }; } diff --git a/apps/ui/src/types/measurement.ts b/apps/ui/src/types/measurement.ts index 27224aa3..7cef0040 100644 --- a/apps/ui/src/types/measurement.ts +++ b/apps/ui/src/types/measurement.ts @@ -448,6 +448,8 @@ export interface SegmentKeyEntry { segmentIndex: number; key: string | null; keyConfidence?: number | null; + /** Which audio the per-segment key ran on (accuracy program PR-B4). */ + source?: "full_mix" | "harmonic_stems"; } /** @@ -479,6 +481,12 @@ export interface ChordDetail { chordTimelineSource?: string | null; /** Phase 1.D #2: true when Viterbi's dominant matches Essentia's dominantChords[0] after enharmonic normalization. */ chordTimelineAgreement?: boolean | null; + /** + * Which audio the chroma ran on (accuracy program PR-B4): `harmonic_stems` + * = bass-removed stem mix (higher chord accuracy on dense material), + * `full_mix` = whole track (when stems were unavailable). + */ + chordSource?: "full_mix" | "harmonic_stems"; } export interface PerceptualDetail { diff --git a/apps/ui/tests/fixtures/phase1FullPayload.ts b/apps/ui/tests/fixtures/phase1FullPayload.ts index e7d2f597..74887542 100644 --- a/apps/ui/tests/fixtures/phase1FullPayload.ts +++ b/apps/ui/tests/fixtures/phase1FullPayload.ts @@ -575,7 +575,7 @@ export const phase1EnvelopeFixture = { }, ], segmentStereo: [{ segmentIndex: 0, stereoWidth: 0.8, stereoCorrelation: 0.9 }], - segmentKey: [{ segmentIndex: 0, key: 'A minor', keyConfidence: 0.85 }], + segmentKey: [{ segmentIndex: 0, key: 'A minor', keyConfidence: 0.85, source: 'full_mix' }], essentiaFeatures: { zeroCrossingRate: 0.12, hfc: 0.45, @@ -594,6 +594,7 @@ export const phase1EnvelopeFixture = { chordChangeCount: 1, chordTimelineSource: 'librosa_viterbi', chordTimelineAgreement: true, + chordSource: 'harmonic_stems', }, perceptual: { sharpness: 1.42,