diff --git a/CLAUDE.md b/CLAUDE.md index 97ac113d..0359ed3b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -193,18 +193,19 @@ Single-page React 19 + Vite + TypeScript + Tailwind CSS v4 app with no router. V 3. **`src/services/backendPhase1Client.ts`**: Legacy multipart transport (typed error classes, `AbortController` timeouts, identity probe via `/openapi.json`). Kept for the compatibility wrappers; new flows should go through `analysisRunsClient.ts`. 4. **`src/services/httpClient.ts`**: Shared fetch helpers and request-header injection used by the run/artifact/sample clients. 5. **`src/services/spectralArtifactsClient.ts`**: Fetches spectrogram/spectral-evolution artifacts via `/api/analysis-runs/{run_id}/artifacts/…`. -6. **`src/services/sampleGenerationClient.ts`**: Phase 3 audition-sample POST/GET against `/api/analysis-runs/{run_id}/samples`, plus per-clip artifact streaming. -7. **`src/services/mixDoctor.ts`**: Mix advisory logic — client-side scoring and suggestions against measured spectral balance. -8. **`src/services/phase2Validator.ts`** + **`loudnessGuardrails.ts`**: Runtime guardrail. Validates Phase 2 consistency against Phase 1 (`validateBPMConsistency`, `validateKeyConsistency`, `validateLUFSConsistency`, `validateGenreDSPConsistency`, `validateNumericBounds`, `validateLoudnessActionPresence`). `loudnessGuardrails.ts` defines the objective loudness defects (digital clipping via `saturationDetail.clippedSampleCount`, true-peak overs via `truePeak`) that a Phase 2 mastering/dynamics card *must* address — a missing action surfaces as a `MISSING_LOUDNESS_ACTION` violation. The aggregate `validatePhase2Consistency` drives **`Phase2ConsistencyReport.tsx`**, which renders the chain-of-custody report on the results surface (`AnalysisResults.tsx`, `hideWhenClean`) and in full inside the diagnostic log (`DiagnosticLog.tsx`); `App.tsx` computes the report and passes it down. -9. **`src/services/appliedRecommendations.ts`** + **`userLabels.ts`**: Applied-recommendations tracker and persisted-label state used by the audit overhaul. +6. **`src/services/transcriptionPianorollClient.ts`**: Fetches the velocity-encoded transcription pianoroll matrix via `/api/analysis-runs/{run_id}/transcription/pianoroll`. Backed by `apps/backend/transcription_pianoroll.py`. Rendered by `TranscriptionPianoroll.tsx` (canvas heatmap) inside `TranscriptionPianorollBlock.tsx`, which `AnalysisResults.tsx` mounts in the Session Musician suite when `transcriptionDetail.noteCount > 0`. +7. **`src/services/sampleGenerationClient.ts`**: Phase 3 audition-sample POST/GET against `/api/analysis-runs/{run_id}/samples`, plus per-clip artifact streaming. +8. **`src/services/mixDoctor.ts`**: Mix advisory logic — client-side scoring and suggestions against measured spectral balance. +9. **`src/services/phase2Validator.ts`** + **`loudnessGuardrails.ts`**: Runtime guardrail. Validates Phase 2 consistency against Phase 1 (`validateBPMConsistency`, `validateKeyConsistency`, `validateLUFSConsistency`, `validateGenreDSPConsistency`, `validateNumericBounds`, `validateLoudnessActionPresence`). `loudnessGuardrails.ts` defines the objective loudness defects (digital clipping via `saturationDetail.clippedSampleCount`, true-peak overs via `truePeak`) that a Phase 2 mastering/dynamics card *must* address — a missing action surfaces as a `MISSING_LOUDNESS_ACTION` violation. The aggregate `validatePhase2Consistency` drives **`Phase2ConsistencyReport.tsx`**, which renders the chain-of-custody report on the results surface (`AnalysisResults.tsx`, `hideWhenClean`) and in full inside the diagnostic log (`DiagnosticLog.tsx`); `App.tsx` computes the report and passes it down. +10. **`src/services/appliedRecommendations.ts`** + **`userLabels.ts`**: Applied-recommendations tracker and persisted-label state used by the audit overhaul. 9a. **`src/services/recommendationVerification.ts`** + **`src/data/recommendationVerification.ts`** + **`components/RecommendationVerificationBadge.tsx`**: Per-recommendation corpus-verification badge (`GOAL.md` sub-goal 4). The data module is a generated artifact from `apps/backend/scripts/evaluate_recommendations.py --verification-artifact` (all-`NONE` until the ground-truth corpus has renders); the service infers a card's domain (mirroring the backend scorer's `infer_domain`) and looks up its confidence band; the badge renders on `AnalysisResults.tsx` recommendation cards, hidden when there is no corpus evidence. Research/proof surface — see [`apps/backend/NEEDS.md`](apps/backend/NEEDS.md). -10. **`src/services/phase1Picker.ts`** + **`phaseLabels.ts`**: Phase-snapshot projection helpers consumed by the results surface. -11. **`src/services/audioFile.ts`**: Client-side audio validation, blank-MIME extension fallback, preview-URL lifecycle. -12. **`src/services/fieldAnalytics.ts`** + **`diagnosticLogs.ts`**: Instrumentation hooks and diagnostic-log capture for the request panel. -13. **`src/services/midi/`**: MIDI export, preview, and quantization utilities (`midiExport.ts`, `midiPreview.ts`, `quantization.ts`). -14. **`src/services/sessionMusician/`**: Session Musician helpers — `confidenceBand.ts`, `noteConversion.ts`, `renderState.ts`, `stemListeningNotes.ts`. -15. **`src/types.ts`** + **`src/types/`**: `types.ts` is a barrel re-export of `./types/{measurement,interpretation,backend}.ts`. `Phase1Result` lives in `types/measurement.ts`; `AnalysisRunSnapshot` in `types/backend.ts`; `Phase2Result` in `types/interpretation.ts`; `./types/samples.ts` is imported directly, not through the barrel. -16. **`src/config.ts`**: Runtime resolution of `VITE_API_BASE_URL` and feature flags; falls back to `http://127.0.0.1:8100`. Supports window-level overrides (`window.__VITE_API_BASE_URL_OVERRIDE__`, `window.__VITE_ENABLE_PHASE2_GEMINI_OVERRIDE__`) for hosted deployments that inject config at runtime without a rebuild. +11. **`src/services/phase1Picker.ts`** + **`phaseLabels.ts`**: Phase-snapshot projection helpers consumed by the results surface. +12. **`src/services/audioFile.ts`**: Client-side audio validation, blank-MIME extension fallback, preview-URL lifecycle. +13. **`src/services/fieldAnalytics.ts`** + **`diagnosticLogs.ts`**: Instrumentation hooks and diagnostic-log capture for the request panel. +14. **`src/services/midi/`**: MIDI export, preview, and quantization utilities (`midiExport.ts`, `midiPreview.ts`, `quantization.ts`). +15. **`src/services/sessionMusician/`**: Session Musician helpers — `confidenceBand.ts`, `noteConversion.ts`, `renderState.ts`, `stemListeningNotes.ts`. +16. **`src/types.ts`** + **`src/types/`**: `types.ts` is a barrel re-export of `./types/{measurement,interpretation,backend}.ts`. `Phase1Result` lives in `types/measurement.ts`; `AnalysisRunSnapshot` in `types/backend.ts`; `Phase2Result` in `types/interpretation.ts`; `./types/samples.ts` is imported directly, not through the barrel. +17. **`src/config.ts`**: Runtime resolution of `VITE_API_BASE_URL` and feature flags; falls back to `http://127.0.0.1:8100`. Supports window-level overrides (`window.__VITE_API_BASE_URL_OVERRIDE__`, `window.__VITE_ENABLE_PHASE2_GEMINI_OVERRIDE__`) for hosted deployments that inject config at runtime without a rebuild. `AnalysisResults.tsx` is the large results surface, lazy-loaded via Suspense. Manual vendor chunks in `vite.config.ts` control bundle splitting. diff --git a/apps/backend/analyze_rhythm.py b/apps/backend/analyze_rhythm.py index e813e56b..377cc554 100644 --- a/apps/backend/analyze_rhythm.py +++ b/apps/backend/analyze_rhythm.py @@ -456,7 +456,11 @@ def analyze_melody( midi_file_path = None try: - import mido + # Symusic handles the tick math and event ordering — we just build + # a Score in seconds and dump. Failure (e.g. an unwritable output + # dir) is non-critical: melody MIDI is a nice-to-have artifact, so + # we swallow the exception and report no file. + from symusic import Note, Score, Tempo, Track bpm = 120.0 if rhythm_data is not None and rhythm_data.get("bpm") is not None: @@ -464,42 +468,25 @@ def analyze_melody( if not np.isfinite(bpm) or bpm <= 0: bpm = 120.0 - ppq = 96 - ticks_per_second = (ppq * bpm) / 60.0 - midi_out = mido.MidiFile(ticks_per_beat=ppq) - track = mido.MidiTrack() - midi_out.tracks.append(track) - track.append( - mido.MetaMessage("set_tempo", tempo=int(mido.bpm2tempo(bpm)), time=0) - ) - - events = [] + score = Score(96, ttype="Second") + score.tempos.append(Tempo(time=0.0, qpm=float(bpm), ttype="Second")) + track = Track(name="melody", program=0, is_drum=False, ttype="Second") for onset, duration, midi_note in note_events: - start_tick = max(0, int(round(onset * ticks_per_second))) - end_tick = max( - start_tick + 1, int(round((onset + duration) * ticks_per_second)) - ) - events.append((start_tick, 1, midi_note)) - events.append((end_tick, 0, midi_note)) - events.sort(key=lambda e: (e[0], e[1])) - - prev_tick = 0 - for tick, is_note_on, midi_note in events: - delta = max(0, tick - prev_tick) - if is_note_on == 1: - track.append( - mido.Message("note_on", note=midi_note, velocity=90, time=delta) + track.notes.append( + Note( + time=float(onset), + duration=float(max(0.0, duration)), + pitch=int(midi_note), + velocity=90, + ttype="Second", ) - else: - track.append( - mido.Message("note_off", note=midi_note, velocity=0, time=delta) - ) - prev_tick = tick + ) + score.tracks.append(track) output_dir = os.path.dirname(audio_path) base_name = os.path.splitext(os.path.basename(audio_path))[0] midi_file_path = os.path.join(output_dir, f"{base_name}_melody.mid") - midi_out.save(midi_file_path) + score.dump_midi(midi_file_path) except Exception as e: print(f"[warn] Melody MIDI export failed: {e}", file=sys.stderr) midi_file_path = None diff --git a/apps/backend/beat_evaluation.py b/apps/backend/beat_evaluation.py index 99250f65..5734c8a0 100644 --- a/apps/backend/beat_evaluation.py +++ b/apps/backend/beat_evaluation.py @@ -16,6 +16,15 @@ beat_this and mir_eval are OPTIONAL: imported lazily/guarded so stride-vs-kick_accent runs in the product venv with zero new dependencies (metrics fall back to a hand-rolled F-measure when mir_eval is absent). + +Note on symusic (post-PR-A/D investigation): symusic's ``Score.get_beats`` / +``get_downbeats`` derive beats from a symbolic score's *tempo map and time +signatures*, not from audio. This harness operates on audio inputs, so symusic +has nothing to offer as an additional method here. The original consolidation +plan (PR-E) called for adding ``symusic-beat`` as an observational fourth +method — it was dropped after the probe confirmed symusic is not an +audio-analysis library. The pre-registered ``beat_this`` vs ``kick_accent`` +gate is unaffected. """ import json diff --git a/apps/backend/polyphonic_evaluation.py b/apps/backend/polyphonic_evaluation.py index 78131924..aaf37c5e 100644 --- a/apps/backend/polyphonic_evaluation.py +++ b/apps/backend/polyphonic_evaluation.py @@ -17,8 +17,8 @@ from pathlib import Path from typing import Any, Callable -import pretty_midi import soundfile as sf +from symusic import Score REPO_DIR = Path(__file__).resolve().parent DEFAULT_OUTPUT_DIR = REPO_DIR / ".runtime" / "polyphonic_eval" @@ -53,8 +53,10 @@ def build_manual_scorecard(existing: dict[str, Any] | None = None) -> dict[str, def summarize_midi_file(midi_path: Path, audio_duration_seconds: float) -> dict[str, Any]: - midi_data = pretty_midi.PrettyMIDI(str(midi_path)) - notes = [note for instrument in midi_data.instruments for note in instrument.notes] + # Loaded scores arrive in Tick units; convert to Second so note.time and + # note.duration are seconds, matching the metric definitions below. + score = Score(midi_path).to("Second") + notes = [note for track in score.tracks for note in track.notes] if len(notes) == 0: return { "noteCount": 0, @@ -72,11 +74,13 @@ def summarize_midi_file(midi_path: Path, audio_duration_seconds: float) -> dict[ pitch_values: list[int] = [] events: list[tuple[float, int]] = [] for note in notes: - duration = max(0.0, float(note.end) - float(note.start)) + start = float(note.time) + duration = max(0.0, float(note.duration)) + end = start + duration total_note_duration += duration pitch_values.append(int(note.pitch)) - events.append((float(note.start), 1)) - events.append((float(note.end), -1)) + events.append((start, 1)) + events.append((end, -1)) events.sort(key=lambda item: (item[0], item[1])) max_polyphony = 0 diff --git a/apps/backend/requirements.txt b/apps/backend/requirements.txt index bb6278cd..91fa9f57 100644 --- a/apps/backend/requirements.txt +++ b/apps/backend/requirements.txt @@ -31,7 +31,6 @@ lazy-loader==0.5 librosa==0.11.0 matplotlib==3.10.8 llvmlite==0.46.0 -mido==1.3.3 mpmath==1.3.0 msgpack==1.1.2 networkx==3.6.1 @@ -42,7 +41,6 @@ openunmix==1.3.0 packaging==26.0 platformdirs==4.9.4 pooch==1.9.0 -pretty_midi==0.2.11 protobuf==7.34.0 pyaml==26.2.1 pycparser==3.0 diff --git a/apps/backend/sample_generation.py b/apps/backend/sample_generation.py index 69e97982..182ad21c 100644 --- a/apps/backend/sample_generation.py +++ b/apps/backend/sample_generation.py @@ -50,15 +50,15 @@ def generate_samples( phase2: dict[str, Any] | None, output_dir: Path, pitch_note_hints: list[int] | None = None, - prefer_fluidsynth: bool = True, + allow_soundfont_backends: bool = True, ) -> GenerationResult: """Build the full audition set for a single run. `pitch_note_hints` may carry scale degrees pulled from the stem-summary pitch/note translation output. When absent we render the default ascent. - `prefer_fluidsynth` is plumbed through so tests can force the fallback - even if FluidSynth is installed on the dev machine. + `allow_soundfont_backends` is plumbed through so tests can force the + sine fallback even if FluidSynth (or symusic) could otherwise run. """ output_dir.mkdir(parents=True, exist_ok=True) artifact_paths: dict[str, Path] = {} @@ -86,7 +86,7 @@ def generate_samples( chord_plan = sample_theory.plan_chord_progression(ctx, bars=8) chord_result = sample_synthesis.render_clip( - chord_plan, prefer_fluidsynth=prefer_fluidsynth + chord_plan, allow_soundfont_backends=allow_soundfont_backends ) selected_backend = chord_result.backend selected_soundfont = chord_result.soundfont_path @@ -118,7 +118,7 @@ def generate_samples( bass_plan = sample_theory.plan_bass_root(ctx, bars=8) bass_result = sample_synthesis.render_clip( - bass_plan, prefer_fluidsynth=prefer_fluidsynth + bass_plan, allow_soundfont_backends=allow_soundfont_backends ) # `selected_backend` is already pinned by the chord render above — # `render_clip` always returns a concrete backend string, so the bass @@ -242,7 +242,7 @@ def generate_samples( ) if melody_plan is not None: melody_result = sample_synthesis.render_clip( - melody_plan, prefer_fluidsynth=prefer_fluidsynth + melody_plan, allow_soundfont_backends=allow_soundfont_backends ) # See bass-render note above — `selected_backend` is pinned at the # first chord render and `render_clip` never returns falsy. diff --git a/apps/backend/sample_synthesis.py b/apps/backend/sample_synthesis.py index b81a1547..c492d72d 100644 --- a/apps/backend/sample_synthesis.py +++ b/apps/backend/sample_synthesis.py @@ -1,17 +1,35 @@ """MIDI plan → WAV rendering for audition samples. -Two render paths: - -1. **FluidSynth** (`pyfluidsynth` + a General MIDI soundfont) — high-quality. - Used when both the Python binding and a soundfont file are available. -2. **Sine-additive fallback** — pure NumPy. Always available; in-tune; raw. - -Both paths produce the same float32 mono 44.1 kHz numpy array, so callers -don't need to branch on which backend was selected. The selected backend is -recorded on `RenderResult.backend` so it can flow into the manifest. - -MIDI artifacts are emitted via `pretty_midi`, which is already a hard dep, so -the user can drop a `.mid` into Ableton even if the audio render is rough. +Three render backends, picked under the default ``auto`` policy in this order +(conservative — operators with a working FluidSynth keep getting FluidSynth +output, since the cross-backend parity has not been measured): + +1. **FluidSynth** (``pyfluidsynth`` + SF2/SF3 soundfont) — the historically + shipping path and the reference rendering for users with a tuned + ``SONIC_ANALYZER_SOUNDFONT``. Preferred under ``auto`` when both the + binding and a soundfont are reachable. +2. **symusic Synthesizer** (Prestosynth + SF2/SF3 soundfont) — only reached + under ``auto`` when FluidSynth is *not* importable but a soundfont is + still locatable. Otherwise opt in via ``ASA_SAMPLE_SYNTH_BACKEND=symusic``. +3. **Sine-additive fallback** — pure NumPy. Always available; in-tune; raw. + +All three produce the same float32 mono 44.1 kHz numpy array, so callers +don't need to branch on which backend ran. The selected backend is recorded +on ``RenderResult.backend`` so it can flow into the citation manifest +(samples cite which synth rendered them, preserving chain-of-custody on the +audition surface). + +The ``ASA_SAMPLE_SYNTH_BACKEND`` env var (values: ``auto``, ``symusic``, +``fluidsynth``, ``sine``) overrides the precedence. ``auto`` is the shipping +default. An operator who's measured symusic parity locally can pin +``symusic`` to take advantage of the faster Prestosynth path. + +MIDI artifacts are emitted via ``symusic`` (fast C++ core), so the user can +drop a ``.mid`` into Ableton even if the audio render is rough. The MIDI file +is a spec-conformant artifact that DAWs read as a fixed format — switching +writers is a non-audible change, but the test in +``tests/test_sample_synthesis.WriteMidiTests`` round-trips via ``pretty_midi`` +to keep the parity contract honest. """ from __future__ import annotations @@ -23,8 +41,8 @@ from typing import Literal import numpy as np -import pretty_midi # type: ignore[import-untyped] import soundfile as sf +from symusic import Note, Score, Synthesizer, Tempo, Track from sample_theory import ClipPlan @@ -41,7 +59,7 @@ except Exception as exc: # pragma: no cover - exercised when pyfluidsynth absent fluidsynth = None _FLUIDSYNTH_IMPORTABLE = False - logger.info("pyfluidsynth not importable (%s); using sine fallback.", exc) + logger.info("pyfluidsynth not importable (%s); skipping FluidSynth path.", exc) _DEFAULT_SOUNDFONT_CANDIDATES: tuple[Path, ...] = ( @@ -53,7 +71,10 @@ ) -Backend = Literal["fluidsynth", "sine_fallback"] +Backend = Literal["symusic", "fluidsynth", "sine_fallback"] + +_BACKEND_ENV_VAR = "ASA_SAMPLE_SYNTH_BACKEND" +_VALID_BACKEND_OVERRIDES = {"auto", "symusic", "fluidsynth", "sine"} @dataclass(frozen=True) @@ -79,27 +100,92 @@ def locate_soundfont(explicit: Path | str | None = None) -> Path | None: return None +def _read_backend_override() -> str: + raw = os.environ.get(_BACKEND_ENV_VAR, "auto").strip().lower() + if raw not in _VALID_BACKEND_OVERRIDES: + logger.warning( + "Ignoring unknown %s=%r; valid values: %s.", + _BACKEND_ENV_VAR, + raw, + sorted(_VALID_BACKEND_OVERRIDES), + ) + return "auto" + return raw + + +def _resolve_backend( + *, + soundfont_path: Path | None, + allow_soundfont_backends: bool, +) -> Backend: + """Pick the backend the next render should use. + + ``allow_soundfont_backends=False`` is the test/escape-hatch flag: force + the deterministic sine path regardless of what's installed. The env var + still wins when set, so an operator can explicitly pin a backend even + while passing ``allow_soundfont_backends=False`` — that combination is + treated as the operator's deliberate choice. + """ + override = _read_backend_override() + if override == "sine" or not allow_soundfont_backends: + return "sine_fallback" + + if override == "symusic": + return "symusic" if soundfont_path is not None else "sine_fallback" + if override == "fluidsynth": + if _FLUIDSYNTH_IMPORTABLE and soundfont_path is not None: + return "fluidsynth" + return "sine_fallback" + + # ``auto`` — conservative precedence. FluidSynth keeps its long-standing + # default role for operators who tuned their setup against it; symusic + # only takes over when FluidSynth isn't importable but a soundfont is + # still reachable. Cross-backend audio parity has not been measured, so + # silently switching the rendering engine for working installs is the + # specific regression risk we're avoiding here. + if soundfont_path is None: + return "sine_fallback" + if _FLUIDSYNTH_IMPORTABLE: + return "fluidsynth" + return "symusic" + + def render_clip( plan: ClipPlan, *, soundfont: Path | str | None = None, - prefer_fluidsynth: bool = True, + allow_soundfont_backends: bool = True, ) -> RenderResult: """Render a ClipPlan to an in-memory float32 audio buffer. - Picks the best available backend. Caller can force the fallback by passing - `prefer_fluidsynth=False`; tests use that to exercise the fallback path - even when FluidSynth happens to be installed. + Picks the best available backend using :func:`_resolve_backend`. Pass + ``allow_soundfont_backends=False`` to force the sine path (tests do this + to keep coverage deterministic regardless of what's installed locally). + Each soundfont backend has the sine path as its own backstop — if the + selected backend raises at runtime, the call still returns a buffer. """ - if prefer_fluidsynth and _FLUIDSYNTH_IMPORTABLE: - sf_path = locate_soundfont(soundfont) - if sf_path is not None: - try: - return _render_with_fluidsynth(plan, sf_path) - except Exception as exc: # pragma: no cover - hard to trigger in CI - logger.warning( - "FluidSynth render failed (%s); falling back to sine synth.", exc - ) + sf_path = locate_soundfont(soundfont) + backend = _resolve_backend( + soundfont_path=sf_path, + allow_soundfont_backends=allow_soundfont_backends, + ) + + if backend == "fluidsynth" and sf_path is not None and _FLUIDSYNTH_IMPORTABLE: + try: + return _render_with_fluidsynth(plan, sf_path) + except Exception as exc: # pragma: no cover - hard to trigger in CI + logger.warning( + "FluidSynth render failed (%s); falling back to sine.", exc + ) + + if backend == "symusic" and sf_path is not None: + try: + return _render_with_symusic_synth(plan, sf_path) + except Exception as exc: # pragma: no cover - hard to trigger in CI + logger.warning( + "symusic render failed (%s); falling back to sine.", exc + ) + return _render_with_sine_fallback(plan) @@ -111,24 +197,101 @@ def write_wav(samples: np.ndarray, *, path: Path, sample_rate: int = SAMPLE_RATE def write_midi(plan: ClipPlan, *, path: Path) -> None: - """Emit a MIDI file from a ClipPlan so users can audition in Ableton.""" - pm = pretty_midi.PrettyMIDI(initial_tempo=plan.tempo_bpm) - inst = pretty_midi.Instrument(program=plan.program) + """Emit a MIDI file from a ClipPlan so users can audition in Ableton. + + Uses ``symusic`` (C++ core) so the backend has a single canonical MIDI + library; the output is a spec-conformant Standard MIDI file that any DAW + parses identically. Tempo is emitted at ``t=0`` to match the static plan, + and the GM program is set on the track header so Ableton picks the right + default sound. + """ + score = Score(480, ttype="Second") + score.tempos.append(Tempo(time=0.0, qpm=float(plan.tempo_bpm), ttype="Second")) + track = Track( + name="audition", program=int(plan.program), is_drum=False, ttype="Second" + ) beats_per_second = plan.tempo_bpm / 60.0 for note in plan.notes: start_seconds = note.start_beat / beats_per_second - end_seconds = (note.start_beat + note.duration_beats) / beats_per_second - inst.notes.append( - pretty_midi.Note( - velocity=int(np.clip(note.velocity, 1, 127)), + duration_seconds = max(0.0, note.duration_beats / beats_per_second) + track.notes.append( + Note( + time=float(start_seconds), + duration=float(duration_seconds), pitch=int(np.clip(note.pitch_midi, 0, 127)), - start=float(start_seconds), - end=float(end_seconds), + velocity=int(np.clip(note.velocity, 1, 127)), + ttype="Second", ) ) - pm.instruments.append(inst) + score.tracks.append(track) path.parent.mkdir(parents=True, exist_ok=True) - pm.write(str(path)) + score.dump_midi(str(path)) + + +# --- symusic Synthesizer path ---------------------------------------------- # + + +def _build_score_from_plan(plan: ClipPlan) -> Score: + """Build a Second-unit Score from a ClipPlan. + + Shared by the symusic synth and ``write_midi`` to keep the plan-to-Score + translation in one place; future plan extensions only need updating here. + """ + score = Score(480, ttype="Second") + score.tempos.append(Tempo(time=0.0, qpm=float(plan.tempo_bpm), ttype="Second")) + track = Track( + name="audition", program=int(plan.program), is_drum=False, ttype="Second" + ) + beats_per_second = plan.tempo_bpm / 60.0 + for note in plan.notes: + start_seconds = note.start_beat / beats_per_second + duration_seconds = max(0.0, note.duration_beats / beats_per_second) + track.notes.append( + Note( + time=float(start_seconds), + duration=float(duration_seconds), + pitch=int(np.clip(note.pitch_midi, 0, 127)), + velocity=int(np.clip(note.velocity, 1, 127)), + ttype="Second", + ) + ) + score.tracks.append(track) + return score + + +def _render_with_symusic_synth( + plan: ClipPlan, soundfont_path: Path +) -> RenderResult: # pragma: no cover - exercised only when a real SF2/SF3 is available locally + """Render a ClipPlan via symusic.Synthesizer (Prestosynth backend). + + Returns the same float32 mono 44.1 kHz buffer shape as the other paths so + the caller doesn't have to branch. + """ + score = _build_score_from_plan(plan) + synth = Synthesizer(str(soundfont_path), sample_rate=SAMPLE_RATE, quality=0) + audio = synth.render(score, stereo=False) + samples = np.asarray(audio, dtype=np.float32) + + # Symusic returns 1D mono when stereo=False, but be defensive in case a + # future version changes the shape — we always reduce to a 1D float32. + if samples.ndim == 2: + # Average across the non-time axis. With 2 channels in either layout, + # the shorter axis is the channel axis. + channel_axis = 0 if samples.shape[0] <= samples.shape[1] else 1 + samples = samples.mean(axis=channel_axis).astype(np.float32) + elif samples.ndim != 1: + raise RuntimeError( + f"symusic.Synthesizer.render returned unexpected shape {samples.shape}" + ) + + seconds_total = (plan.duration_beats / plan.tempo_bpm) * 60.0 + return RenderResult( + samples=samples, + sample_rate=SAMPLE_RATE, + backend="symusic", + soundfont_path=str(soundfont_path), + duration_seconds=seconds_total, + ) # --- FluidSynth path -------------------------------------------------------- # diff --git a/apps/backend/scripts/import_midi_to_ground_truth.py b/apps/backend/scripts/import_midi_to_ground_truth.py index e75a1f59..0e678af6 100755 --- a/apps/backend/scripts/import_midi_to_ground_truth.py +++ b/apps/backend/scripts/import_midi_to_ground_truth.py @@ -13,16 +13,16 @@ import sys from pathlib import Path -import pretty_midi +from symusic import Score -def _flatten_notes(midi: pretty_midi.PrettyMIDI, offset_seconds: float) -> list[dict]: +def _flatten_notes(score: Score, offset_seconds: float) -> list[dict]: + """Flatten all notes in a Second-unit Score into the ground-truth shape.""" notes: list[dict] = [] - for instrument in midi.instruments: - for note in instrument.notes: - start = float(note.start) + offset_seconds - end = float(note.end) + offset_seconds - duration = max(0.0, end - start) + for track in score.tracks: + for note in track.notes: + start = float(note.time) + offset_seconds + duration = max(0.0, float(note.duration)) notes.append( { "pitchMidi": int(note.pitch), @@ -108,12 +108,14 @@ def main() -> None: raise SystemExit(2) try: - midi = pretty_midi.PrettyMIDI(str(args.midi_path)) + # Score loads as Tick by default; convert so note.time / note.duration + # are in seconds for the ground-truth schema. + score = Score(args.midi_path).to("Second") except Exception as exc: print(f"error: failed to parse MIDI: {exc}", file=sys.stderr) raise SystemExit(2) from exc - notes = _flatten_notes(midi, args.offset_seconds) + notes = _flatten_notes(score, args.offset_seconds) overlaps = _detect_overlaps(notes) if len(overlaps) > 0: diff --git a/apps/backend/scripts/parity_probe_synth_backends.py b/apps/backend/scripts/parity_probe_synth_backends.py new file mode 100755 index 00000000..3962fe5d --- /dev/null +++ b/apps/backend/scripts/parity_probe_synth_backends.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""FluidSynth vs symusic.Synthesizer parity probe. + +The PR-F audio-backend swap (`apps/backend/sample_synthesis.py`) wires symusic +as an optional audition-sample renderer behind ``ASA_SAMPLE_SYNTH_BACKEND``. +This script gives a maintainer the cheapest possible way to verify the two +backends produce comparable audio on the same SoundFont before flipping the +auto default away from FluidSynth. + +What it does: + +1. Builds a deterministic ``ClipPlan`` (a single-octave C-major triad + sustained over 4 beats at 120 BPM — short enough to render quickly, rich + enough that any voice-leveling or pitching drift between backends shows + up in the spectrum). +2. Renders the same plan through both backends with the same soundfont. +3. Compares mono float32 buffers along three axes: RMS (overall loudness), + peak (clipping headroom), and spectral centroid (timbral character). +4. Emits a JSON report at ``apps/backend/.runtime/parity/synth_parity.json`` + with the deltas and a pass/fail verdict using the documented tolerances. + +The script is intentionally **non-prescriptive about adoption**. Passing +parity does not flip the auto default — that's a separate code change. A +failing run is just evidence the engines diverge enough that the default +swap would be audible. + +Skip behaviour: + + * pyfluidsynth missing → exit 0, status ``skipped_no_fluidsynth``. + * No SF2/SF3 reachable via ``SONIC_ANALYZER_SOUNDFONT`` or the known + candidate paths → exit 0, status ``skipped_no_soundfont``. + * symusic should always be importable (it's a hard dep); if absent, the + script raises rather than skips because that's a deeper integrity issue. + +Usage: + + ./venv/bin/python scripts/parity_probe_synth_backends.py \\ + [--soundfont /path/to/font.sf2] \\ + [--out-dir .runtime/parity] +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np + +REPO_DIR = Path(__file__).resolve().parent.parent +if str(REPO_DIR) not in sys.path: + sys.path.insert(0, str(REPO_DIR)) + +import sample_synthesis # noqa: E402 +from sample_theory import ClipPlan, NoteEvent # noqa: E402 + +# Tolerance bands. These are loose by design — soundfont-driven synths +# legitimately differ in voice rendering, envelope shapes, and reverb tails. +# The intent is "audible parity at typical monitor levels", not "bit-exact". +RMS_DB_TOLERANCE = 1.0 +PEAK_DB_TOLERANCE = 2.0 +CENTROID_HZ_TOLERANCE = 200.0 + + +@dataclass(frozen=True) +class BufferStats: + rms_dbfs: float + peak_dbfs: float + spectral_centroid_hz: float + nonzero: bool + + +def _build_probe_plan() -> ClipPlan: + """Single-octave C-major triad sustained for 4 beats at 120 BPM (2 s).""" + return ClipPlan( + tempo_bpm=120.0, + duration_beats=4.0, + notes=[ + NoteEvent(pitch_midi=60, start_beat=0.0, duration_beats=4.0, velocity=100), + NoteEvent(pitch_midi=64, start_beat=0.0, duration_beats=4.0, velocity=100), + NoteEvent(pitch_midi=67, start_beat=0.0, duration_beats=4.0, velocity=100), + ], + program=0, # Acoustic Grand Piano — a stable preset across SF banks. + ) + + +def _compute_buffer_stats(samples: np.ndarray, sample_rate: int) -> BufferStats: + mono = np.asarray(samples, dtype=np.float64).reshape(-1) + if mono.size == 0: + return BufferStats( + rms_dbfs=-float("inf"), + peak_dbfs=-float("inf"), + spectral_centroid_hz=0.0, + nonzero=False, + ) + + rms = float(np.sqrt(np.mean(mono * mono))) + peak = float(np.max(np.abs(mono))) + if peak <= 0: + return BufferStats( + rms_dbfs=-float("inf"), + peak_dbfs=-float("inf"), + spectral_centroid_hz=0.0, + nonzero=False, + ) + + # FFT on the steady-state portion (skip the attack edge) for a stable + # centroid even when one backend has a sharper transient than the other. + skip_samples = min(int(0.1 * sample_rate), mono.size // 4) + window = mono[skip_samples:] + if window.size == 0: + window = mono + hann = np.hanning(window.size) + spectrum = np.abs(np.fft.rfft(window * hann)) + freqs = np.fft.rfftfreq(window.size, d=1.0 / sample_rate) + spectrum_sum = float(spectrum.sum()) + centroid = ( + float((freqs * spectrum).sum() / spectrum_sum) + if spectrum_sum > 0 + else 0.0 + ) + + return BufferStats( + rms_dbfs=20.0 * float(np.log10(max(rms, 1e-12))), + peak_dbfs=20.0 * float(np.log10(max(peak, 1e-12))), + spectral_centroid_hz=centroid, + nonzero=True, + ) + + +def _stats_to_dict(stats: BufferStats) -> dict[str, Any]: + return { + "rmsDbfs": round(stats.rms_dbfs, 3), + "peakDbfs": round(stats.peak_dbfs, 3), + "spectralCentroidHz": round(stats.spectral_centroid_hz, 2), + "nonzero": stats.nonzero, + } + + +def _verdict( + fluidsynth: BufferStats, symusic: BufferStats +) -> dict[str, Any]: + """Apply the tolerance bands and emit a pass/fail per axis.""" + rms_delta_db = abs(fluidsynth.rms_dbfs - symusic.rms_dbfs) + peak_delta_db = abs(fluidsynth.peak_dbfs - symusic.peak_dbfs) + centroid_delta_hz = abs( + fluidsynth.spectral_centroid_hz - symusic.spectral_centroid_hz + ) + + axes = { + "rms": { + "deltaDb": round(rms_delta_db, 3), + "toleranceDb": RMS_DB_TOLERANCE, + "pass": rms_delta_db <= RMS_DB_TOLERANCE, + }, + "peak": { + "deltaDb": round(peak_delta_db, 3), + "toleranceDb": PEAK_DB_TOLERANCE, + "pass": peak_delta_db <= PEAK_DB_TOLERANCE, + }, + "spectralCentroid": { + "deltaHz": round(centroid_delta_hz, 2), + "toleranceHz": CENTROID_HZ_TOLERANCE, + "pass": centroid_delta_hz <= CENTROID_HZ_TOLERANCE, + }, + } + return { + "axes": axes, + "overallPass": all(axis["pass"] for axis in axes.values()), + } + + +def _render_with_backend( + backend: sample_synthesis.Backend, + plan: ClipPlan, + soundfont: Path, +) -> sample_synthesis.RenderResult: + if backend == "fluidsynth": + return sample_synthesis._render_with_fluidsynth(plan, soundfont) + if backend == "symusic": + return sample_synthesis._render_with_symusic_synth(plan, soundfont) + raise ValueError(f"Unknown backend for parity probe: {backend!r}") + + +def _emit_report(report: dict[str, Any], out_path: Path) -> None: + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--soundfont", + type=Path, + default=None, + help="Explicit SF2/SF3 path. Overrides SONIC_ANALYZER_SOUNDFONT and the candidate-paths fallback.", + ) + parser.add_argument( + "--out-dir", + type=Path, + default=REPO_DIR / ".runtime" / "parity", + help="Directory to write synth_parity.json. Defaults to apps/backend/.runtime/parity/.", + ) + args = parser.parse_args() + + report: dict[str, Any] = { + "generatedAt": datetime.now(timezone.utc).isoformat(), + "probe": "fluidsynth_vs_symusic", + "soundfontPath": None, + "fluidsynth": None, + "symusic": None, + "verdict": None, + "status": "pending", + "tolerances": { + "rmsDb": RMS_DB_TOLERANCE, + "peakDb": PEAK_DB_TOLERANCE, + "centroidHz": CENTROID_HZ_TOLERANCE, + }, + } + + if not sample_synthesis._FLUIDSYNTH_IMPORTABLE: + report["status"] = "skipped_no_fluidsynth" + report["reason"] = ( + "pyfluidsynth is not importable in this venv; cannot measure parity " + "without both backends running side-by-side." + ) + _emit_report(report, args.out_dir / "synth_parity.json") + print(json.dumps(report, indent=2)) + return 0 + + soundfont = sample_synthesis.locate_soundfont(args.soundfont) + if soundfont is None: + report["status"] = "skipped_no_soundfont" + report["reason"] = ( + "No SF2/SF3 reachable. Set SONIC_ANALYZER_SOUNDFONT or pass " + "--soundfont to enable the probe." + ) + _emit_report(report, args.out_dir / "synth_parity.json") + print(json.dumps(report, indent=2)) + return 0 + + report["soundfontPath"] = str(soundfont) + plan = _build_probe_plan() + + try: + fluidsynth_result = _render_with_backend("fluidsynth", plan, soundfont) + symusic_result = _render_with_backend("symusic", plan, soundfont) + except Exception as exc: + report["status"] = "error" + report["error"] = f"{type(exc).__name__}: {exc}" + _emit_report(report, args.out_dir / "synth_parity.json") + print(json.dumps(report, indent=2)) + return 1 + + fluidsynth_stats = _compute_buffer_stats( + fluidsynth_result.samples, fluidsynth_result.sample_rate + ) + symusic_stats = _compute_buffer_stats( + symusic_result.samples, symusic_result.sample_rate + ) + + report["fluidsynth"] = { + "sampleRate": fluidsynth_result.sample_rate, + "durationSeconds": round(fluidsynth_result.duration_seconds, 3), + "stats": _stats_to_dict(fluidsynth_stats), + } + report["symusic"] = { + "sampleRate": symusic_result.sample_rate, + "durationSeconds": round(symusic_result.duration_seconds, 3), + "stats": _stats_to_dict(symusic_stats), + } + + if not (fluidsynth_stats.nonzero and symusic_stats.nonzero): + report["status"] = "error" + report["error"] = ( + "One or both backends returned silent audio — check soundfont preset " + "loading and program-select wiring." + ) + _emit_report(report, args.out_dir / "synth_parity.json") + print(json.dumps(report, indent=2)) + return 1 + + verdict = _verdict(fluidsynth_stats, symusic_stats) + report["verdict"] = verdict + report["status"] = "completed_pass" if verdict["overallPass"] else "completed_fail" + + _emit_report(report, args.out_dir / "synth_parity.json") + print(json.dumps(report, indent=2)) + return 0 if verdict["overallPass"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/backend/server.py b/apps/backend/server.py index 6ee33f55..bd8d91c8 100644 --- a/apps/backend/server.py +++ b/apps/backend/server.py @@ -2687,13 +2687,22 @@ async def get_transcription_pianoroll( }, ) + # The pitch-note worker stores the transcriptionDetail dict *directly* as + # the stage result (see _execute_pitch_note_attempt → + # complete_pitch_note_attempt(result=transcription_detail) above), so the + # canonical shape is unwrapped. Tolerate a legacy ``{"transcriptionDetail": + # {...}}`` wrapper defensively, then require a ``notes`` list so a stray + # non-transcription dict (or a ``{"transcriptionDetail": None}`` null) can't + # masquerade as a payload. pn_result = pn_stage.get("result") - transcription_detail = ( - pn_result.get("transcriptionDetail") - if isinstance(pn_result, dict) - else None - ) - if not isinstance(transcription_detail, dict): + if isinstance(pn_result, dict): + nested = pn_result.get("transcriptionDetail") + transcription_detail = nested if isinstance(nested, dict) else pn_result + else: + transcription_detail = None + if not isinstance(transcription_detail, dict) or not isinstance( + transcription_detail.get("notes"), list + ): # Covers: completed-but-null result, failed attempt, interrupted # attempt — the user's recourse is identical (re-run transcription). return JSONResponse( diff --git a/apps/backend/server_samples.py b/apps/backend/server_samples.py index a6e05c61..b22e630a 100644 --- a/apps/backend/server_samples.py +++ b/apps/backend/server_samples.py @@ -68,7 +68,7 @@ def generate_and_register_samples( run_id: str, snapshot: dict[str, Any], force: bool = False, - prefer_fluidsynth: bool = True, + allow_soundfont_backends: bool = True, ) -> dict[str, Any]: """Run the orchestrator, persist artifacts, return a decorated manifest. @@ -109,7 +109,7 @@ def generate_and_register_samples( phase2=phase2, output_dir=tmp_dir, pitch_note_hints=None, # Pitch/note translation hints are a follow-up. - prefer_fluidsynth=prefer_fluidsynth, + allow_soundfont_backends=allow_soundfont_backends, ) # Persist each WAV/MIDI as a run artifact. The artifact kind names the diff --git a/apps/backend/tests/test_phase1_evaluation_transcription.py b/apps/backend/tests/test_phase1_evaluation_transcription.py index a5496d77..dd7aa020 100644 --- a/apps/backend/tests/test_phase1_evaluation_transcription.py +++ b/apps/backend/tests/test_phase1_evaluation_transcription.py @@ -13,7 +13,7 @@ import unittest from pathlib import Path -import pretty_midi +from symusic import Note, Score, Tempo, Track from phase1_evaluation import ( DEFAULT_MANIFEST_PATH, @@ -272,14 +272,22 @@ def fake_runner(_path: Path, _flags: list[str]) -> dict: class ImportMidiScriptTests(unittest.TestCase): def _write_midi(self, path: Path, notes: list[tuple[int, float, float]]) -> None: - midi = pretty_midi.PrettyMIDI() - instrument = pretty_midi.Instrument(program=0) + score = Score(480, ttype="Second") + score.tempos.append(Tempo(time=0.0, qpm=120.0, ttype="Second")) + track = Track(name="ref", program=0, ttype="Second") for pitch, start, end in notes: - instrument.notes.append( - pretty_midi.Note(velocity=90, pitch=pitch, start=start, end=end) + duration = max(0.0, end - start) + track.notes.append( + Note( + time=float(start), + duration=float(duration), + pitch=int(pitch), + velocity=90, + ttype="Second", + ) ) - midi.instruments.append(instrument) - midi.write(str(path)) + score.tracks.append(track) + score.dump_midi(str(path)) def _invoke(self, midi_path: Path, *extra_args: str) -> subprocess.CompletedProcess: return subprocess.run( diff --git a/apps/backend/tests/test_polyphonic_evaluation.py b/apps/backend/tests/test_polyphonic_evaluation.py index c99bf481..bea645fc 100644 --- a/apps/backend/tests/test_polyphonic_evaluation.py +++ b/apps/backend/tests/test_polyphonic_evaluation.py @@ -6,8 +6,8 @@ from pathlib import Path import numpy as np -import pretty_midi import soundfile as sf +from symusic import Note, Score, Tempo, Track from polyphonic_evaluation import ( build_manual_scorecard, @@ -28,19 +28,27 @@ def _write_wav(path: Path, duration_seconds: float = 1.0, sample_rate: int = 220 def _write_midi(path: Path, note_specs: list[tuple[int, float, float]]) -> None: - midi = pretty_midi.PrettyMIDI() - instrument = pretty_midi.Instrument(program=0) + """Build a tiny seconds-unit Score and dump as MIDI. + + Mirrors the polyphonic-eval candidate output shape: one track, one + program, notes at `(start, end)` in seconds with constant velocity 90. + """ + score = Score(480, ttype="Second") + score.tempos.append(Tempo(time=0.0, qpm=120.0, ttype="Second")) + track = Track(name="candidate", program=0, ttype="Second") for pitch, start, end in note_specs: - instrument.notes.append( - pretty_midi.Note( + duration = max(0.0, end - start) + track.notes.append( + Note( + time=float(start), + duration=float(duration), + pitch=int(pitch), velocity=90, - pitch=pitch, - start=start, - end=end, + ttype="Second", ) ) - midi.instruments.append(instrument) - midi.write(str(path)) + score.tracks.append(track) + score.dump_midi(str(path)) class PolyphonicEvaluationTests(unittest.TestCase): diff --git a/apps/backend/tests/test_sample_audio_content.py b/apps/backend/tests/test_sample_audio_content.py index 46a85b41..914acb49 100644 --- a/apps/backend/tests/test_sample_audio_content.py +++ b/apps/backend/tests/test_sample_audio_content.py @@ -14,8 +14,8 @@ canned clip. If sample generation ever regresses to fixed output — or stops threading the measured key through to the render — that test fails. -Runs against the sine-additive fallback (`prefer_fluidsynth=False`) so the -assertions are deterministic and don't depend on a system soundfont. +Runs against the sine-additive fallback (`allow_soundfont_backends=False`) so +the assertions are deterministic and don't depend on a system soundfont. """ import sys @@ -95,7 +95,7 @@ def _generate(tmp: str, phase1: dict, **kwargs) -> Path: phase1=phase1, phase2=None, output_dir=Path(tmp), - prefer_fluidsynth=False, + allow_soundfont_backends=False, **kwargs, ) return Path(tmp) diff --git a/apps/backend/tests/test_sample_generation.py b/apps/backend/tests/test_sample_generation.py index 04019b6a..393d10cb 100644 --- a/apps/backend/tests/test_sample_generation.py +++ b/apps/backend/tests/test_sample_generation.py @@ -59,7 +59,7 @@ def test_full_input_produces_all_sample_categories(self) -> None: phase2=_baseline_phase2(), output_dir=Path(tmp), pitch_note_hints=[1, 2, 3, 5], - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) sample_ids = {s["id"] for s in result.manifest["samples"]} @@ -92,7 +92,7 @@ def test_tonal_samples_skipped_when_key_missing(self) -> None: phase1=phase1, phase2=None, output_dir=Path(tmp), - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) sample_ids = {s["id"] for s in result.manifest["samples"]} self.assertNotIn("tonal_chord_progression", sample_ids) @@ -110,7 +110,7 @@ def test_low_confidence_key_flags_tonal_samples(self) -> None: phase1=phase1, phase2=None, output_dir=Path(tmp), - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) tonal = [s for s in result.manifest["samples"] if s["category"] == "tonal"] self.assertTrue(tonal, "expected tonal samples even at low confidence") @@ -130,7 +130,7 @@ def test_every_sample_cites_or_explains_absence(self) -> None: phase1=_baseline_phase1(), phase2=_baseline_phase2(), output_dir=Path(tmp), - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) for sample in result.manifest["samples"]: cites = sample["cites"] @@ -151,7 +151,7 @@ def test_kick_uses_measured_fundamental_when_present(self) -> None: phase1=_baseline_phase1(), phase2=None, output_dir=Path(tmp), - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) kick = next(s for s in result.manifest["samples"] if s["id"] == "drum_kick") self.assertEqual(kick["cites"]["phase1Fields"], [ @@ -167,7 +167,7 @@ def test_manifest_records_theory_backend(self) -> None: phase1=_baseline_phase1(), phase2=None, output_dir=Path(tmp), - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) self.assertIn(result.manifest["theoryBackend"], {"pytheory", "fallback"}) diff --git a/apps/backend/tests/test_sample_synthesis.py b/apps/backend/tests/test_sample_synthesis.py index 917e0639..9506c326 100644 --- a/apps/backend/tests/test_sample_synthesis.py +++ b/apps/backend/tests/test_sample_synthesis.py @@ -1,16 +1,23 @@ """Tests for the MIDI-plan → WAV/MIDI synthesis layer. -The sine fallback path is the one exercised by these tests; the FluidSynth -path requires a system library and a soundfont, which we don't assume are -present in CI. The fallback is the everywhere-available backstop and must -stay correct. +The sine fallback path is the one exercised by audio-level tests; the +FluidSynth and symusic.Synthesizer paths require a system soundfont, which +we don't assume is present in CI. The fallback is the everywhere-available +backstop and must stay correct. + +BackendResolutionTests covers the env-flag dispatch logic in pure isolation — +no synthesis, no soundfont, just verifying that +``_resolve_backend`` picks the right path under each combination of env +override + soundfont availability + ``allow_soundfont_backends`` flag. """ +import os import sys import tempfile import unittest import wave from pathlib import Path +from unittest.mock import patch import numpy as np @@ -39,7 +46,7 @@ def _single_note_plan(pitch: int = 69, bpm: float = 120.0, duration_beats: float class SineFallbackTests(unittest.TestCase): def test_render_returns_expected_shape(self) -> None: plan = _single_note_plan(pitch=69, bpm=120.0, duration_beats=4.0) - result = sample_synthesis.render_clip(plan, prefer_fluidsynth=False) + result = sample_synthesis.render_clip(plan, allow_soundfont_backends=False) expected_samples = int(round(2.0 * sample_synthesis.SAMPLE_RATE)) # 4 beats @ 120 = 2 s self.assertEqual(result.samples.shape, (expected_samples,)) self.assertEqual(result.samples.dtype, np.float32) @@ -49,7 +56,7 @@ def test_render_returns_expected_shape(self) -> None: def test_render_contains_energy_at_target_frequency(self) -> None: # A4 = MIDI 69 = 440 Hz exactly. plan = _single_note_plan(pitch=69, bpm=120.0, duration_beats=4.0) - result = sample_synthesis.render_clip(plan, prefer_fluidsynth=False) + result = sample_synthesis.render_clip(plan, allow_soundfont_backends=False) # Look at the middle of the note to avoid envelope transients. mid_start = result.samples.size // 4 @@ -65,7 +72,7 @@ def test_render_silent_when_no_notes(self) -> None: plan = sample_theory.ClipPlan( tempo_bpm=120.0, duration_beats=4.0, notes=[], program=0 ) - result = sample_synthesis.render_clip(plan, prefer_fluidsynth=False) + result = sample_synthesis.render_clip(plan, allow_soundfont_backends=False) peak = float(np.max(np.abs(result.samples))) self.assertEqual(peak, 0.0) @@ -73,7 +80,7 @@ def test_render_silent_when_no_notes(self) -> None: class WriteWavTests(unittest.TestCase): def test_write_wav_round_trips(self) -> None: plan = _single_note_plan(pitch=60, bpm=120.0, duration_beats=2.0) - result = sample_synthesis.render_clip(plan, prefer_fluidsynth=False) + result = sample_synthesis.render_clip(plan, allow_soundfont_backends=False) with tempfile.TemporaryDirectory() as tmp: wav_path = Path(tmp) / "out.wav" @@ -89,7 +96,18 @@ def test_write_wav_round_trips(self) -> None: class WriteMidiTests(unittest.TestCase): def test_write_midi_emits_expected_note(self) -> None: - import pretty_midi # local import — only test needs it + # Parity is layered: + # 1. Library-independent: the bytes must start with a valid MThd + # header and contain at least one MTrk chunk. This catches the + # "writer emits something only its own reader can parse" failure + # mode that worried us when pretty_midi was dropped as the + # independent reader. + # 2. Within-library: symusic.Score must read the file back as the + # same notes (pitch, onset, duration). Weaker than cross-library + # but still a real round-trip. + import struct + + from symusic import Score plan = sample_theory.ClipPlan( tempo_bpm=120.0, @@ -104,11 +122,220 @@ def test_write_midi_emits_expected_note(self) -> None: mid_path = Path(tmp) / "out.mid" sample_synthesis.write_midi(plan, path=mid_path) self.assertTrue(mid_path.is_file()) - pm = pretty_midi.PrettyMIDI(str(mid_path)) - self.assertEqual(len(pm.instruments), 1) - self.assertEqual(len(pm.instruments[0].notes), 2) - pitches = sorted(n.pitch for n in pm.instruments[0].notes) - self.assertEqual(pitches, [60, 64]) + + # Layer 1 — library-independent structural check. + raw = mid_path.read_bytes() + self.assertEqual(raw[:4], b"MThd", "Should start with MThd chunk") + mthd_length = struct.unpack(">I", raw[4:8])[0] + self.assertEqual( + mthd_length, 6, "MThd content must be exactly 6 bytes per the MIDI spec" + ) + self.assertIn( + b"MTrk", raw, "Should contain at least one MTrk chunk" + ) + + # Layer 2 — symusic round-trip on the same bytes. + loaded = Score(mid_path).to("Second") + self.assertEqual(len(loaded.tracks), 1) + notes = sorted(loaded.tracks[0].notes, key=lambda n: n.time) + self.assertEqual(len(notes), 2) + self.assertEqual([int(n.pitch) for n in notes], [60, 64]) + # Onsets at beats 0 and 2 → 0.0 s and 1.0 s at 120 BPM. + self.assertAlmostEqual(float(notes[0].time), 0.0, places=3) + self.assertAlmostEqual(float(notes[1].time), 1.0, places=3) + # Both notes last 2 beats → 1.0 s at 120 BPM. + self.assertAlmostEqual(float(notes[0].duration), 1.0, places=3) + self.assertAlmostEqual(float(notes[1].duration), 1.0, places=3) + + +class BackendResolutionTests(unittest.TestCase): + """Pure-logic tests for `_resolve_backend` — no synthesis, no soundfont. + + Precedence under ``auto`` is *deliberately conservative*: FluidSynth wins + when both importable and a soundfont is reachable, because operators with + existing FluidSynth setups already have proven audio output and we don't + have cross-backend parity evidence yet. Symusic is the auto fallback only + when FluidSynth isn't importable; operators opt in explicitly via + ``ASA_SAMPLE_SYNTH_BACKEND=symusic`` to get the faster Prestosynth path. + """ + + def _resolve( + self, + *, + env: str | None = None, + soundfont_path: Path | None = None, + allow_soundfont_backends: bool = True, + ) -> str: + env_dict = {} if env is None else {sample_synthesis._BACKEND_ENV_VAR: env} + with patch.dict(os.environ, env_dict, clear=False): + if env is None: + os.environ.pop(sample_synthesis._BACKEND_ENV_VAR, None) + return sample_synthesis._resolve_backend( + soundfont_path=soundfont_path, + allow_soundfont_backends=allow_soundfont_backends, + ) + + def test_auto_with_soundfont_and_fluidsynth_prefers_fluidsynth(self) -> None: + # The conservative default — operators with a working FluidSynth keep + # getting the same engine they had pre-campaign. + with patch.object(sample_synthesis, "_FLUIDSYNTH_IMPORTABLE", True): + result = self._resolve(env="auto", soundfont_path=Path("/tmp/fake.sf2")) + self.assertEqual(result, "fluidsynth") + + def test_auto_with_soundfont_no_fluidsynth_uses_symusic(self) -> None: + # Symusic backstops in auto only when FluidSynth isn't importable — + # the new capability that wasn't reachable before this campaign. + with patch.object(sample_synthesis, "_FLUIDSYNTH_IMPORTABLE", False): + result = self._resolve(env="auto", soundfont_path=Path("/tmp/fake.sf2")) + self.assertEqual(result, "symusic") + + def test_auto_without_soundfont_falls_through_to_sine(self) -> None: + result = self._resolve(env="auto", soundfont_path=None) + self.assertEqual(result, "sine_fallback") + + def test_explicit_sine_always_wins(self) -> None: + # Even with a soundfont, ``ASA_SAMPLE_SYNTH_BACKEND=sine`` forces the + # deterministic path — the documented escape hatch. + result = self._resolve(env="sine", soundfont_path=Path("/tmp/fake.sf2")) + self.assertEqual(result, "sine_fallback") + + def test_explicit_symusic_overrides_fluidsynth_preference(self) -> None: + # Explicit opt-in: operator has measured parity locally and wants the + # faster Prestosynth path even when FluidSynth would otherwise win. + with patch.object(sample_synthesis, "_FLUIDSYNTH_IMPORTABLE", True): + result = self._resolve( + env="symusic", soundfont_path=Path("/tmp/fake.sf2") + ) + self.assertEqual(result, "symusic") + + def test_explicit_symusic_without_soundfont_degrades(self) -> None: + # Refusing to auto-download a built-in SF3 is deliberate — that's a + # network call on the request path and could silently fail in + # hosted-mode workers. + result = self._resolve(env="symusic", soundfont_path=None) + self.assertEqual(result, "sine_fallback") + + def test_explicit_fluidsynth_requires_both_binding_and_soundfont(self) -> None: + with patch.object(sample_synthesis, "_FLUIDSYNTH_IMPORTABLE", False): + result = self._resolve( + env="fluidsynth", soundfont_path=Path("/tmp/fake.sf2") + ) + self.assertEqual(result, "sine_fallback") + + with patch.object(sample_synthesis, "_FLUIDSYNTH_IMPORTABLE", True): + result = self._resolve( + env="fluidsynth", soundfont_path=Path("/tmp/fake.sf2") + ) + self.assertEqual(result, "fluidsynth") + + def test_allow_soundfont_false_forces_sine(self) -> None: + # The caller-side escape hatch (used by tests) — even with a + # soundfont and a hardcoded env override, ``allow_soundfont_backends + # =False`` forces the deterministic sine backend. + result = self._resolve( + env="symusic", + soundfont_path=Path("/tmp/fake.sf2"), + allow_soundfont_backends=False, + ) + self.assertEqual(result, "sine_fallback") + + def test_unknown_env_value_falls_back_to_auto(self) -> None: + # Robustness: a typo or misconfiguration shouldn't break rendering. + # The module logs a warning and treats the value as "auto", which + # then resolves to FluidSynth-first when both backends could run. + with patch.object(sample_synthesis, "_FLUIDSYNTH_IMPORTABLE", True): + result = self._resolve(env="bogus", soundfont_path=Path("/tmp/fake.sf2")) + self.assertEqual(result, "fluidsynth") + + +class SymusicRenderPathTests(unittest.TestCase): + """Coverage for ``_render_with_symusic_synth`` itself, not just dispatch. + + The mocked tests verify the function builds the right Score and dispatches + to ``symusic.Synthesizer`` with the documented arguments — runs everywhere. + The integration test actually invokes the Prestosynth render; it's skipped + when no SF2/SF3 is reachable so CI without a system soundfont stays green. + """ + + def test_render_with_symusic_synth_builds_score_and_returns_mono_float32(self) -> None: + plan = _single_note_plan(pitch=60, bpm=120.0, duration_beats=2.0) + sf_path = Path("/tmp/fake.sf2") + expected_samples = int(round(2 * 60 / 120 * sample_synthesis.SAMPLE_RATE)) + mock_synth = unittest.mock.MagicMock() + mock_synth.render.return_value = np.zeros(expected_samples, dtype=np.float32) + + with patch.object( + sample_synthesis, "Synthesizer", return_value=mock_synth + ) as mock_factory: + result = sample_synthesis._render_with_symusic_synth(plan, sf_path) + + # The factory must receive the soundfont path + the canonical sample + # rate. quality=0 is the documented default but pinned here so a + # silent drift in the call surface still fails the test. + mock_factory.assert_called_once() + call = mock_factory.call_args + self.assertEqual(call.args[0], str(sf_path)) + self.assertEqual(call.kwargs.get("sample_rate"), sample_synthesis.SAMPLE_RATE) + self.assertEqual(call.kwargs.get("quality"), 0) + + # render() receives the built Score positionally; stereo=False is the + # contract that yields a 1D buffer (no axis-reduction guesswork). + mock_synth.render.assert_called_once() + render_call = mock_synth.render.call_args + score_arg = render_call.args[0] + self.assertEqual(len(score_arg.tracks), 1) + self.assertEqual(len(score_arg.tracks[0].notes), 1) + self.assertEqual(int(score_arg.tracks[0].notes[0].pitch), 60) + self.assertEqual(render_call.kwargs.get("stereo"), False) + + # Result shape — float32 mono at the canonical sample rate. + self.assertEqual(result.backend, "symusic") + self.assertEqual(result.sample_rate, sample_synthesis.SAMPLE_RATE) + self.assertEqual(result.samples.dtype, np.float32) + self.assertEqual(result.samples.ndim, 1) + self.assertEqual(result.samples.shape[0], expected_samples) + self.assertEqual(result.soundfont_path, str(sf_path)) + + def test_render_with_symusic_synth_reduces_stereo_to_mono(self) -> None: + # Defensive path: if a future symusic version returns 2D (stereo), + # the wrapper still hands the caller a 1D mono float32. + plan = _single_note_plan(pitch=60, bpm=120.0, duration_beats=1.0) + sf_path = Path("/tmp/fake.sf2") + expected_samples = int(round(1 * 60 / 120 * sample_synthesis.SAMPLE_RATE)) + # Shape (channels=2, samples) — typical interleaved-then-deinterleaved + # layout. Both channels carry the same signal so the mono should match. + stereo_buffer = np.full((2, expected_samples), 0.25, dtype=np.float32) + mock_synth = unittest.mock.MagicMock() + mock_synth.render.return_value = stereo_buffer + + with patch.object(sample_synthesis, "Synthesizer", return_value=mock_synth): + result = sample_synthesis._render_with_symusic_synth(plan, sf_path) + + self.assertEqual(result.samples.ndim, 1) + self.assertEqual(result.samples.shape[0], expected_samples) + self.assertTrue(np.allclose(result.samples, 0.25)) + + @unittest.skipUnless( + sample_synthesis.locate_soundfont() is not None, + "No SF2/SF3 soundfont reachable; set SONIC_ANALYZER_SOUNDFONT to enable.", + ) + def test_symusic_synth_produces_nonzero_audio_with_real_soundfont(self) -> None: + # Integration: the only assertion the audible-output story rests on + # in actual practice. Skipped when no soundfont is on disk; runs + # locally for any maintainer with SONIC_ANALYZER_SOUNDFONT set. + plan = _single_note_plan(pitch=60, bpm=120.0, duration_beats=1.0) + sf_path = sample_synthesis.locate_soundfont() + assert sf_path is not None # narrowing for type-checkers + + result = sample_synthesis._render_with_symusic_synth(plan, sf_path) + self.assertEqual(result.backend, "symusic") + self.assertEqual(result.sample_rate, sample_synthesis.SAMPLE_RATE) + self.assertEqual(result.samples.dtype, np.float32) + self.assertEqual(result.samples.ndim, 1) + rms = float(np.sqrt(np.mean(result.samples**2))) + self.assertGreater( + rms, 1e-4, "A single sustained note should produce nonzero RMS" + ) if __name__ == "__main__": # pragma: no cover diff --git a/apps/backend/tests/test_server.py b/apps/backend/tests/test_server.py index 655435ec..053d1cdb 100644 --- a/apps/backend/tests/test_server.py +++ b/apps/backend/tests/test_server.py @@ -5068,24 +5068,27 @@ def _make_run( @staticmethod def _transcription_result(notes: list[dict]) -> dict: + # Mirrors production: the pitch-note worker stores the + # transcriptionDetail dict *directly* as the stage result (unwrapped), + # so the route reads these fields off ``pn_stage["result"]`` itself. + # See test_pitch_note_worker_runs_as_subprocess, which asserts + # result["transcriptionMethod"] is readable at the top level. return { - "transcriptionDetail": { - "transcriptionMethod": "torchcrepe-viterbi", - "noteCount": len(notes), - "averageConfidence": 0.85, - "dominantPitches": [], - "pitchRange": { - "minMidi": 60, - "maxMidi": 67, - "minName": "C4", - "maxName": "G4", - }, - "stemSeparationUsed": False, - "fullMixFallback": True, - "stemsTranscribed": ["full_mix"], - "perStemAverageConfidence": {}, - "notes": notes, - } + "transcriptionMethod": "torchcrepe-viterbi", + "noteCount": len(notes), + "averageConfidence": 0.85, + "dominantPitches": [], + "pitchRange": { + "minMidi": 60, + "maxMidi": 67, + "minName": "C4", + "maxName": "G4", + }, + "stemSeparationUsed": False, + "fullMixFallback": True, + "stemsTranscribed": ["full_mix"], + "perStemAverageConfidence": {}, + "notes": notes, } @staticmethod diff --git a/apps/backend/tests/test_server_samples.py b/apps/backend/tests/test_server_samples.py index b0cc92f1..29ec8c9e 100644 --- a/apps/backend/tests/test_server_samples.py +++ b/apps/backend/tests/test_server_samples.py @@ -72,7 +72,7 @@ def test_generation_succeeds_with_phase1_only(self) -> None: run_id=self.run_id, snapshot=snapshot, force=False, - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) self.assertEqual(manifest["schemaVersion"], "samples.v1") self.assertIn("manifestArtifactId", manifest) @@ -92,7 +92,7 @@ def test_rejects_when_measurement_not_completed(self) -> None: runtime=self.runtime, run_id=self.run_id, snapshot=snapshot, - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) self.assertEqual(ctx.exception.code, "MEASUREMENT_NOT_COMPLETED") self.assertEqual(ctx.exception.status_code, 409) @@ -103,7 +103,7 @@ def test_rejects_regeneration_unless_force(self) -> None: runtime=self.runtime, run_id=self.run_id, snapshot=snapshot, - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) with self.assertRaises(server_samples.SamplesPreconditionError) as ctx: server_samples.generate_and_register_samples( @@ -111,7 +111,7 @@ def test_rejects_regeneration_unless_force(self) -> None: run_id=self.run_id, snapshot=snapshot, force=False, - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) self.assertEqual(ctx.exception.code, "SAMPLES_ALREADY_GENERATED") @@ -121,14 +121,14 @@ def test_force_regenerates(self) -> None: runtime=self.runtime, run_id=self.run_id, snapshot=snapshot, - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) second = server_samples.generate_and_register_samples( runtime=self.runtime, run_id=self.run_id, snapshot=snapshot, force=True, - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) # New manifest artifact id; the SQLite ids are UUIDs so they will differ. self.assertNotEqual(first["manifestArtifactId"], second["manifestArtifactId"]) @@ -146,7 +146,7 @@ def test_fetch_existing_returns_decorated_manifest_after_generation(self) -> Non runtime=self.runtime, run_id=self.run_id, snapshot=snapshot, - prefer_fluidsynth=False, + allow_soundfont_backends=False, ) fetched = server_samples.fetch_existing_manifest( runtime=self.runtime, run_id=self.run_id diff --git a/apps/ui/src/components/AnalysisResults.tsx b/apps/ui/src/components/AnalysisResults.tsx index 5010c2f5..517c5bfd 100644 --- a/apps/ui/src/components/AnalysisResults.tsx +++ b/apps/ui/src/components/AnalysisResults.tsx @@ -33,6 +33,7 @@ import { Phase2ConsistencyReport } from './Phase2ConsistencyReport'; import { MeasurementDashboard } from './MeasurementDashboard'; import { SamplePlayback } from './SamplePlayback'; import { SessionMusicianPanel } from './SessionMusicianPanel'; +import { TranscriptionPianorollBlock } from './TranscriptionPianorollBlock'; import { StemListeningNotesPanel } from './StemListeningNotesPanel'; import { hasStemListeningNotesContent } from '../services/sessionMusician'; import { @@ -1911,6 +1912,15 @@ export function AnalysisResults({ /> + {apiBaseUrl && + runId && + phase1.transcriptionDetail && + phase1.transcriptionDetail.noteCount > 0 && ( +