diff --git a/apps/backend/fundamentals_evaluation.py b/apps/backend/fundamentals_evaluation.py index 4cd7e8d4..6e82a91f 100644 --- a/apps/backend/fundamentals_evaluation.py +++ b/apps/backend/fundamentals_evaluation.py @@ -283,6 +283,53 @@ def _evaluate_expected( if isinstance(required_quality, dict): checks.extend(_evaluate_required_quality(payload, required_quality)) + honesty = expected.get("honesty") + if isinstance(honesty, dict): + checks.extend(_evaluate_honesty(payload, honesty)) + + return checks + + +def _evaluate_honesty( + payload: dict[str, Any], + honesty: dict[str, Any], +) -> list[FundamentalsCheck]: + """Abstention checks: assert what the pipeline must NOT claim. + + Beatless/sparse clips carry no honest rhythm evidence, so their gates + invert — a confident tempo, a swing reading, or an evidence-backed meter + on such material is the failure (PURPOSE.md invariant #4). + """ + checks: list[FundamentalsCheck] = [] + + max_bpm_confidence = honesty.get("maxBpmConfidence") + if isinstance(max_bpm_confidence, (int, float)): + actual = _number(payload.get("bpmConfidence")) + passed = actual is None or actual <= float(max_bpm_confidence) + checks.append(FundamentalsCheck( + "honesty:bpmConfidence", + passed, + f"max={max_bpm_confidence} actual={actual}", + )) + + if honesty.get("swingDetailAbsent") is True: + actual = _nested_value(payload, "rhythmDetail.swingDetail") + checks.append(FundamentalsCheck( + "honesty:swingAbsent", + actual is None, + f"expected=absent actual={'present' if actual is not None else 'absent'}", + )) + + meter_sources = honesty.get("meterSources") + if isinstance(meter_sources, list) and meter_sources: + allowed = [str(source) for source in meter_sources] + actual_source = payload.get("timeSignatureSource") + checks.append(FundamentalsCheck( + "honesty:meterSource", + str(actual_source) in allowed, + f"allowed={allowed} actual={actual_source}", + )) + return checks diff --git a/apps/backend/scripts/build_synthetic_corpus.py b/apps/backend/scripts/build_synthetic_corpus.py index 7e74c0e0..09356176 100755 --- a/apps/backend/scripts/build_synthetic_corpus.py +++ b/apps/backend/scripts/build_synthetic_corpus.py @@ -193,6 +193,105 @@ def render_grid_pattern( return RenderedClip(_peak_guard(buf), truth) +def render_broken_grid( + *, + bpm: float, + bars: int, + kick_offsets: tuple[float, ...], + snare_offsets: tuple[float, ...], +) -> RenderedClip: + """Broken-kick 4/4 pattern for the genre-generalization rhythm clips. + + kick_offsets/snare_offsets are 0-indexed beat positions within a 4/4 bar + (e.g. 2-step: kick (0, 1.5), snare (1, 3)). Straight 8th-note hats ride on + top so the beat tracker has pulse evidence at the notated tempo — that is + what makes the halftime clip an honest tempo-octave trap rather than an + ambiguous one (a real halftime arrangement carries its notated tempo in + the hat grid). Truth checks are the same as grid clips: bpm, 4/4, + beatGrid, downbeats. + """ + beats_per_bar = 4 + total_beats = bars * beats_per_bar + beat_s = 60.0 / bpm + buf = _new_buffer(bpm, total_beats) + kick, snare, hat = _eng_kick(), _eng_snare(), _eng_hat() + + for bar in range(bars): + bar_start = bar * beats_per_bar + for offset in kick_offsets: + _overlay(buf, kick, (bar_start + offset) * beat_s, 1.0 if offset == 0.0 else 0.85) + for offset in snare_offsets: + _overlay(buf, snare, (bar_start + offset) * beat_s, 0.85) + for eighth in range(total_beats * 2): + _overlay(buf, hat, eighth * 0.5 * beat_s, 0.45) + + truth = { + "bpm": bpm, + "timeSignature": "4/4", + "beatGrid": [round(b * beat_s, 6) for b in range(total_beats)], + "downbeats": [round(i * beats_per_bar * beat_s, 6) for i in range(bars)], + "hitTimes": { + "kick": [round((bar * beats_per_bar + o) * beat_s, 6) for bar in range(bars) for o in kick_offsets], + "snare": [round((bar * beats_per_bar + o) * beat_s, 6) for bar in range(bars) for o in snare_offsets], + }, + } + return RenderedClip(_peak_guard(buf), truth) + + +def render_shuffle16_pattern(*, bpm: float, bars: int, swing_percent: float) -> RenderedClip: + """Four-on-the-floor kicks + 16th-note shuffled hats (UKG/2-step shuffle). + + The same long/short alternation as the 8th-grid swing clips, one metrical + level down: each 8th-note span [x, x+0.5] carries its inner 16th delayed + to x + (swing/100)*0.5 beats. The truth swingPercent is the 16th-grid + ratio — what the swing measurement SHOULD report once it detects the + dominant grid instead of hardwiring 8ths (accuracy program PR-G5). + """ + total_beats = bars * 4 + beat_s = 60.0 / bpm + buf = _new_buffer(bpm, total_beats) + kick, hat = _eng_kick(), _eng_hat() + + hat_times: list[float] = [] + for beat in range(total_beats): + _overlay(buf, kick, beat * beat_s, 1.0 if beat % 4 == 0 else 0.8) + for eighth_offset in (0.0, 0.5): + base = beat + eighth_offset + _overlay(buf, hat, base * beat_s, 0.6) + swung = base + swing_percent / 100.0 * 0.5 + hat_times.append(round(swung * beat_s, 6)) + _overlay(buf, hat, swung * beat_s, 0.6) + + truth = { + "bpm": bpm, + "swingPercent": swing_percent, + "gridResolution": "16th", + "hitTimes": {"hihat": hat_times}, + } + return RenderedClip(_peak_guard(buf), truth) + + +def render_ambient_pad(key_root: str, mode: str, bpm: float) -> RenderedClip: + """Sparse beatless pad — the abstention clip. + + Two slow triads, no percussion. Active checks assert the key is still + right AND that the rhythm stack abstains: low tempo confidence, no + swingDetail, meter on the assumed-4/4 fallback. The honesty block is + check configuration (it rides in `expected`), not signal truth. + """ + chords = render_chord_progression(key_root, mode, [1, 6], bpm, 16.0) + truth = { + "key": chords.truth["key"], + "honesty": { + "maxBpmConfidence": 0.4, + "swingDetailAbsent": True, + "meterSources": ["assumed_four_four"], + }, + "chordTimeline": chords.truth["chordTimeline"], + } + return RenderedClip(chords.samples, truth) + + def render_count_pattern(*, bpm: float, bars: int) -> RenderedClip: """Band-disjoint, never-coincident placement for percussion-count checks. @@ -329,9 +428,20 @@ def _default_specs() -> list[dict[str, Any]]: ] +# Broken-kick bar patterns (0-indexed beat offsets in a 4/4 bar). Sources: +# classic 2-step (kick 1 + "and" of 2, backbeat snares), halftime (kick 1 / +# snare 3 — the tempo-octave trap), and a breakbeat-style syncopation with +# an anticipated second kick and the "and" of 3. +_BROKEN_PATTERNS: dict[str, dict[str, tuple[float, ...]]] = { + "twostep": {"kick": (0.0, 1.5), "snare": (1.0, 3.0)}, + "halftime": {"kick": (0.0,), "snare": (2.0,)}, + "breakbeat": {"kick": (0.0, 1.75, 2.5), "snare": (1.0, 3.0)}, +} + + def _synthetic_specs() -> list[dict[str, Any]]: specs = list(_default_specs()) - for bpm, meter in [(70, "4/4"), (90, "3/4"), (110, "6/8"), (128, "4/4"), (140, "7/8"), (174, "4/4")]: + for bpm, meter in [(70, "4/4"), (85, "4/4"), (90, "3/4"), (110, "6/8"), (128, "4/4"), (140, "7/8"), (150, "4/4"), (174, "4/4"), (190, "4/4")]: specs.append({"id": f"grid_{meter.replace('/', '_')}_{bpm}", "kind": "grid", "bpm": bpm, "meter": meter, "bars": 8}) roots = ["C", "Db", "D", "Eb", "E", "F", "F#", "G", "Ab", "A", "Bb", "B"] for i, root in enumerate(roots): @@ -342,6 +452,12 @@ def _synthetic_specs() -> list[dict[str, Any]]: specs.append({"id": f"swing_hats_{swing}", "kind": "swing", "bpm": 124, "bars": 8, "swing": swing}) for root, mode, bpm in [("A", "minor", 128), ("F", "major", 122)]: specs.append({"id": f"multi_{root}_{mode}", "kind": "multi", "root": root, "mode": mode, "bpm": bpm}) + specs.append({"id": "twostep_132", "kind": "twostep", "bpm": 132, "bars": 8}) + for bpm in (140, 174): + specs.append({"id": f"halftime_{bpm}", "kind": "halftime", "bpm": bpm, "bars": 8}) + specs.append({"id": "breakbeat_136", "kind": "breakbeat", "bpm": 136, "bars": 8}) + specs.append({"id": "shuffle16_130_62", "kind": "shuffle16", "bpm": 130, "bars": 8, "swing": 62}) + specs.append({"id": "ambient_beatless_70", "kind": "ambient", "root": "A", "mode": "minor", "bpm": 70}) return specs @@ -353,6 +469,18 @@ def _render_spec(spec: dict[str, Any]) -> RenderedClip: return render_count_pattern(bpm=float(spec["bpm"]), bars=int(spec.get("bars", 8))) if kind == "swing": return render_grid_pattern(bpm=float(spec["bpm"]), meter="4/4", bars=int(spec.get("bars", 8)), with_hats=True, swing_percent=float(spec["swing"])) + if kind in _BROKEN_PATTERNS: + pattern = _BROKEN_PATTERNS[kind] + return render_broken_grid( + bpm=float(spec["bpm"]), + bars=int(spec.get("bars", 8)), + kick_offsets=pattern["kick"], + snare_offsets=pattern["snare"], + ) + if kind == "shuffle16": + return render_shuffle16_pattern(bpm=float(spec["bpm"]), bars=int(spec.get("bars", 8)), swing_percent=float(spec["swing"])) + if kind == "ambient": + return render_ambient_pad(spec["root"], spec["mode"], float(spec["bpm"])) if kind == "chords": return render_chord_progression(spec["root"], spec["mode"], list(spec["degrees"]), float(spec["bpm"]), float(spec["beats_per_chord"])) if kind == "multi": @@ -376,6 +504,15 @@ def _render_spec(spec: dict[str, Any]) -> RenderedClip: "chords": ("key", "chordTimeline"), "multi": ("key", "chordTimeline", "timeSignature"), "bass": ("transcriptionNotes",), + # Genre-generalization clips (accuracy program PR-G2): broken-kick + # patterns share the grid checks; shuffle16 shares the swing checks + # (its swung 16ths corrupt the meter autocorrelation the same way); + # ambient is key + abstention-honesty only. + "twostep": ("bpm", "timeSignature", "beatGrid", "downbeats"), + "halftime": ("bpm", "timeSignature", "beatGrid", "downbeats"), + "breakbeat": ("bpm", "timeSignature", "beatGrid", "downbeats"), + "shuffle16": ("bpm", "swingPercent"), + "ambient": ("key", "honesty"), } _THRESHOLDS_BY_KIND: dict[str, dict[str, Any]] = { @@ -385,6 +522,11 @@ def _render_spec(spec: dict[str, Any]) -> RenderedClip: "chords": {"chordSegmentAccuracy": 0.65}, "multi": {"chordSegmentAccuracy": 0.45, "allowRelativeMajorMinor": True}, "bass": {"transcriptionNoteF1": 0.75}, + "twostep": {"bpmTolerance": 1.0, "beatF1": 0.9, "downbeatF1": 0.75}, + "halftime": {"bpmTolerance": 1.0, "beatF1": 0.9, "downbeatF1": 0.75}, + "breakbeat": {"bpmTolerance": 1.0, "beatF1": 0.9, "downbeatF1": 0.75}, + "shuffle16": {"bpmTolerance": 1.0, "swingTolerance": 3.0}, + "ambient": {}, } # Measured baseline weaknesses (calibrated 2026-07-03 by running the full @@ -402,6 +544,20 @@ def _render_spec(spec: dict[str, Any]) -> RenderedClip: # RhythmExtractor halves 174 BPM to 86.9 (octave preference); the beat # grid and downbeats still score >= 0.87 against truth. "grid_4_4_174": ["tempo:bpm"], + # Genre-generalization baseline (calibrated 2026-07-13; PR-G3/PR-G5 own + # these). At 190 the octave halving worsens: unlike 174 (where the beat + # grid survived), the halved grid drags beatGrid to 0.625 and downbeats + # to 0.667. + "grid_4_4_190": ["tempo:bpm", "beatGrid:f1", "downbeats:f1"], + # Halftime at 174 (kick 1 / snare 3, 8th hats) reads 117.0 — a 2:3 + # ratio error, not a clean octave — and the grid follows it down + # (beatGrid 0.370, downbeats 0.308). The same arrangement at 140 passes + # every check, so the trap is specifically tempo-extreme halftime. + "halftime_174": ["tempo:bpm", "beatGrid:f1", "downbeats:f1"], + # 16th-grid shuffle is invisible to the swing measurement (swingDetail + # None): compute_swing_detail keeps only 8th-scale IOIs (0.30-0.70 + # beats) and hardwires gridResolution "8th". PR-G5 target. + "shuffle16_130_62": ["swing:swingPercent"], } diff --git a/apps/backend/tests/fixtures/fundamentals_eval_manifest.synthetic.json b/apps/backend/tests/fixtures/fundamentals_eval_manifest.synthetic.json index 92ca4084..c4e16411 100644 --- a/apps/backend/tests/fixtures/fundamentals_eval_manifest.synthetic.json +++ b/apps/backend/tests/fixtures/fundamentals_eval_manifest.synthetic.json @@ -277,6 +277,65 @@ "downbeatF1": 0.75 } }, + { + "id": "grid_4_4_85", + "audioPath": "synthetic/grid_4_4_85.wav", + "category": "grid", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "bpm": 85.0, + "timeSignature": "4/4", + "beatGrid": [ + 0.0, + 0.705882, + 1.411765, + 2.117647, + 2.823529, + 3.529412, + 4.235294, + 4.941176, + 5.647059, + 6.352941, + 7.058824, + 7.764706, + 8.470588, + 9.176471, + 9.882353, + 10.588235, + 11.294118, + 12.0, + 12.705882, + 13.411765, + 14.117647, + 14.823529, + 15.529412, + 16.235294, + 16.941176, + 17.647059, + 18.352941, + 19.058824, + 19.764706, + 20.470588, + 21.176471, + 21.882353 + ], + "downbeats": [ + 0.0, + 2.823529, + 5.647059, + 8.470588, + 11.294118, + 14.117647, + 16.941176, + 19.764706 + ] + }, + "thresholds": { + "bpmTolerance": 1.0, + "beatF1": 0.9, + "downbeatF1": 0.75 + } + }, { "id": "grid_3_4_90", "audioPath": "synthetic/grid_3_4_90.wav", @@ -558,6 +617,65 @@ "tempo:bpm" ] }, + { + "id": "grid_4_4_150", + "audioPath": "synthetic/grid_4_4_150.wav", + "category": "grid", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "bpm": 150.0, + "timeSignature": "4/4", + "beatGrid": [ + 0.0, + 0.4, + 0.8, + 1.2, + 1.6, + 2.0, + 2.4, + 2.8, + 3.2, + 3.6, + 4.0, + 4.4, + 4.8, + 5.2, + 5.6, + 6.0, + 6.4, + 6.8, + 7.2, + 7.6, + 8.0, + 8.4, + 8.8, + 9.2, + 9.6, + 10.0, + 10.4, + 10.8, + 11.2, + 11.6, + 12.0, + 12.4 + ], + "downbeats": [ + 0.0, + 1.6, + 3.2, + 4.8, + 6.4, + 8.0, + 9.6, + 11.2 + ] + }, + "thresholds": { + "bpmTolerance": 1.0, + "beatF1": 0.9, + "downbeatF1": 0.75 + } + }, { "id": "grid_4_4_174", "audioPath": "synthetic/grid_4_4_174.wav", @@ -620,6 +738,70 @@ "tempo:bpm" ] }, + { + "id": "grid_4_4_190", + "audioPath": "synthetic/grid_4_4_190.wav", + "category": "grid", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "bpm": 190.0, + "timeSignature": "4/4", + "beatGrid": [ + 0.0, + 0.315789, + 0.631579, + 0.947368, + 1.263158, + 1.578947, + 1.894737, + 2.210526, + 2.526316, + 2.842105, + 3.157895, + 3.473684, + 3.789474, + 4.105263, + 4.421053, + 4.736842, + 5.052632, + 5.368421, + 5.684211, + 6.0, + 6.315789, + 6.631579, + 6.947368, + 7.263158, + 7.578947, + 7.894737, + 8.210526, + 8.526316, + 8.842105, + 9.157895, + 9.473684, + 9.789474 + ], + "downbeats": [ + 0.0, + 1.263158, + 2.526316, + 3.789474, + 5.052632, + 6.315789, + 7.578947, + 8.842105 + ] + }, + "thresholds": { + "bpmTolerance": 1.0, + "beatF1": 0.9, + "downbeatF1": 0.75 + }, + "knownGaps": [ + "tempo:bpm", + "beatGrid:f1", + "downbeats:f1" + ] + }, { "id": "chords_C_major", "audioPath": "synthetic/chords_C_major.wav", @@ -1632,6 +1814,502 @@ "analyzeFlags": [ "--separate" ] + }, + { + "id": "twostep_132", + "audioPath": "synthetic/twostep_132.wav", + "category": "twostep", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "bpm": 132.0, + "timeSignature": "4/4", + "beatGrid": [ + 0.0, + 0.454545, + 0.909091, + 1.363636, + 1.818182, + 2.272727, + 2.727273, + 3.181818, + 3.636364, + 4.090909, + 4.545455, + 5.0, + 5.454545, + 5.909091, + 6.363636, + 6.818182, + 7.272727, + 7.727273, + 8.181818, + 8.636364, + 9.090909, + 9.545455, + 10.0, + 10.454545, + 10.909091, + 11.363636, + 11.818182, + 12.272727, + 12.727273, + 13.181818, + 13.636364, + 14.090909 + ], + "downbeats": [ + 0.0, + 1.818182, + 3.636364, + 5.454545, + 7.272727, + 9.090909, + 10.909091, + 12.727273 + ] + }, + "thresholds": { + "bpmTolerance": 1.0, + "beatF1": 0.9, + "downbeatF1": 0.75 + }, + "truth": { + "hitTimes": { + "kick": [ + 0.0, + 0.681818, + 1.818182, + 2.5, + 3.636364, + 4.318182, + 5.454545, + 6.136364, + 7.272727, + 7.954545, + 9.090909, + 9.772727, + 10.909091, + 11.590909, + 12.727273, + 13.409091 + ], + "snare": [ + 0.454545, + 1.363636, + 2.272727, + 3.181818, + 4.090909, + 5.0, + 5.909091, + 6.818182, + 7.727273, + 8.636364, + 9.545455, + 10.454545, + 11.363636, + 12.272727, + 13.181818, + 14.090909 + ] + } + } + }, + { + "id": "halftime_140", + "audioPath": "synthetic/halftime_140.wav", + "category": "halftime", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "bpm": 140.0, + "timeSignature": "4/4", + "beatGrid": [ + 0.0, + 0.428571, + 0.857143, + 1.285714, + 1.714286, + 2.142857, + 2.571429, + 3.0, + 3.428571, + 3.857143, + 4.285714, + 4.714286, + 5.142857, + 5.571429, + 6.0, + 6.428571, + 6.857143, + 7.285714, + 7.714286, + 8.142857, + 8.571429, + 9.0, + 9.428571, + 9.857143, + 10.285714, + 10.714286, + 11.142857, + 11.571429, + 12.0, + 12.428571, + 12.857143, + 13.285714 + ], + "downbeats": [ + 0.0, + 1.714286, + 3.428571, + 5.142857, + 6.857143, + 8.571429, + 10.285714, + 12.0 + ] + }, + "thresholds": { + "bpmTolerance": 1.0, + "beatF1": 0.9, + "downbeatF1": 0.75 + }, + "truth": { + "hitTimes": { + "kick": [ + 0.0, + 1.714286, + 3.428571, + 5.142857, + 6.857143, + 8.571429, + 10.285714, + 12.0 + ], + "snare": [ + 0.857143, + 2.571429, + 4.285714, + 6.0, + 7.714286, + 9.428571, + 11.142857, + 12.857143 + ] + } + } + }, + { + "id": "halftime_174", + "audioPath": "synthetic/halftime_174.wav", + "category": "halftime", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "bpm": 174.0, + "timeSignature": "4/4", + "beatGrid": [ + 0.0, + 0.344828, + 0.689655, + 1.034483, + 1.37931, + 1.724138, + 2.068966, + 2.413793, + 2.758621, + 3.103448, + 3.448276, + 3.793103, + 4.137931, + 4.482759, + 4.827586, + 5.172414, + 5.517241, + 5.862069, + 6.206897, + 6.551724, + 6.896552, + 7.241379, + 7.586207, + 7.931034, + 8.275862, + 8.62069, + 8.965517, + 9.310345, + 9.655172, + 10.0, + 10.344828, + 10.689655 + ], + "downbeats": [ + 0.0, + 1.37931, + 2.758621, + 4.137931, + 5.517241, + 6.896552, + 8.275862, + 9.655172 + ] + }, + "thresholds": { + "bpmTolerance": 1.0, + "beatF1": 0.9, + "downbeatF1": 0.75 + }, + "truth": { + "hitTimes": { + "kick": [ + 0.0, + 1.37931, + 2.758621, + 4.137931, + 5.517241, + 6.896552, + 8.275862, + 9.655172 + ], + "snare": [ + 0.689655, + 2.068966, + 3.448276, + 4.827586, + 6.206897, + 7.586207, + 8.965517, + 10.344828 + ] + } + }, + "knownGaps": [ + "tempo:bpm", + "beatGrid:f1", + "downbeats:f1" + ] + }, + { + "id": "breakbeat_136", + "audioPath": "synthetic/breakbeat_136.wav", + "category": "breakbeat", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "bpm": 136.0, + "timeSignature": "4/4", + "beatGrid": [ + 0.0, + 0.441176, + 0.882353, + 1.323529, + 1.764706, + 2.205882, + 2.647059, + 3.088235, + 3.529412, + 3.970588, + 4.411765, + 4.852941, + 5.294118, + 5.735294, + 6.176471, + 6.617647, + 7.058824, + 7.5, + 7.941176, + 8.382353, + 8.823529, + 9.264706, + 9.705882, + 10.147059, + 10.588235, + 11.029412, + 11.470588, + 11.911765, + 12.352941, + 12.794118, + 13.235294, + 13.676471 + ], + "downbeats": [ + 0.0, + 1.764706, + 3.529412, + 5.294118, + 7.058824, + 8.823529, + 10.588235, + 12.352941 + ] + }, + "thresholds": { + "bpmTolerance": 1.0, + "beatF1": 0.9, + "downbeatF1": 0.75 + }, + "truth": { + "hitTimes": { + "kick": [ + 0.0, + 0.772059, + 1.102941, + 1.764706, + 2.536765, + 2.867647, + 3.529412, + 4.301471, + 4.632353, + 5.294118, + 6.066176, + 6.397059, + 7.058824, + 7.830882, + 8.161765, + 8.823529, + 9.595588, + 9.926471, + 10.588235, + 11.360294, + 11.691176, + 12.352941, + 13.125, + 13.455882 + ], + "snare": [ + 0.441176, + 1.323529, + 2.205882, + 3.088235, + 3.970588, + 4.852941, + 5.735294, + 6.617647, + 7.5, + 8.382353, + 9.264706, + 10.147059, + 11.029412, + 11.911765, + 12.794118, + 13.676471 + ] + } + } + }, + { + "id": "shuffle16_130_62", + "audioPath": "synthetic/shuffle16_130_62.wav", + "category": "shuffle16", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "bpm": 130.0, + "swingPercent": 62.0 + }, + "thresholds": { + "bpmTolerance": 1.0, + "swingTolerance": 3.0 + }, + "truth": { + "gridResolution": "16th", + "hitTimes": { + "hihat": [ + 0.143077, + 0.373846, + 0.604615, + 0.835385, + 1.066154, + 1.296923, + 1.527692, + 1.758462, + 1.989231, + 2.22, + 2.450769, + 2.681538, + 2.912308, + 3.143077, + 3.373846, + 3.604615, + 3.835385, + 4.066154, + 4.296923, + 4.527692, + 4.758462, + 4.989231, + 5.22, + 5.450769, + 5.681538, + 5.912308, + 6.143077, + 6.373846, + 6.604615, + 6.835385, + 7.066154, + 7.296923, + 7.527692, + 7.758462, + 7.989231, + 8.22, + 8.450769, + 8.681538, + 8.912308, + 9.143077, + 9.373846, + 9.604615, + 9.835385, + 10.066154, + 10.296923, + 10.527692, + 10.758462, + 10.989231, + 11.22, + 11.450769, + 11.681538, + 11.912308, + 12.143077, + 12.373846, + 12.604615, + 12.835385, + 13.066154, + 13.296923, + 13.527692, + 13.758462, + 13.989231, + 14.22, + 14.450769, + 14.681538 + ] + } + }, + "knownGaps": [ + "swing:swingPercent" + ] + }, + { + "id": "ambient_beatless_70", + "audioPath": "synthetic/ambient_beatless_70.wav", + "category": "ambient", + "description": "Deterministic NumPy-rendered synthetic fundamentals fixture.", + "expected": { + "key": "A minor", + "honesty": { + "maxBpmConfidence": 0.4, + "swingDetailAbsent": true, + "meterSources": [ + "assumed_four_four" + ] + } + }, + "thresholds": {}, + "truth": { + "chordTimeline": [ + { + "startSec": 0.0, + "endSec": 13.714286, + "label": "Am" + }, + { + "startSec": 13.714286, + "endSec": 27.428571, + "label": "F" + } + ] + } } ] } diff --git a/apps/backend/tests/fixtures/fundamentals_tracks/README.md b/apps/backend/tests/fixtures/fundamentals_tracks/README.md index 0a10c622..97803de3 100644 --- a/apps/backend/tests/fixtures/fundamentals_tracks/README.md +++ b/apps/backend/tests/fixtures/fundamentals_tracks/README.md @@ -36,6 +36,13 @@ missing audio fail the gate instead of producing a vacuous green run. `scripts/build_synthetic_corpus.py`. - **Swing clips** carry swung-hat ground truth (under each manifest entry's `truth` key) for the future swing measurement; only BPM is an active check. +- **Genre-generalization clips** (accuracy program PR-G2) widen the rhythm + surface beyond steady four-on-the-floor: broken-kick patterns (2-step, + halftime, breakbeat — all with straight 8th hats carrying the notated + pulse), a 16th-grid shuffle, a beatless ambient pad whose active checks + assert *abstention* (the `honesty` block in `expected`), and grid clips + at the 85/150/190 BPM extremes. See `_BROKEN_PATTERNS` and the PR-G2 + entries in `_KNOWN_GAPS_BY_ID`. - **`knownGaps`** entries mark measured baseline weaknesses (odd-meter detection, tempo octave errors at range extremes). Those checks still run and report scores but don't gate the run — they are the accuracy program's diff --git a/apps/backend/tests/test_build_synthetic_corpus.py b/apps/backend/tests/test_build_synthetic_corpus.py index 7171eddf..5a9cd718 100644 --- a/apps/backend/tests/test_build_synthetic_corpus.py +++ b/apps/backend/tests/test_build_synthetic_corpus.py @@ -10,10 +10,14 @@ ) from scripts.build_synthetic_corpus import ( _EXPECTED_KEYS_BY_KIND, + _THRESHOLDS_BY_KIND, build_corpus, + render_ambient_pad, + render_broken_grid, render_chord_progression, render_count_pattern, render_grid_pattern, + render_shuffle16_pattern, _render_spec, _synthetic_specs, ) @@ -132,6 +136,50 @@ def test_chord_labels_are_flat_spelled_and_fold_enharmonically(self) -> None: self.assertNotEqual(_normalize_chord_label("Am"), _normalize_chord_label("A")) self.assertEqual(_chord_segment_accuracy(rendered.truth["chordTimeline"], rendered.truth["chordTimeline"]), 1.0) + def test_broken_grid_truth_matches_placement(self) -> None: + # 2-step at 120 BPM (beat = 0.5 s): kicks at 0 and 1.5 beats, snares + # at 1 and 3 — the beat grid and downbeats stay on the notated pulse. + rendered = render_broken_grid( + bpm=120, bars=2, kick_offsets=(0.0, 1.5), snare_offsets=(1.0, 3.0) + ) + self.assertEqual(rendered.truth["bpm"], 120) + self.assertEqual(rendered.truth["timeSignature"], "4/4") + self.assertEqual(len(rendered.truth["beatGrid"]), 8) + self.assertEqual(rendered.truth["downbeats"], [0.0, 2.0]) + self.assertEqual(rendered.truth["hitTimes"]["kick"], [0.0, 0.75, 2.0, 2.75]) + self.assertEqual(rendered.truth["hitTimes"]["snare"], [0.5, 1.5, 2.5, 3.5]) + + def test_shuffle16_truth_places_swung_16ths(self) -> None: + rendered = render_shuffle16_pattern(bpm=120, bars=1, swing_percent=62) + self.assertEqual(rendered.truth["swingPercent"], 62) + self.assertEqual(rendered.truth["gridResolution"], "16th") + # First swung 16th: 62% of a half-beat into beat 0 => 0.31 beats = 0.155 s. + self.assertAlmostEqual(rendered.truth["hitTimes"]["hihat"][0], 0.155, places=4) + + def test_ambient_pad_expected_carries_key_and_honesty_only(self) -> None: + rendered = render_ambient_pad("A", "minor", 70) + self.assertEqual(rendered.truth["key"], "A minor") + self.assertEqual( + set(rendered.truth["honesty"].keys()), + {"maxBpmConfidence", "swingDetailAbsent", "meterSources"}, + ) + # chordTimeline stays inert truth: it is not an active ambient check. + self.assertNotIn("chordTimeline", _EXPECTED_KEYS_BY_KIND["ambient"]) + self.assertIn("chordTimeline", rendered.truth) + + def test_genre_generalization_specs_are_declared(self) -> None: + ids = {spec["id"] for spec in _synthetic_specs()} + for expected_id in ( + "grid_4_4_85", "grid_4_4_150", "grid_4_4_190", + "twostep_132", "halftime_140", "halftime_174", + "breakbeat_136", "shuffle16_130_62", "ambient_beatless_70", + ): + self.assertIn(expected_id, ids) + # Every declared kind has both check tables filled in. + for spec in _synthetic_specs(): + self.assertIn(spec["kind"], _EXPECTED_KEYS_BY_KIND) + self.assertIn(spec["kind"], _THRESHOLDS_BY_KIND) + def test_keys_match_folds_enharmonics(self) -> None: self.assertTrue(_keys_match("C# Minor", "Db minor", allow_relative=False)) self.assertFalse(_keys_match("C# Major", "Db minor", allow_relative=False)) diff --git a/apps/backend/tests/test_fundamentals_evaluation.py b/apps/backend/tests/test_fundamentals_evaluation.py index 609c02bc..3dd84086 100644 --- a/apps/backend/tests/test_fundamentals_evaluation.py +++ b/apps/backend/tests/test_fundamentals_evaluation.py @@ -210,6 +210,88 @@ def test_present_track_failure_fails_report(self) -> None: self.assertFalse(report["summary"]["allPassed"]) self.assertEqual(report["summary"]["checksFailed"], 1) + def test_honesty_checks_assert_abstention_on_beatless_material(self) -> None: + with tempfile.TemporaryDirectory(prefix="asa_fundamentals_eval_honesty_") as temp_dir: + temp_root = Path(temp_dir) + tracks_dir = temp_root / "tracks" + tracks_dir.mkdir() + (tracks_dir / "pad.wav").write_bytes(b"placeholder") + manifest_path = temp_root / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "tracks": [ + { + "id": "pad", + "audioPath": "pad.wav", + "category": "ambient", + "expected": { + "key": "A minor", + "honesty": { + "maxBpmConfidence": 0.4, + "swingDetailAbsent": True, + "meterSources": ["assumed_four_four"], + }, + }, + "thresholds": {}, + } + ] + } + ), + encoding="utf-8", + ) + + def honest_runner(_path: Path, _flags: list[str] | None) -> dict: + return { + "key": "A Minor", + "bpmConfidence": 0.12, + "timeSignatureSource": "assumed_four_four", + "rhythmDetail": {"swingDetail": None}, + } + + report = run_fundamentals_evaluation( + manifest_path=manifest_path, + tracks_dir=tracks_dir, + report_path=temp_root / "report.json", + runner=honest_runner, + ) + self.assertTrue(report["summary"]["allPassed"]) + names = {check["name"] for check in report["tracks"][0]["checks"]} + self.assertEqual( + names, + {"key:label", "honesty:bpmConfidence", "honesty:swingAbsent", "honesty:meterSource"}, + ) + + def overconfident_runner(_path: Path, _flags: list[str] | None) -> dict: + return { + "key": "A Minor", + "bpmConfidence": 0.93, + "timeSignatureSource": "onset_autocorrelation", + "rhythmDetail": {"swingDetail": {"swingPercent": 58.0}}, + } + + report = run_fundamentals_evaluation( + manifest_path=manifest_path, + tracks_dir=tracks_dir, + report_path=temp_root / "report.json", + runner=overconfident_runner, + ) + self.assertFalse(report["summary"]["allPassed"]) + failed = {c["name"] for c in report["tracks"][0]["checks"] if not c["passed"]} + self.assertEqual( + failed, + {"honesty:bpmConfidence", "honesty:swingAbsent", "honesty:meterSource"}, + ) + + def test_honesty_bpm_confidence_passes_when_field_is_absent(self) -> None: + # A pipeline that cannot say anything (bpmConfidence None/missing) is + # abstaining, which is exactly what the honesty gate wants. + from fundamentals_evaluation import _evaluate_honesty + + checks = _evaluate_honesty({}, {"maxBpmConfidence": 0.4}) + self.assertEqual(len(checks), 1) + self.assertTrue(checks[0].passed) + if __name__ == "__main__": unittest.main()