diff --git a/apps/backend/giantsteps_evaluation.py b/apps/backend/giantsteps_evaluation.py new file mode 100644 index 00000000..4f1d6c7f --- /dev/null +++ b/apps/backend/giantsteps_evaluation.py @@ -0,0 +1,253 @@ +"""GiantSteps key/tempo evaluation harness (research-only). + +Scores ASA's Phase 1 key and tempo measurements against the GiantSteps Key +and GiantSteps Tempo datasets — Beatport preview clips with expert +annotations, i.e. *real electronic music*, not synthetic renders. This is +the real-audio check the accuracy program requires before promoting any +key/tempo-affecting change (see audits/accuracy-baseline-2026-07-03.md). + +Not on the product path: driven by scripts/evaluate_giantsteps.py, corpus +fetched locally by the operator via scripts/fetch_giantsteps.py (audio is +Beatport's and is never committed — see tests/fixtures/giantsteps/README.md). + +The analyzer subprocess inherits the environment, so the same harness scores +alternative backends: e.g. `ASA_KEY_BACKEND=... scripts/evaluate_giantsteps.py`. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Literal + +from fundamentals_evaluation import _parse_key_pc + +BACKEND_DIR = Path(__file__).resolve().parent +DEFAULT_CORPUS_ROOT = BACKEND_DIR / "tests" / "fixtures" / "giantsteps" +DEFAULT_REPORT_DIR = BACKEND_DIR / ".runtime" / "reports" + +_AUDIO_EXTENSIONS = (".mp3", ".wav", ".flac", ".ogg", ".m4a", ".aif", ".aiff") + +Subset = Literal["key", "tempo"] +Runner = Callable[[Path, list[str] | None], dict[str, Any]] + + +@dataclass(frozen=True) +class GiantstepsClip: + clip_id: str + audio_path: Path + expected_key: str | None + expected_bpm: float | None + + +def load_giantsteps_corpus(root: Path, subset: Subset) -> list[GiantstepsClip]: + """Load clips from ``root//{annotations,audio}/``. + + One clip per annotation file (``*.key`` = key string like "D minor", + ``*.bpm`` = a single BPM float). Audio is matched by stem under + ``audio/`` with any common extension; clips whose audio is absent are + still returned (with the conventional path) so the caller can count and + report them as missing rather than silently shrinking the corpus. + """ + annotations_dir = root / subset / "annotations" + audio_dir = root / subset / "audio" + suffix = ".key" if subset == "key" else ".bpm" + clips: list[GiantstepsClip] = [] + if not annotations_dir.is_dir(): + return clips + for annotation_path in sorted(annotations_dir.rglob(f"*{suffix}")): + raw = annotation_path.read_text(encoding="utf-8", errors="replace").strip() + expected_key: str | None = None + expected_bpm: float | None = None + if subset == "key": + expected_key = raw if raw else None + else: + try: + expected_bpm = float(raw.split()[0]) if raw else None + except ValueError: + expected_bpm = None + stem = annotation_path.name[: -len(suffix)] + audio_path = audio_dir / f"{stem}.mp3" + for extension in _AUDIO_EXTENSIONS: + candidate = audio_dir / f"{stem}{extension}" + if candidate.exists(): + audio_path = candidate + break + clips.append( + GiantstepsClip( + clip_id=stem, + audio_path=audio_path, + expected_key=expected_key, + expected_bpm=expected_bpm, + ) + ) + return clips + + +def mirex_key_score(expected: str, actual: str | None) -> float: + """MIREX weighted key score, following the mir_eval.key convention. + + 1.0 exact (enharmonic-folded) · 0.5 estimate a perfect fifth above the + truth, same mode · 0.3 relative major/minor · 0.2 parallel (same tonic, + other mode) · 0.0 otherwise or unparseable. + """ + truth = _parse_key_pc(expected) + estimate = _parse_key_pc(actual) + if truth is None or estimate is None: + return 0.0 + truth_pc, truth_mode = truth + est_pc, est_mode = estimate + if truth_mode not in ("major", "minor") or est_mode not in ("major", "minor"): + return 1.0 if (truth_pc, truth_mode) == (est_pc, est_mode) else 0.0 + if (est_pc, est_mode) == (truth_pc, truth_mode): + return 1.0 + if est_mode == truth_mode and est_pc == (truth_pc + 7) % 12: + return 0.5 + if truth_mode == "major" and est_mode == "minor" and est_pc == (truth_pc + 9) % 12: + return 0.3 + if truth_mode == "minor" and est_mode == "major" and est_pc == (truth_pc + 3) % 12: + return 0.3 + if est_pc == truth_pc and est_mode != truth_mode: + return 0.2 + return 0.0 + + +def tempo_accuracies( + expected: float, actual: float | None, tolerance: float = 0.04 +) -> tuple[bool, bool]: + """Standard tempo Accuracy1 / Accuracy2. + + Acc1: estimate within ±tolerance (relative) of the truth. + Acc2: Acc1, or within tolerance of truth × {2, 3, 1/2, 1/3} — the + octave/triple family RhythmExtractor's range preference collapses into. + """ + if actual is None or not expected or expected <= 0: + return False, False + + def within(target: float) -> bool: + return abs(actual - target) <= tolerance * target + + acc1 = within(expected) + acc2 = acc1 or any(within(expected * factor) for factor in (2.0, 3.0, 0.5, 1.0 / 3.0)) + return acc1, acc2 + + +def _run_analyze(audio_path: Path, extra_flags: list[str] | None = None) -> dict[str, Any]: + command = [sys.executable, str(BACKEND_DIR / "analyze.py"), str(audio_path), "--yes"] + if extra_flags: + command.extend(extra_flags) + completed = subprocess.run( + command, + cwd=BACKEND_DIR, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(completed.stdout) + if not isinstance(payload, dict): + raise ValueError("analyze.py did not emit a JSON object") + return payload + + +def evaluate_corpus( + clips: list[GiantstepsClip], + *, + subset: Subset, + runner: Runner | None = None, + fast: bool = True, + max_clips: int | None = None, + report_path: Path | None = None, +) -> dict[str, Any]: + """Score the corpus; returns the aggregate report dict. + + ``fast=True`` runs ``analyze.py --fast`` (BPM/key are populated in fast + mode), which keeps a ~600-clip subset run tractable. The report's + ``status`` is ``"underpowered"`` when zero clips were evaluable — callers + must treat that as failure, never as a vacuous green. + """ + if runner is None: + runner = _run_analyze + flags = ["--fast"] if fast else None + + selected = clips[:max_clips] if max_clips else clips + per_clip: list[dict[str, Any]] = [] + missing_audio = 0 + analyze_failed = 0 + key_scores: list[float] = [] + key_exact = 0 + key_exact_or_relative = 0 + acc1_hits = 0 + acc2_hits = 0 + + for clip in selected: + if not clip.audio_path.exists(): + missing_audio += 1 + per_clip.append({"id": clip.clip_id, "status": "missing_audio"}) + continue + try: + payload = runner(clip.audio_path, flags) + except subprocess.CalledProcessError as exc: + analyze_failed += 1 + per_clip.append( + { + "id": clip.clip_id, + "status": "analyze_failed", + "error": (exc.stderr or "")[-300:], + } + ) + continue + + entry: dict[str, Any] = {"id": clip.clip_id, "status": "evaluated"} + if subset == "key" and clip.expected_key: + actual_key = payload.get("key") if isinstance(payload.get("key"), str) else None + score = mirex_key_score(clip.expected_key, actual_key) + entry.update({"expectedKey": clip.expected_key, "actualKey": actual_key, "mirex": score}) + key_scores.append(score) + if score == 1.0: + key_exact += 1 + if score in (1.0, 0.3): + key_exact_or_relative += 1 + if subset == "tempo" and clip.expected_bpm: + actual_bpm = payload.get("bpm") + actual_bpm = float(actual_bpm) if isinstance(actual_bpm, (int, float)) else None + acc1, acc2 = tempo_accuracies(clip.expected_bpm, actual_bpm) + entry.update( + {"expectedBpm": clip.expected_bpm, "actualBpm": actual_bpm, "acc1": acc1, "acc2": acc2} + ) + acc1_hits += int(acc1) + acc2_hits += int(acc2) + per_clip.append(entry) + + evaluated = sum(1 for entry in per_clip if entry["status"] == "evaluated") + summary: dict[str, Any] = { + "subset": subset, + "clipsListed": len(selected), + "clipsEvaluated": evaluated, + "clipsMissingAudio": missing_audio, + "clipsAnalyzeFailed": analyze_failed, + "fastMode": fast, + "status": "evaluated" if evaluated > 0 else "underpowered", + } + if subset == "key" and key_scores: + summary["keyExactRate"] = round(key_exact / len(key_scores), 4) + summary["keyExactOrRelativeRate"] = round(key_exact_or_relative / len(key_scores), 4) + summary["mirexWeighted"] = round(sum(key_scores) / len(key_scores), 4) + if subset == "tempo" and evaluated: + summary["tempoAcc1"] = round(acc1_hits / evaluated, 4) + summary["tempoAcc2"] = round(acc2_hits / evaluated, 4) + + report: dict[str, Any] = { + "schemaVersion": "giantsteps-eval-report.v1", + "generatedAt": datetime.now(timezone.utc).isoformat(), + "summary": summary, + "clips": per_clip, + } + if report_path is not None: + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + report["reportPath"] = str(report_path) + return report diff --git a/apps/backend/scripts/build_beat_manifest.py b/apps/backend/scripts/build_beat_manifest.py index bf20635d..b028433a 100644 --- a/apps/backend/scripts/build_beat_manifest.py +++ b/apps/backend/scripts/build_beat_manifest.py @@ -25,17 +25,29 @@ def main() -> int: parser.add_argument("--root", type=Path, required=True, help="beat_tracks root (contains gtzan/audio + gtzan/annotations).") parser.add_argument("--out", type=Path, required=True, help="Output manifest JSON path.") parser.add_argument("--annotation-suffix", type=str, default=".beats", help="Annotation file suffix.") + parser.add_argument( + "--asa-slice", + action="store_true", + help=( + "Build the ASA electronic slice instead of GTZAN: scans a flat " + "/asa/{audio,annotations}/ layout (hand-annotated electronic " + "clips, e.g. GiantSteps Beatport previews) and marks every clip " + "asaRelevant. This is the frozen beat gate's MIN_CLIPS_ASA subset." + ), + ) args = parser.parse_args() - audio_root = args.root / "gtzan" / "audio" - annotation_root = args.root / "gtzan" / "annotations" + subset = "asa" if args.asa_slice else "gtzan" + audio_root = args.root / subset / "audio" + annotation_root = args.root / subset / "annotations" if not audio_root.is_dir(): raise SystemExit(f"No audio directory at {audio_root}") clips = [] missing_annotations = 0 - for genre_dir in sorted(p for p in audio_root.iterdir() if p.is_dir()): - genre = genre_dir.name + genre_dirs = [audio_root] if args.asa_slice else sorted(p for p in audio_root.iterdir() if p.is_dir()) + for genre_dir in genre_dirs: + genre = "electronic" if args.asa_slice else genre_dir.name for audio in sorted(genre_dir.iterdir()): if audio.suffix.lower() not in AUDIO_SUFFIXES: continue @@ -49,12 +61,16 @@ def main() -> int: "genre": genre, "audioPath": str(audio.relative_to(args.root.parent)), "annotationPath": str(annotation.relative_to(args.root.parent)), - "asaRelevant": genre in ASA_RELEVANT_GENRES, + "asaRelevant": True if args.asa_slice else genre in ASA_RELEVANT_GENRES, } ) manifest = { - "datasetName": "GTZAN + GTZAN-Rhythm (Marchand/Fresnel/Peeters 2015)", + "datasetName": ( + "ASA electronic slice (hand-annotated)" + if args.asa_slice + else "GTZAN + GTZAN-Rhythm (Marchand/Fresnel/Peeters 2015)" + ), "annotationFormat": "gtzan_rhythm", "currentShippingMethod": "kick_accent", "clips": clips, diff --git a/apps/backend/scripts/evaluate_giantsteps.py b/apps/backend/scripts/evaluate_giantsteps.py new file mode 100644 index 00000000..6557ab20 --- /dev/null +++ b/apps/backend/scripts/evaluate_giantsteps.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Score ASA key/tempo against the GiantSteps datasets (research-only). + +Corpus layout is produced by scripts/fetch_giantsteps.py (operator-run; see +tests/fixtures/giantsteps/README.md). The analyzer subprocess inherits this +process's environment, so backend experiments are scored the same way, e.g.: + + ASA_LOUDNESS_BACKEND=wasm ./venv/bin/python scripts/evaluate_giantsteps.py --subset key +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +BACKEND_DIR = Path(__file__).resolve().parent.parent +if str(BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(BACKEND_DIR)) + +from giantsteps_evaluation import ( # noqa: E402 + DEFAULT_CORPUS_ROOT, + DEFAULT_REPORT_DIR, + evaluate_corpus, + load_giantsteps_corpus, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate Phase 1 key/tempo accuracy on the GiantSteps corpus." + ) + parser.add_argument("--root", type=Path, default=DEFAULT_CORPUS_ROOT) + parser.add_argument("--subset", choices=("key", "tempo"), required=True) + parser.add_argument("--max-clips", type=int, default=None) + parser.add_argument("--report", type=Path, default=None) + parser.add_argument( + "--full", + action="store_true", + help="Run the full analyze pipeline instead of --fast (slower; same key/bpm fields).", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + clips = load_giantsteps_corpus(args.root, args.subset) + report_path = args.report or (DEFAULT_REPORT_DIR / f"giantsteps_{args.subset}.json") + report = evaluate_corpus( + clips, + subset=args.subset, + fast=not args.full, + max_clips=args.max_clips, + report_path=report_path, + ) + print(json.dumps({"summary": report["summary"], "reportPath": report.get("reportPath")}, indent=2)) + if report["summary"]["status"] != "evaluated": + print( + f"No evaluable clips under {args.root} — fetch the corpus first " + "(scripts/fetch_giantsteps.py; see tests/fixtures/giantsteps/README.md).", + file=sys.stderr, + ) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/apps/backend/scripts/fetch_giantsteps.py b/apps/backend/scripts/fetch_giantsteps.py new file mode 100644 index 00000000..518968df --- /dev/null +++ b/apps/backend/scripts/fetch_giantsteps.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Fetch the GiantSteps Key + Tempo corpora (operator-run, local only). + +Clones the two annotation repositories (MIT-licensed annotations) and +downloads the matching Beatport preview clips, verifying each file against +the repos' committed MD5 checksums. Audio is Beatport's — it lives only in +the gitignored corpus directory and is NEVER committed. + +Run this on your own machine (cloud sessions typically sit behind a proxy +that blocks the download hosts): + + ./venv/bin/python scripts/fetch_giantsteps.py # both subsets + ./venv/bin/python scripts/fetch_giantsteps.py --subset key + ./venv/bin/python scripts/fetch_giantsteps.py --verify-only + +Preview URLs rot (the datasets are from 2015); each file is tried against +the Beatport sample host first and the JKU mirror second, and an existing +file that passes its checksum is never re-downloaded. Re-run with +--verify-only after any interruption to see corpus health. +""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import subprocess +import sys +import urllib.request +from pathlib import Path + +BACKEND_DIR = Path(__file__).resolve().parent.parent +DEFAULT_ROOT = BACKEND_DIR / "tests" / "fixtures" / "giantsteps" + +REPOS = { + "key": "https://github.com/GiantSteps/giantsteps-key-dataset", + "tempo": "https://github.com/GiantSteps/giantsteps-tempo-dataset", +} + +# Preview mirrors, tried in order. {name} is e.g. "1234567.LOFI.mp3". +AUDIO_MIRRORS = ( + "http://geo-samples.beatport.com/lofi/{name}", + "http://www.cp.jku.at/datasets/giantsteps/backup/{name}", +) + + +def _clone_or_update(repo_url: str, dest: Path) -> None: + if (dest / ".git").is_dir(): + subprocess.run(["git", "-C", str(dest), "pull", "--ff-only"], check=True) + return + dest.parent.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "clone", "--depth", "1", repo_url, str(dest)], check=True) + + +def _md5(path: Path) -> str: + digest = hashlib.md5() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _load_checksums(repo_dir: Path) -> dict[str, str]: + """Map audio filename -> expected md5 from the repo's md5/ directory.""" + checksums: dict[str, str] = {} + md5_dir = repo_dir / "md5" + if not md5_dir.is_dir(): + return checksums + for md5_file in md5_dir.rglob("*.md5"): + text = md5_file.read_text(encoding="utf-8", errors="replace").strip() + if not text: + continue + # Formats seen upstream: " " or just "". + parts = text.split() + digest = parts[0].lower() + name = parts[1] if len(parts) > 1 else md5_file.name[: -len(".md5")] + checksums[Path(name).name] = digest + return checksums + + +def _annotation_stems(repo_dir: Path, subset: str) -> list[str]: + suffix = ".key" if subset == "key" else ".bpm" + annotations = repo_dir / "annotations" + return sorted( + path.name[: -len(suffix)] + for path in annotations.rglob(f"*{suffix}") + ) + + +def _stage_annotations(repo_dir: Path, subset_root: Path, subset: str) -> int: + suffix = ".key" if subset == "key" else ".bpm" + dest = subset_root / "annotations" + dest.mkdir(parents=True, exist_ok=True) + count = 0 + for path in (repo_dir / "annotations").rglob(f"*{suffix}"): + shutil.copy2(path, dest / path.name) + count += 1 + return count + + +def _download(name: str, dest: Path) -> bool: + for mirror in AUDIO_MIRRORS: + url = mirror.format(name=name) + try: + with urllib.request.urlopen(url, timeout=60) as response: + data = response.read() + if len(data) < 10_000: # error pages are small; previews are ~2 MB + continue + dest.write_bytes(data) + return True + except Exception as exc: # noqa: BLE001 — report and try next mirror + print(f" [warn] {url}: {exc}", file=sys.stderr) + return False + + +def process_subset(root: Path, subset: str, *, verify_only: bool) -> dict[str, int]: + repo_dir = root / "_repos" / f"giantsteps-{subset}-dataset" + subset_root = root / subset + audio_dir = subset_root / "audio" + audio_dir.mkdir(parents=True, exist_ok=True) + + if not verify_only: + _clone_or_update(REPOS[subset], repo_dir) + staged = _stage_annotations(repo_dir, subset_root, subset) + print(f"[{subset}] staged {staged} annotations") + + checksums = _load_checksums(repo_dir) + stems = _annotation_stems(repo_dir, subset) if repo_dir.is_dir() else [] + stats = {"total": len(stems), "present": 0, "downloaded": 0, "checksum_failed": 0, "unavailable": 0} + + for stem in stems: + name = f"{stem}.mp3" + dest = audio_dir / name + expected = checksums.get(name) + if dest.exists(): + if expected and _md5(dest) != expected: + print(f" [warn] checksum mismatch: {name} (delete to re-fetch)", file=sys.stderr) + stats["checksum_failed"] += 1 + else: + stats["present"] += 1 + continue + if verify_only: + stats["unavailable"] += 1 + continue + if _download(name, dest): + if expected and _md5(dest) != expected: + print(f" [warn] checksum mismatch after download: {name}", file=sys.stderr) + dest.unlink(missing_ok=True) + stats["checksum_failed"] += 1 + else: + stats["downloaded"] += 1 + else: + stats["unavailable"] += 1 + + print(f"[{subset}] {stats}") + return stats + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch/verify the GiantSteps corpora locally.") + parser.add_argument("--root", type=Path, default=DEFAULT_ROOT) + parser.add_argument("--subset", choices=("key", "tempo", "both"), default="both") + parser.add_argument("--verify-only", action="store_true") + args = parser.parse_args() + + subsets = ("key", "tempo") if args.subset == "both" else (args.subset,) + any_audio = False + for subset in subsets: + stats = process_subset(args.root, subset, verify_only=args.verify_only) + any_audio = any_audio or stats["present"] + stats["downloaded"] > 0 + if not any_audio: + print( + "No audio present. Preview mirrors may have rotted — see the README " + "for alternate acquisition (mirdata) and keep annotations from the repos.", + file=sys.stderr, + ) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/apps/backend/tests/fixtures/beat_tracks/README.md b/apps/backend/tests/fixtures/beat_tracks/README.md index 1a0082a0..96b93c8a 100644 --- a/apps/backend/tests/fixtures/beat_tracks/README.md +++ b/apps/backend/tests/fixtures/beat_tracks/README.md @@ -36,7 +36,13 @@ beat_tracks/ harness reports per-genre metrics so outliers are visible; consider excluding the documented corrupt files. - **Genre skew**: GTZAN is mixed-genre. The pass bar is judged on the - `asaRelevant` subset (disco, hiphop, pop — the electronic-adjacent genres). A - modern ASA electronic slice (`beat_eval_manifest_asa.json`) is the representativeness - fast-follow; use only **user-owned / unreleased** tracks so the slice stays - contamination-free for beat_this. + `asaRelevant` subset (disco, hiphop, pop — the electronic-adjacent genres). The + modern ASA electronic slice (`beat_eval_manifest_asa.json`, built with + `build_beat_manifest.py --asa-slice` from `/asa/{audio,annotations}/`) + is the representativeness requirement — the frozen gate needs >= 15 clips + (`MIN_CLIPS_ASA`) or it reports `underpowered`. Contamination-safe sources: + **user-owned / unreleased** tracks, or **hand-annotated GiantSteps Beatport + previews** (fetched by `scripts/fetch_giantsteps.py`; GiantSteps carries no + beat annotations upstream, so it is outside beat_this's training data — + annotate bar-1 downbeats by hand, ~1-2 h for 15-20 clips, the one manual + labeling task in the accuracy program). diff --git a/apps/backend/tests/fixtures/giantsteps/.gitignore b/apps/backend/tests/fixtures/giantsteps/.gitignore new file mode 100644 index 00000000..c228bb25 --- /dev/null +++ b/apps/backend/tests/fixtures/giantsteps/.gitignore @@ -0,0 +1,3 @@ +* +!README.md +!.gitignore diff --git a/apps/backend/tests/fixtures/giantsteps/README.md b/apps/backend/tests/fixtures/giantsteps/README.md new file mode 100644 index 00000000..7b03d5fd --- /dev/null +++ b/apps/backend/tests/fixtures/giantsteps/README.md @@ -0,0 +1,53 @@ +# GiantSteps Key + Tempo corpora (operator-fetched, never committed) + +Real-audio ground truth for ASA's key and tempo measurements: ~600 annotated +Beatport preview clips per subset, i.e. actual electronic music across the +EDM genre surface. This is the accuracy program's real-world check for the +key/tempo domains — synthetic-corpus wins must reproduce here before any +backend promotion (see `audits/accuracy-baseline-2026-07-03.md`). + +## Layout + +``` +giantsteps/ + key/ + annotations/.LOFI.key # e.g. "D minor" + audio/.LOFI.mp3 # gitignored + tempo/ + annotations/.LOFI.bpm # single BPM float + audio/.LOFI.mp3 # gitignored + _repos/ # gitignored upstream clones +``` + +## Fetch (run locally — cloud proxies usually block the audio hosts) + +```bash +cd apps/backend +./venv/bin/python scripts/fetch_giantsteps.py # both subsets +./venv/bin/python scripts/fetch_giantsteps.py --verify-only +``` + +The fetcher clones the upstream annotation repos, stages annotations, +downloads previews from the Beatport sample host with the JKU mirror as +fallback, and verifies the repos' MD5 checksums. Preview URLs rot (2015-era +datasets); if both mirrors fail, `mirdata` (`pip install mirdata`, datasets +`giantsteps_key` / `giantsteps_tempo` — install into a scratch venv, not the +product venv) is the alternate acquisition path; keep the same layout. + +## Evaluate + +```bash +./venv/bin/python scripts/evaluate_giantsteps.py --subset key +./venv/bin/python scripts/evaluate_giantsteps.py --subset tempo --max-clips 100 +``` + +Reports land in `.runtime/reports/giantsteps_.json` with MIREX +weighted key score / exact / exact-or-relative rates, and tempo Acc1/Acc2. +Zero evaluable clips exits 1 (`status: underpowered`) — never vacuous green. + +## Licence + +The annotation repositories are MIT-licensed and redistributable; the audio +previews are Beatport's content served for preview purposes. **Never commit +audio.** Everything under `key/audio/`, `tempo/audio/`, and `_repos/` is +gitignored; only this README is committed. diff --git a/apps/backend/tests/test_giantsteps_evaluation.py b/apps/backend/tests/test_giantsteps_evaluation.py new file mode 100644 index 00000000..d7e24fab --- /dev/null +++ b/apps/backend/tests/test_giantsteps_evaluation.py @@ -0,0 +1,158 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from giantsteps_evaluation import ( + GiantstepsClip, + evaluate_corpus, + load_giantsteps_corpus, + mirex_key_score, + tempo_accuracies, +) + + +class MirexKeyScoreTests(unittest.TestCase): + def test_score_table(self) -> None: + self.assertEqual(mirex_key_score("D minor", "D minor"), 1.0) + self.assertEqual(mirex_key_score("Db minor", "C# Minor"), 1.0) # enharmonic exact + self.assertEqual(mirex_key_score("C major", "G major"), 0.5) # fifth above + self.assertEqual(mirex_key_score("C major", "F major"), 0.0) # fifth below ≠ MIREX fifth + self.assertEqual(mirex_key_score("C major", "A minor"), 0.3) # relative + self.assertEqual(mirex_key_score("A minor", "C major"), 0.3) # relative, other way + self.assertEqual(mirex_key_score("C major", "C minor"), 0.2) # parallel + self.assertEqual(mirex_key_score("C major", "D major"), 0.0) + self.assertEqual(mirex_key_score("C major", None), 0.0) + self.assertEqual(mirex_key_score("garbage", "C major"), 0.0) + + +class TempoAccuracyTests(unittest.TestCase): + def test_acc1_tolerance_boundary(self) -> None: + self.assertEqual(tempo_accuracies(128.0, 128.0), (True, True)) + # 4% of 128 = 5.12 — just inside vs just outside the tolerance. + self.assertEqual(tempo_accuracies(128.0, 133.1), (True, True)) + self.assertEqual(tempo_accuracies(128.0, 133.2)[0], False) + + def test_acc2_octave_and_triple_families(self) -> None: + self.assertEqual(tempo_accuracies(174.0, 87.0), (False, True)) # half + self.assertEqual(tempo_accuracies(87.0, 174.0), (False, True)) # double + self.assertEqual(tempo_accuracies(60.0, 180.0), (False, True)) # triple + self.assertEqual(tempo_accuracies(180.0, 60.0), (False, True)) # third + self.assertEqual(tempo_accuracies(128.0, 100.0), (False, False)) + + def test_missing_or_invalid_values(self) -> None: + self.assertEqual(tempo_accuracies(128.0, None), (False, False)) + self.assertEqual(tempo_accuracies(0.0, 128.0), (False, False)) + + +class LoadCorpusTests(unittest.TestCase): + def test_loads_annotations_and_matches_audio_by_stem(self) -> None: + with tempfile.TemporaryDirectory(prefix="asa_giantsteps_") as temp_dir: + root = Path(temp_dir) + (root / "key" / "annotations").mkdir(parents=True) + (root / "key" / "audio").mkdir(parents=True) + (root / "key" / "annotations" / "111.LOFI.key").write_text("D minor") + (root / "key" / "annotations" / "222.LOFI.key").write_text("Ab major") + (root / "key" / "audio" / "111.LOFI.mp3").write_bytes(b"x") + + clips = load_giantsteps_corpus(root, "key") + + self.assertEqual([c.clip_id for c in clips], ["111.LOFI", "222.LOFI"]) + self.assertTrue(clips[0].audio_path.exists()) + self.assertFalse(clips[1].audio_path.exists()) + self.assertEqual(clips[0].expected_key, "D minor") + + def test_parses_bpm_annotations(self) -> None: + with tempfile.TemporaryDirectory(prefix="asa_giantsteps_") as temp_dir: + root = Path(temp_dir) + (root / "tempo" / "annotations").mkdir(parents=True) + (root / "tempo" / "annotations" / "333.LOFI.bpm").write_text("137.5\n") + (root / "tempo" / "annotations" / "444.LOFI.bpm").write_text("not-a-number") + + clips = load_giantsteps_corpus(root, "tempo") + + self.assertEqual(clips[0].expected_bpm, 137.5) + self.assertIsNone(clips[1].expected_bpm) + + def test_missing_directory_returns_empty(self) -> None: + self.assertEqual(load_giantsteps_corpus(Path("/nonexistent"), "key"), []) + + +class EvaluateCorpusTests(unittest.TestCase): + def _clip(self, tmp: Path, clip_id: str, *, key: str | None = None, bpm: float | None = None, audio: bool = True) -> GiantstepsClip: + audio_path = tmp / f"{clip_id}.mp3" + if audio: + audio_path.write_bytes(b"x") + return GiantstepsClip(clip_id=clip_id, audio_path=audio_path, expected_key=key, expected_bpm=bpm) + + def test_key_subset_aggregates_mirex_rates(self) -> None: + with tempfile.TemporaryDirectory(prefix="asa_gs_eval_") as temp_dir: + tmp = Path(temp_dir) + clips = [ + self._clip(tmp, "exact", key="D minor"), + self._clip(tmp, "relative", key="C major"), + self._clip(tmp, "wrong", key="E major"), + ] + answers = {"exact": "D minor", "relative": "A minor", "wrong": "Bb minor"} + + def runner(path: Path, flags): + self.assertEqual(flags, ["--fast"]) + return {"key": answers[path.stem]} + + report = evaluate_corpus(clips, subset="key", runner=runner, report_path=tmp / "r.json") + + summary = report["summary"] + self.assertEqual(summary["status"], "evaluated") + self.assertEqual(summary["clipsEvaluated"], 3) + self.assertAlmostEqual(summary["keyExactRate"], 1 / 3, places=4) + self.assertAlmostEqual(summary["keyExactOrRelativeRate"], 2 / 3, places=4) + self.assertAlmostEqual(summary["mirexWeighted"], (1.0 + 0.3 + 0.0) / 3, places=4) + self.assertTrue((tmp / "r.json").exists()) + + def test_tempo_subset_counts_acc1_acc2(self) -> None: + with tempfile.TemporaryDirectory(prefix="asa_gs_eval_") as temp_dir: + tmp = Path(temp_dir) + clips = [ + self._clip(tmp, "hit", bpm=128.0), + self._clip(tmp, "octave", bpm=174.0), + ] + answers = {"hit": 128.2, "octave": 87.0} + + def runner(path: Path, flags): + return {"bpm": answers[path.stem]} + + report = evaluate_corpus(clips, subset="tempo", runner=runner) + + summary = report["summary"] + self.assertEqual(summary["tempoAcc1"], 0.5) + self.assertEqual(summary["tempoAcc2"], 1.0) + + def test_empty_or_missing_audio_is_underpowered_not_green(self) -> None: + with tempfile.TemporaryDirectory(prefix="asa_gs_eval_") as temp_dir: + tmp = Path(temp_dir) + clips = [self._clip(tmp, "ghost", key="C major", audio=False)] + + def runner(path: Path, flags): # pragma: no cover - must not be called + raise AssertionError("runner must not run for missing audio") + + report = evaluate_corpus(clips, subset="key", runner=runner) + + self.assertEqual(report["summary"]["status"], "underpowered") + self.assertEqual(report["summary"]["clipsMissingAudio"], 1) + + def test_max_clips_limits_work(self) -> None: + with tempfile.TemporaryDirectory(prefix="asa_gs_eval_") as temp_dir: + tmp = Path(temp_dir) + clips = [self._clip(tmp, f"c{i}", key="C major") for i in range(5)] + calls: list[str] = [] + + def runner(path: Path, flags): + calls.append(path.stem) + return {"key": "C major"} + + evaluate_corpus(clips, subset="key", runner=runner, max_clips=2) + self.assertEqual(len(calls), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/plans/owner-actions-accuracy-program.md b/plans/owner-actions-accuracy-program.md new file mode 100644 index 00000000..3cc324a7 --- /dev/null +++ b/plans/owner-actions-accuracy-program.md @@ -0,0 +1,62 @@ +# Owner actions — Phase 1 accuracy program + +Operator tasks the accuracy program needs that require your machine (the +cloud sessions sit behind a proxy that blocks the dataset hosts). Everything +here is one-time setup; each item unblocks a specific gate. Code side is +done — see `audits/accuracy-baseline-2026-07-03.md` for what's already +measured without you. + +## 1. Fetch GiantSteps Key + Tempo (~30 min, mostly download time) + +Unblocks: real-audio key/tempo accuracy numbers (the synthetic baseline's +reality check), and the key-ensemble gate (PR-B3). + +```bash +cd apps/backend +./venv/bin/python scripts/fetch_giantsteps.py # both subsets +./venv/bin/python scripts/evaluate_giantsteps.py --subset key +./venv/bin/python scripts/evaluate_giantsteps.py --subset tempo +``` + +If the 2015-era preview mirrors have rotted, the fallback is `mirdata` +(scratch venv, datasets `giantsteps_key` / `giantsteps_tempo`) — keep the +layout in `apps/backend/tests/fixtures/giantsteps/README.md`. + +## 2. Fetch GTZAN + GTZAN-Rhythm and build venv-eval (~45 min) + +Unblocks: the frozen, pre-registered beat_this ship/no-ship decision +(`incorporations/beat-this-measurement-gate-2026-05-20.md`). + +- GTZAN audio + the GTZAN-Rhythm annotations (mirror: + `github.com/TempoBeatDownbeat/gtzan_tempo_beat`), laid out per + `apps/backend/tests/fixtures/beat_tracks/README.md`. +- `python3.11 -m venv apps/backend/venv-eval && venv-eval/bin/pip install -r apps/backend/requirements-eval.txt` + (beat_this + mir_eval — never into the product venv). +- `./venv/bin/python scripts/build_beat_manifest.py --root tests/fixtures/beat_tracks --out tests/fixtures/beat_eval_manifest.gtzan.json` + +## 3. Hand-annotate the ASA electronic slice (~1–2 h — the ONE manual labeling task) + +Unblocks: the beat gate's `MIN_CLIPS_ASA=15` requirement (without it the +gate reports `underpowered` and must not be finalized). + +- Pick 15–20 GiantSteps previews (from action 1) spanning house/techno/dnb/garage. +- Annotate beat times + bar-1 downbeats (Audacity label track or Sonic + Visualiser; export as `.beats` — one time per line, GTZAN-Rhythm format). +- Place under `apps/backend/tests/fixtures/beat_tracks/asa/{audio,annotations}/`. +- `./venv/bin/python scripts/build_beat_manifest.py --asa-slice --root tests/fixtures/beat_tracks --out tests/fixtures/beat_eval_manifest_asa.json` + +GiantSteps previews are contamination-safe for beat_this (the dataset has no +upstream beat annotations, so it was never in the model's training data). + +## 4. (Later, optional) Calibration corpus tracks + +Unblocks: PR-D3 confidence-threshold calibration. Place ~10 tracks (synthetic +renders + GiantSteps previews are fine — your personal library is NOT +required) under `tests/ground_truth/tracks/` and fill `labels.json` with +verified values. The fake cache stubs were deleted 2026-07-03. + +## Sequencing + +Action 1 is independent and highest-value-per-minute. Actions 2+3 together +unblock the beat gate run (PR-C2) — the program's biggest expected accuracy +jump (meter → downbeats is the measured weak layer). diff --git a/tests/ground_truth/README.md b/tests/ground_truth/README.md index fd28821b..8e573831 100644 --- a/tests/ground_truth/README.md +++ b/tests/ground_truth/README.md @@ -6,9 +6,18 @@ This directory holds the ground truth dataset used by `scripts/calibrate_confide ## Current state -**Warning: this dataset is not ready for real calibration yet.** `labels.json` currently contains placeholder entries, `cache/` currently contains hand-crafted stubs that do not represent real analyzer output, and `tracks/` does not exist yet, so no audio files have been added. If you run the calibration script against this directory as-is, it aborts with exit code `1` after reporting that all tracks are being served from cache with no audio files, that the cache stubs may not represent real analysis output, and that calibration has been aborted. - -`ARTIFACT_CLEANUP_MAX` is a separate cleanup safeguard elsewhere in the backend. It is not the error message currently emitted by `scripts/calibrate_confidence.py` for this missing-audio ground truth directory. +**Status: awaiting real audio.** `labels.json` contains placeholder entries +documenting the expected schema; `tracks/` does not exist yet. The fake +`cache/` stubs that previously lived here were deleted (2026-07-03) — they +did not represent real analyzer output and would have silently corrupted any +calibration report. Running `scripts/calibrate_confidence.py` against this +directory as-is aborts with exit code `1` until audio is added. + +**Corpus strategy (accuracy program):** the calibration corpus is built from +the synthetic fundamentals corpus (`apps/backend/scripts/build_synthetic_corpus.py`) +plus public EDM previews (GiantSteps intake — `apps/backend/scripts/fetch_giantsteps.py`). +The owner's personal library is NOT required. Update `labels.json` with +human-verified values for whichever tracks are placed in `tracks/`. ## How to populate diff --git a/tests/ground_truth/cache/track_01_techno.json b/tests/ground_truth/cache/track_01_techno.json deleted file mode 100644 index e898e1fa..00000000 --- a/tests/ground_truth/cache/track_01_techno.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.82}, - "chordDetail": {"chordStrength": 0.45}, - "sidechainDetail": {"pumpingConfidence": 0.65} -} diff --git a/tests/ground_truth/cache/track_02_house.json b/tests/ground_truth/cache/track_02_house.json deleted file mode 100644 index 3753a765..00000000 --- a/tests/ground_truth/cache/track_02_house.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.88}, - "chordDetail": {"chordStrength": 0.85}, - "sidechainDetail": {"pumpingConfidence": 0.72} -} diff --git a/tests/ground_truth/cache/track_03_dnb.json b/tests/ground_truth/cache/track_03_dnb.json deleted file mode 100644 index 2853721c..00000000 --- a/tests/ground_truth/cache/track_03_dnb.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.12}, - "chordDetail": {"chordStrength": 0.35}, - "sidechainDetail": {"pumpingConfidence": 0.15} -} diff --git a/tests/ground_truth/cache/track_04_ambient.json b/tests/ground_truth/cache/track_04_ambient.json deleted file mode 100644 index 1954852c..00000000 --- a/tests/ground_truth/cache/track_04_ambient.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.75}, - "chordDetail": {"chordStrength": 0.78}, - "sidechainDetail": {"pumpingConfidence": 0.10} -} diff --git a/tests/ground_truth/cache/track_05_electro.json b/tests/ground_truth/cache/track_05_electro.json deleted file mode 100644 index 7637caf3..00000000 --- a/tests/ground_truth/cache/track_05_electro.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.85}, - "chordDetail": {"chordStrength": 0.82}, - "sidechainDetail": {"pumpingConfidence": 0.68} -} diff --git a/tests/ground_truth/cache/track_06_breaks.json b/tests/ground_truth/cache/track_06_breaks.json deleted file mode 100644 index 782fc4c8..00000000 --- a/tests/ground_truth/cache/track_06_breaks.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.08}, - "chordDetail": {"chordStrength": 0.75}, - "sidechainDetail": {"pumpingConfidence": 0.20} -} diff --git a/tests/ground_truth/cache/track_07_psy.json b/tests/ground_truth/cache/track_07_psy.json deleted file mode 100644 index f5a16855..00000000 --- a/tests/ground_truth/cache/track_07_psy.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.91}, - "chordDetail": {"chordStrength": 0.48}, - "sidechainDetail": {"pumpingConfidence": 0.55} -} diff --git a/tests/ground_truth/cache/track_08_dub.json b/tests/ground_truth/cache/track_08_dub.json deleted file mode 100644 index a2b57117..00000000 --- a/tests/ground_truth/cache/track_08_dub.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.79}, - "chordDetail": {"chordStrength": 0.88}, - "sidechainDetail": {"pumpingConfidence": 0.58} -} diff --git a/tests/ground_truth/cache/track_09_idm.json b/tests/ground_truth/cache/track_09_idm.json deleted file mode 100644 index da8dd3a8..00000000 --- a/tests/ground_truth/cache/track_09_idm.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.15}, - "chordDetail": {"chordStrength": 0.42}, - "sidechainDetail": {"pumpingConfidence": 0.18} -} diff --git a/tests/ground_truth/cache/track_10_industrial.json b/tests/ground_truth/cache/track_10_industrial.json deleted file mode 100644 index a8cee5cb..00000000 --- a/tests/ground_truth/cache/track_10_industrial.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "melodyDetail": {"pitchConfidence": 0.05}, - "chordDetail": {"chordStrength": 0.80}, - "sidechainDetail": {"pumpingConfidence": 0.62} -}