From 3e02a4089f9981a893feff9e8e39d661cef62010 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 12 Jul 2026 20:48:46 -0300 Subject: [PATCH 01/31] =?UTF-8?q?feat(#870):=20mocap=20spike=20=E2=80=94?= =?UTF-8?q?=20MediaPipe=E2=86=92ONNX=20conversion+parity,=20SAM=20licensin?= =?UTF-8?q?g,=20MHR=20skeleton?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice A of epic #869 (performance capture). Offline dev tooling + decision records only; no app code. - scripts/export-facecap-onnx.py: converts the five MediaPipe TFLite models (face detector/landmarks/blendshapes, pose detector/landmarks) to ONNX and asserts numerical parity against the python mediapipe reference in the same run (landmarks <=0.59px, blendshapes <=0.0148, pose world <=1.02cm on the MediaPipe Apache-2.0 test images). Handles the fp16 block-sparse DENSIFY weights in the pose detector (interpreter densify pass) and the tf2onnx-optimizer-broken blendshapes graph (unoptimized fallback). - scripts/export-bodycap-onnx.py: extracts the 127-joint MHR skeleton (names/hierarchy/pre-rotations/rest world pose) from the Apache-2.0 mhr_model.pt into mhr_skeleton.json for the Slice E retarget; documents the SAM 3D Body export recipe (blocked on gated HF access). - docs/MOCAP_SPIKE.md: full pre/post-processing contracts (the Slice C/E implementation spec), conversion gotchas, parity numbers, latencies (face 11.2ms, pose 17.2ms per frame on M-series CPU), go/no-go. - THIRD_PARTY_AI_MODELS.md: MediaPipe (Apache-2.0) entry; SAM License due-diligence verdict (PASS with conditions) + rejected alternatives. Co-Authored-By: Claude Fable 5 --- .gitignore | 3 + THIRD_PARTY_AI_MODELS.md | 55 +++ docs/MOCAP_SPIKE.md | 194 ++++++++ scripts/export-bodycap-onnx.py | 140 ++++++ scripts/export-facecap-onnx.py | 789 +++++++++++++++++++++++++++++++++ 5 files changed, 1181 insertions(+) create mode 100644 docs/MOCAP_SPIKE.md create mode 100644 scripts/export-bodycap-onnx.py create mode 100644 scripts/export-facecap-onnx.py diff --git a/.gitignore b/.gitignore index a9421e6f3..038f102f0 100755 --- a/.gitignore +++ b/.gitignore @@ -154,3 +154,6 @@ __pycache__/ !docs/img/twist_bar_90_lbs.png !docs/img/twist_bar_90_lbs_vp.png !docs/img/twist_bar_90_dqs_vp.png +!docs/MOCAP_SPIKE.md +.mocap_work/ +.venv-mocap/ diff --git a/THIRD_PARTY_AI_MODELS.md b/THIRD_PARTY_AI_MODELS.md index 5384598a6..b72931ca5 100644 --- a/THIRD_PARTY_AI_MODELS.md +++ b/THIRD_PARTY_AI_MODELS.md @@ -271,6 +271,61 @@ the binary). Attribution + licenses for the models and their training data: checkbox; the template library remains the default and the automatic fallback. Same CMU licensing basis as above. +## Performance capture (epic #869) + +### MediaPipe Face Landmarker + Pose Landmarker — face/pose capture (#870/#872/#874) + +- **Models:** Google MediaPipe `face_landmarker.task` (BlazeFace short-range + detector + Face Mesh V2 478-landmark model + 52-blendshape MLP-Mixer) and + `pose_landmarker_full.task` (BlazePose detector + 39-landmark model with + world coordinates), converted TFLite → ONNX. +- **License:** **Apache-2.0, code AND models** (Google's MediaPipe release — + the stack the entire VTuber ecosystem builds on). Ship the Apache-2.0 + notice next to the hosted weights. +- The ONNX export is produced by `scripts/export-facecap-onnx.py` (one-time, + offline dev tool — not shipped), which also **asserts numerical parity** + against the Python `mediapipe` reference (landmarks ≤ 0.59 px, blendshapes + ≤ 0.0148 abs, pose world landmarks ≤ 1.02 cm on the test set). Conversion + recipe, pre/post-processing contracts and measured latencies: + `docs/MOCAP_SPIKE.md`. The app runs the five graphs via ONNX Runtime, + downloading them on first use to `AppData/ai_models/mocap/{face,pose}/`. +- **Rejected face alternatives:** DECA / EMOCA / SPECTRE (all regress the + FLAME 3DMM — research-only license), ARKit (iOS-only), OpenSeeFace (MIT + code but weaker blendshape story). + +### SAM 3D Body + MHR — body capture quality path (#870/#874) — decision record + +- **MHR (Momentum Human Rig)** — the 127-joint parametric rig SAM 3D Body + poses — is **Apache-2.0** (https://github.com/facebookresearch/MHR, assets + v1.0.1). Skeleton definition (names, hierarchy, pre-rotations, derived rest + world pose) extracted to `mhr_skeleton.json` by + `scripts/export-bodycap-onnx.py --mhr-assets`. No restrictions. +- **SAM 3D Body checkpoints** (`facebook/sam-3d-body-dinov3`, 2.1 GB) are + under the **SAM License** (2025-11-19 text reviewed in full, 2026-07-12). + **Verdict: PASS with conditions** — usable as an OPTIONAL, + downloaded-on-first-use backend, never bundled: + - §1.a grants use/reproduction/distribution/modification (ONNX conversion + is a permitted modification, not the prohibited "reverse engineering" of + §1.b.iv). §1.b.i permits redistributing derivatives **only under the SAM + License with a copy attached** — rehosting converted ONNX on our HF repo + is compliant with the license file shipped next to the weights. + - **No non-commercial clause; model outputs are unrestricted** (§3 only + disclaims warranty over outputs). No EU exclusion (unlike Hunyuan3D). + - Conditions/risks recorded: the SAM backend is NOT permissive-equivalent — + users of that one optional feature are bound by the SAM License + (AUP-style trade-controls/military restrictions pass through); Meta may + unilaterally amend the terms (§8); upstream access is gated (HF + click-through sharing contact info with Meta). + - **Status:** checkpoint download blocked pending the gated-access + acceptance on the HF model page (our token is not yet on the authorized + list). Export recipe proven by the community port + (AmmarkoV/SAM3DBody-cpp: DINOv3-H+ backbone ~4.8 GB fp32 + 93 MB decoder + → 519 MHR params). Until unblocked the body path ships **MediaPipe Pose + + analytic IK only** (Apache-2.0, zero conditions). +- **Rejected body alternatives:** WHAM / GVHMR / TRAM / 4D-Humans (all regress + **SMPL** — weights non-commercial, Meshcapade sells the commercial + license), OpenPose (CMU non-commercial), FreeMoCap (AGPL). + All of the above clear QtMeshEditor's permissive-redistribution bar (MIT app, distributed via Homebrew / WinGet / Snap / Docker). GPL/CC-BY-NC/unlicensed models are deliberately excluded (e.g. RigNet was rejected for #408 — GPL code + diff --git a/docs/MOCAP_SPIKE.md b/docs/MOCAP_SPIKE.md new file mode 100644 index 000000000..79340c178 --- /dev/null +++ b/docs/MOCAP_SPIKE.md @@ -0,0 +1,194 @@ +# Performance capture — Spike Findings & Contracts (#869 / slice A #870) + +**Epic:** [#869 — AI: Performance capture: video/webcam → facial morph + skeletal body animation](https://github.com/fernandotonon/QtMeshEditor/issues/869) +**Slice:** [#870 — Spike: model conversion + licensing due diligence](https://github.com/fernandotonon/QtMeshEditor/issues/870) (de-risk first) +**Status:** Spike — **GO** for the face path and the MediaPipe-Pose-IK body +fallback (both risks fully retired, parity proven). SAM 3D Body: **license +passes with conditions**, export recipe proven by a community port, but the +checkpoint download is **gated** behind a per-account HF click-through we have +not completed yet — Slice E starts with Pose-IK and lights the SAM backend up +when access is granted (the UniRig "plumbing first, hosting later" precedent). + +--- + +## TL;DR + +- **MediaPipe → ONNX conversion — DONE, parity proven.** All five TFLite + models (face detector / face landmarks / face blendshapes / pose detector / + pose landmarks) convert with `tf2onnx` and load under ONNX Runtime. + `scripts/export-facecap-onnx.py` converts AND asserts parity against the + Python `mediapipe` reference in one run: + - face landmarks: worst **0.59 px** (target ≤ 2 px), typical 0.01–0.12 px mean + - blendshapes: worst **0.0148** abs (target ≤ 0.02), typical ≤ 0.003 mean + - pose world landmarks: worst **1.02 cm** (target ≤ 2 cm) +- **Latency (M-series CPU, ONNX Runtime CPU EP, detector every frame):** + face full pipeline **11.2 ms/frame (~89 fps)**, pose full **17.2 ms (~58 fps)**. + Live-mode ≥15 fps target clears with 5× headroom before detector-skip. +- **Model sizes:** face 6.4 MB total (0.4 + 4.9 + 1.1), pose 24.7 MB + (11.9 + 12.8). Trivial hosting/download. +- **Licenses:** all five MediaPipe models Apache-2.0 (code AND weights). + MHR rig definition Apache-2.0. SAM 3D Body weights under the **SAM License** + — verdict + conditions in `THIRD_PARTY_AI_MODELS.md`. +- **MHR skeleton — DONE.** 127-joint definition (names, hierarchy, + translation offsets, pre-rotations, derived rest world rotations/positions) + extracted from the Apache-2.0 `mhr_model.pt` by + `scripts/export-bodycap-onnx.py --mhr-assets` → `mhr_skeleton.json`. + +--- + +## Conversion recipe (pinned versions) + +Python 3.12 venv (arm64): `mediapipe 0.10.35`, `tensorflow 2.21.0`, +`tf2onnx 1.17.0`, `onnx 1.22.0`, `onnxruntime 1.27.0`, opencv (mediapipe dep). + +```bash +# .task bundles are zips of TFLite models (Apache-2.0): +# https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task +# https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task +python scripts/export-facecap-onnx.py \ + --face-task .mocap_work/models/face_landmarker.task \ + --pose-task .mocap_work/models/pose_landmarker_full.task \ + --images .mocap_work/images --out-dir .mocap_work/out +``` + +Three conversion gotchas, all handled inside the script: + +1. **`pose_detector.tflite` has fp16 block-sparse weights behind DENSIFY ops** + which tf2onnx cannot parse (`ValueError: cannot reshape array...`). Fix: + run the TFLite interpreter with `experimental_preserve_all_tensors=True` + **and `BUILTIN_WITHOUT_DEFAULT_DELEGATES`** (XNNPACK folds DENSIFY away and + `get_tensor()` then reads garbage — this cost an hour), capture each + DENSIFY output, rewrite the flatbuffer dense, drop the DENSIFY ops. +2. **`face_blendshapes.tflite` (MLP-Mixer) converts but tf2onnx's optimizer + emits a graph ORT rejects** (transpose optimizer breaks a LayerNorm Mul → + `ShapeInferenceError`). Fix: convert that model with optimizers disabled + (auto-fallback in `convert_with_fallback`). +3. **Resampling fidelity is the parity maker-or-breaker**: PIL's antialiased + BILINEAR gave 2.5 px / 0.31-blendshape errors on a rotated face; switching + letterbox + rotated-crop to cv2 `warpAffine` (plain bilinear, exact float + matrices, BORDER_ZERO — what MediaPipe itself does) collapsed errors to + 0.01 px. **Slice C must use non-antialiased bilinear in C++.** + +## The pre/post-processing contract (what Slice C implements) + +Machine-readable copy: `mocap_contract.json` (written next to the converted +models). Landmark/blendshape name tables live in the script constants. + +### Face (three sessions) + +1. **Detector** (BlazeFace short-range): letterbox (keep aspect, centre, zero + border) to **128×128**, RGB `/127.5 - 1` → `regressors [1,896,16]`, + `classificators [1,896,1]`. SSD anchors: strides `[8,16,16,16]`, + min/max scale 0.1484375/0.75, offset 0.5, fixed size, interpolated aspect + 1.0 → 896 anchors (each `[cx,cy]`). Decode: `xy = raw/128 + anchor`, + `wh = raw/128`; 6 keypoints (right eye, left eye, nose, mouth, right ear, + left ear) same decode; score `sigmoid(clip(raw,-100,100))`, threshold 0.5, + **weighted** NMS at IoU 0.3. Un-letterbox all coords. +2. **ROI**: rotation `θ = normalize(-atan2(-(eyeL.y-eyeR.y), eyeL.x-eyeR.x))` + (target angle 0, y-down image coords, R(θ)=[[c,-s],[s,c]]); rect = detection + box centre, side `max(w,h)·1.5` (square_long → scale). Crop: sample the + rotated rect to **256×256**, RGB `/255` (range [0,1]), zero border: + `p_img = centre + R(θ)·((u/256-0.5)·w, (v/256-0.5)·h)`. +3. **Landmarks** (Face Mesh V2): outputs `[N,1,1,1434]` = **478 landmarks × + (x,y,z) in 256-crop pixels** (no attention-refinement assembly needed — V2 + is single-tensor), `[N,1,1,1]` face-presence **logit** (apply sigmoid), + `[N,1]` unused aux. Project back: divide by 256, then + `x' = cx + dx·w·cosθ - dy·h·sinθ` (same R(θ)); `z' = z·w`. +4. **Blendshapes**: gather the 146-landmark subset (indices in the script / + contract JSON), feed **pixel coords** `(x·imgW, y·imgH)` as `[1,146,2]` → + `[52]` scores in [0,1]. Order = MediaPipe's canonical 52 (ARKit-compatible, + `_neutral` first) — the `FaceCap::kBlendshapeNames` vocabulary. +5. **Head pose**: weighted Kabsch/Umeyama fit of the 468 canonical face model + vertices (`canonical_face_model.obj`, cm) onto screen landmarks + `(x·W, -y·H, -z·W)` using MediaPipe's 33 Procrustes basis weights + (`geometry_pipeline_metadata_landmarks.pbtxt`); keep rotation (+translation), + solve-and-discard scale. Convention: +X right, +Y up, camera looks -Z; + identity = facing the camera. This intentionally replaces MediaPipe's + perspective geometry pipeline — measured delta vs its facial transformation + matrix is 4–9.3° on the test images (fine for driving a Head bone; the + deltas are systematic, not jitter, and the recorder calibrates the first + confident frame as neutral anyway). + +**Detector-skip strategy (live mode):** re-run the detector only when the +landmark presence drops below ~0.5; otherwise derive the next ROI from the +previous frame's landmarks (bounding box of the 478 points, same 1.5× +square-long expansion, rotation from eye corners 33→263 — MediaPipe's own +tracking mode). + +### Pose (two sessions) + +1. **Detector** (BlazePose): letterbox to **224×224**, `/127.5 - 1` → + `[1,2254,12]` + `[1,2254,1]`. Anchors: strides `[8,16,32,32,32]`, same + scale params → 2254. 4 keypoints; decode identical to face (scale 224). +2. **ROI**: centre = keypoint 0 (mid-hip), radius = |kp1 − kp0|, + box = square of side `2·radius·1.25`, rotation + `θ = normalize(π/2 − atan2(-(dy), dx))` (target angle 90°). +3. **Landmarks**: crop 256×256, `/255` → `[1,195]` = **39 × (x,y,z, + visibility-logit, presence-logit)** in 256-crop pixels (apply sigmoid to + vis/presence; landmarks 33..38 are auxiliary — drop), `[1,1]` pose-presence + (**already a probability** — do NOT sigmoid, unlike the face flag), + `[1,256,256,1]` segmentation (ignore), `[1,64,64,39]` heatmap (ignore), + `[1,117]` = **39 × (x,y,z) world landmarks in metres, hip-centred**. + Screen landmarks project like face; world landmarks are rotated by the ROI + rotation ONLY (no scale/translate). +4. Landmark index → body part table: 0 nose, 1-6 eyes, 7/8 ears, 9/10 mouth, + 11/12 shoulders, 13/14 elbows, 15/16 wrists, 17-22 fingers, 23/24 hips, + 25/26 knees, 27/28 ankles, 29/30 heels, 31/32 foot index (L/R = odd/even). + +## Parity numbers (2026-07-12, the script asserts these on every run) + +| image | landmarks mean/max (px) | blendshapes mean/max | head-pose Δ | +|---|---|---|---| +| portrait.jpg | 0.01 / 0.03 | 0.0002 / 0.0009 | 5.1° | +| portrait_rotated.jpg | 0.01 / 0.08 | 0.0004 / 0.0016 | 9.3° | +| portrait_small.jpg | 0.02 / 0.09 | 0.0011 / 0.0080 | 5.4° | +| face_stylizer_raw_face_demo.png | 0.12 / 0.59 | 0.0026 / 0.0148 | 4.0° | +| pose.jpg (world landmarks) | 0.61 cm mean / 1.02 cm max | — | — | + +Test images are MediaPipe's own Apache-2.0 test assets +(`storage.googleapis.com/mediapipe-assets`). + +## SAM 3D Body (the body quality path) + +- **License verdict — PASS with conditions** (full analysis in + `THIRD_PARTY_AI_MODELS.md`): the SAM License permits modification (ONNX + conversion) and redistribution of derivatives **only under the SAM License + with a copy attached** — rehosting converted weights on our HF repo is + compliant if the license file ships next to them. No non-commercial clause; + outputs unrestricted. Conditions to carry: license text alongside weights + + surfaced in-app, AUP-style trade-controls restrictions pass through to + users, Meta may unilaterally amend (§8). +- **Access — BLOCKED on a human click-through**: the HF repo + (`facebook/sam-3d-body-dinov3`, 2.1 GB `model.ckpt`) is gated; our token + reads repo metadata but file downloads return "not in the authorized list". + Action item (Slice E): accept the SAM License on the model page with the + project's HF account, then run the export. +- **Export feasibility — proven by a community port**: + [SAM3DBody-cpp / Fast-SAM-3D-Body](https://github.com/AmmarkoV/SAM3DBody-cpp) + already runs the model under ONNX Runtime as `backbone.onnx` (DINOv3-H+, + ~4.8 GB fp32, image → [1280,32,32] features) + `decoder.onnx` (~93 MB, + → [B,1024] token → FFN heads → **519 MHR params: global 6D rotation, + per-joint Euler XYZ, shape, hands, face**). ~150–200 ms/frame on an RTX + 3090 ⇒ seconds/frame on M-series CPU — offline video only, never live. + fp16 (~2.4 GB) is the realistic hosting tier (SkinTokens 2.3 GB precedent). +- **MHR (Momentum Human Rig) — Apache-2.0, extracted**: 127 joints + (73 core + twist/null helpers), parent-before-child ordered, + `(x,y,z,w)` pre-rotation quats, translation offsets in cm, rest pose ≈ + A-pose standing at Y-up ~185 cm. `scripts/export-bodycap-onnx.py + --mhr-assets` writes `mhr_skeleton.json` with derived + `restWorldRotation`/`restWorldPosition` per joint — the `W · clip · W⁻¹` + conjugation input Slice E's retarget needs (the `cmuRestWorld` analogue). + Momentum rest semantics: `worldRot(j) = worldRot(parent)·preRot(j)`, + `worldPos(j) = worldPos(parent) + worldRot(parent)·offset(j)`; animated + local rotation composes AFTER the pre-rotation. + +## Go/No-Go + +- **Face path: GO.** Conversion + parity + latency all retired; Slice C is a + wiring job against the contract above. +- **Pose-IK body fallback: GO.** Models converted with the same parity bar; + world landmarks are the IK solver's input (Slice E). +- **SAM 3D Body backend: GO-when-unblocked.** License compatible (with the + redistribution conditions recorded), recipe proven externally, skeleton + data in hand. Blocked solely on the gated-access acceptance; Slice E ships + Pose-IK first regardless (the epic's degrade-don't-die plan). diff --git a/scripts/export-bodycap-onnx.py b/scripts/export-bodycap-onnx.py new file mode 100644 index 000000000..ec794f680 --- /dev/null +++ b/scripts/export-bodycap-onnx.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""SAM 3D Body / MHR spike tooling for performance capture (#870 / #874). + +ONE-TIME, OFFLINE developer tool — NOT shipped with the app, and the app never +runs Python. See docs/MOCAP_SPIKE.md for the full spike record and +THIRD_PARTY_AI_MODELS.md for the SAM License verdict. + +WHAT IT DOES TODAY + --mhr-assets Extract the Momentum Human Rig (MHR, Apache-2.0) + skeleton definition from the released mhr_model.pt + (assets.zip of https://github.com/facebookresearch/MHR, + v1.0.1) into mhr_skeleton.json: + { "joints": [ { "name", "parent", # index, -1 root + "translationOffset": [x,y,z], # cm, parent-relative + "preRotation": [x,y,z,w], # parent-relative + "restWorldRotation": [x,y,z,w], + "restWorldPosition": [x,y,z] } ] } + Slice E's retarget needs restWorldRotation for the same + W . clip . W^-1 conjugation AnimationMerger::applyMotionClip + does with cmuRestWorld. Momentum rest-pose semantics: + worldRot(j) = worldRot(parent) * preRotation(j) + worldPos(j) = worldPos(parent) + worldRot(parent) * translationOffset(j) + (joint local animation rotations compose AFTER the + pre-rotation: local = preRot * eulerXYZ(joint_params)). + + --ckpt SAM 3D Body ONNX export. NOT RUNNABLE YET in this repo: + the checkpoints on HF (facebook/sam-3d-body-dinov3) are + GATED — the account whose token sits in + ~/.cache/huggingface/token must click through the SAM + License acceptance on the model page first (as of + 2026-07 our token gets 403 on file downloads). The + community port that proves the export path is + https://github.com/AmmarkoV/Fast-SAM-3D-Body / + SAM3DBody-cpp: backbone.onnx (DINOv3-H+, ~4.8 GB fp32, + [1,3,H,W] -> [1280,32,32] features), decoder.onnx + (~93 MB, 6-layer transformer -> [B,1024] pose token -> + FFN heads -> 519 MHR params: global 6D rotation + + per-joint Euler XYZ + shape + hands + face). + When access is granted, follow that recipe here and + validate against the python inference from + https://github.com/facebookresearch/sam-3d-body. + +USAGE + python scripts/export-bodycap-onnx.py --mhr-assets .mocap_work/mhr_assets/assets \ + --out .mocap_work/out/mhr_skeleton.json +""" + +import argparse +import json +import sys + + +def quat_mul(a, b): + """(x,y,z,w) Hamilton product a*b.""" + ax, ay, az, aw = a + bx, by, bz, bw = b + return ( + aw * bx + ax * bw + ay * bz - az * by, + aw * by - ax * bz + ay * bw + az * bx, + aw * bz + ax * by - ay * bx + az * bw, + aw * bw - ax * bx - ay * by - az * bz, + ) + + +def quat_rotate(q, v): + qv = (v[0], v[1], v[2], 0.0) + qc = (-q[0], -q[1], -q[2], q[3]) + r = quat_mul(quat_mul(q, qv), qc) + return (r[0], r[1], r[2]) + + +def extract_mhr_skeleton(assets_dir, out_path): + import torch # local import: only needed for this mode + + model = torch.jit.load(f"{assets_dir}/mhr_model.pt", map_location="cpu") + sk = model.character_torch.skeleton + names = list(sk.joint_names) + parents = sk.joint_parents.tolist() + offsets = sk.joint_translation_offsets.tolist() + prerot = sk.joint_prerotations.tolist() # (x,y,z,w), parent-relative + n = len(names) + assert len(parents) == len(offsets) == len(prerot) == n + + world_rot = [None] * n + world_pos = [None] * n + joints = [] + for j in range(n): + p = parents[j] + assert p < j, "joints must be parent-before-child ordered" + if p < 0: + world_rot[j] = tuple(prerot[j]) + world_pos[j] = tuple(offsets[j]) + else: + world_rot[j] = quat_mul(world_rot[p], tuple(prerot[j])) + off = quat_rotate(world_rot[p], tuple(offsets[j])) + wp = world_pos[p] + world_pos[j] = (wp[0] + off[0], wp[1] + off[1], wp[2] + off[2]) + joints.append({ + "name": names[j], + "parent": p, + "translationOffset": [round(v, 6) for v in offsets[j]], + "preRotation": [round(v, 8) for v in prerot[j]], + "restWorldRotation": [round(v, 8) for v in world_rot[j]], + "restWorldPosition": [round(v, 6) for v in world_pos[j]], + }) + + doc = { + "format": "qtmesh-mhr-skeleton-v1", + "source": "facebookresearch/MHR v1.0.1 assets (Apache-2.0)", + "units": "cm", + "quaternionOrder": "xyzw", + "jointCount": n, + "joints": joints, + } + with open(out_path, "w") as f: + json.dump(doc, f, indent=1) + print(f"{n} joints -> {out_path}") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--mhr-assets", help="dir containing mhr_model.pt") + ap.add_argument("--out", default=".mocap_work/out/mhr_skeleton.json") + ap.add_argument("--ckpt", help="SAM 3D Body model.ckpt (gated; see header)") + args = ap.parse_args() + + if args.mhr_assets: + extract_mhr_skeleton(args.mhr_assets, args.out) + if args.ckpt: + sys.exit("SAM 3D Body ONNX export: checkpoint access is gated (see the " + "header comment). Accept the SAM License on " + "https://huggingface.co/facebook/sam-3d-body-dinov3 with the " + "HF account of ~/.cache/huggingface/token, then implement the " + "Fast-SAM-3D-Body recipe here (tracked in #874).") + if not args.mhr_assets and not args.ckpt: + sys.exit("nothing to do: pass --mhr-assets and/or --ckpt") + + +if __name__ == "__main__": + main() diff --git a/scripts/export-facecap-onnx.py b/scripts/export-facecap-onnx.py new file mode 100644 index 000000000..9fe013dd7 --- /dev/null +++ b/scripts/export-facecap-onnx.py @@ -0,0 +1,789 @@ +#!/usr/bin/env python3 +"""Convert MediaPipe Face/Pose Landmarker TFLite models to ONNX, with parity proof (#870). + +ONE-TIME, OFFLINE developer tool — NOT shipped with the app, and the app never +runs Python. The app runs the resulting .onnx files in C++ via ONNX Runtime +(src/Mocap/FaceCapPredictor.cpp / PoseIK pipeline), downloading them on first +use to AppData/ai_models/mocap/{face,pose}/. + +WHAT IT PRODUCES (in --out-dir, default .mocap_work/out) + face/face_detector.onnx BlazeFace short-range: in [1,128,128,3] RGB in [-1,1] + out regressors [1,896,16], classificators [1,896,1] + face/face_landmarks.onnx Face Mesh V2: in [N,256,256,3] RGB in [0,1] + out [N,1,1,1434] (478 xyz landmarks, px in 256-space), + [N,1,1,1] face-presence logit, [N,1] (unused aux) + face/face_blendshapes.onnx MLP-Mixer blendshapes: in [1,146,2] (x*W, y*H pixel coords + of the 146-landmark subset), out [52] scores in [0,1] + pose/pose_detector.onnx BlazePose detector: in [1,224,224,3] RGB in [-1,1] + out [1,2254,12], [1,2254,1] + pose/pose_landmarks.onnx BlazePose full: in [1,256,256,3] RGB in [0,1] + out [1,195] (39 x,y,z,visibility,presence in 256-space), + [1,1] pose-presence, [1,256,256,1] seg, [1,64,64,39] heatmap, + [1,117] world landmarks (39 xyz, metres, hip-centred) + +DATA + LICENSE + All five models are Google MediaPipe models, Apache-2.0 (code AND weights) — + see THIRD_PARTY_AI_MODELS.md. Downloaded .task bundles are zip files + containing the TFLite graphs this script converts. + +CONVERSION NOTES (discovered empirically, kept for reproducibility) + * tf2onnx 1.17.0 + tensorflow 2.21 + python 3.12 work. + * pose_detector.tflite ships fp16 block-SPARSE weights behind DENSIFY ops, + which tf2onnx cannot parse. densify_tflite() runs the TFLite interpreter + with experimental_preserve_all_tensors, captures each DENSIFY output, and + rewrites the flatbuffer with dense buffers + the DENSIFY ops removed. + * face_blendshapes.tflite converts, but tf2onnx's post-conversion optimizer + produces a graph onnxruntime rejects (transpose optimizer breaks a + LayerNorm Mul). We convert that model with optimizers DISABLED. + +PRE/POST-PROCESSING CONTRACT (the spec Slice C implements in C++; also in +docs/MOCAP_SPIKE.md): + * Detector input: letterbox (keep aspect, centre, pad both sides) to the + input size, RGB, normalize to [-1,1] ((v/255)*2-1). + * Face SSD anchors: 4 layers, strides [8,16,16,16], 2 anchors on the two + 16x16 layers' merged... generated by gen_ssd_anchors() below (896 anchors, + all 1.0 aspect, fixed size). Pose: 5 layers, strides [8,16,32,32,32] + (2254 anchors). Decode: offsets / input_size scaled around anchor centres; + score = sigmoid(clip(raw, -100, 100)); weighted NMS at IoU 0.3. + * Face ROI: rotation from detector keypoint 0 (right eye) -> 1 (left eye), + target angle 0; rect = detection box made square-long then scaled 1.5x. + * Pose ROI: centre = detector keypoint 0 (mid-hip), radius = |kp1-kp0|, + box = 2*radius square, rotation target angle 90 deg, then scaled 1.25x. + * Landmark crop: rotated rect resampled to 256x256, RGB in [0,1]. + * Landmark decode: xyz / 256 -> normalized crop coords -> rotate/scale back + through the ROI transform (landmark projection); z scaled by rect w. + * Blendshape input: the 146-subset landmark (x*imgW, y*imgH) pixel coords. + * Pose world landmarks: rotate x,y by the ROI rotation only (no scale). + * Head pose: weighted Umeyama/Kabsch fit of the 468 canonical-face-model + vertices (Procrustes basis weights) against screen landmarks + (x*W, -y*H, -z*W); rotation kept, scale discarded. This intentionally + replaces MediaPipe's perspective geometry pipeline; parity vs its + facial_transformation_matrix is reported in degrees (a few deg is fine). + +USAGE (offline, in a venv with mediapipe tensorflow tf2onnx onnx onnxruntime pillow): + python scripts/export-facecap-onnx.py \ + --face-task .mocap_work/models/face_landmarker.task \ + --pose-task .mocap_work/models/pose_landmarker_full.task \ + --images .mocap_work/images --out-dir .mocap_work/out +""" + +import argparse +import json +import math +import os +import shutil +import sys +import zipfile + +import numpy as np + +# --------------------------------------------------------------------------- +# constants extracted from MediaPipe source (Apache-2.0), pinned here so the +# C++ side and this script share one source of truth (also emitted to JSON). +# --------------------------------------------------------------------------- + +# mediapipe/tasks/cc/vision/face_landmarker/face_blendshapes_graph.cc +BLENDSHAPE_LANDMARK_SUBSET = [ + 0, 1, 4, 5, 6, 7, 8, 10, 13, 14, 17, 21, 33, 37, 39, + 40, 46, 52, 53, 54, 55, 58, 61, 63, 65, 66, 67, 70, 78, 80, + 81, 82, 84, 87, 88, 91, 93, 95, 103, 105, 107, 109, 127, 132, 133, + 136, 144, 145, 146, 148, 149, 150, 152, 153, 154, 155, 157, 158, 159, 160, + 161, 162, 163, 168, 172, 173, 176, 178, 181, 185, 191, 195, 197, 234, 246, + 249, 251, 263, 267, 269, 270, 276, 282, 283, 284, 285, 288, 291, 293, 295, + 296, 297, 300, 308, 310, 311, 312, 314, 317, 318, 321, 323, 324, 332, 334, + 336, 338, 356, 361, 362, 365, 373, 374, 375, 377, 378, 379, 380, 381, 382, + 384, 385, 386, 387, 388, 389, 390, 397, 398, 400, 402, 405, 409, 415, 454, + 466, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, +] + +BLENDSHAPE_NAMES = [ + "_neutral", "browDownLeft", "browDownRight", "browInnerUp", + "browOuterUpLeft", "browOuterUpRight", "cheekPuff", "cheekSquintLeft", + "cheekSquintRight", "eyeBlinkLeft", "eyeBlinkRight", "eyeLookDownLeft", + "eyeLookDownRight", "eyeLookInLeft", "eyeLookInRight", "eyeLookOutLeft", + "eyeLookOutRight", "eyeLookUpLeft", "eyeLookUpRight", "eyeSquintLeft", + "eyeSquintRight", "eyeWideLeft", "eyeWideRight", "jawForward", "jawLeft", + "jawOpen", "jawRight", "mouthClose", "mouthDimpleLeft", "mouthDimpleRight", + "mouthFrownLeft", "mouthFrownRight", "mouthFunnel", "mouthLeft", + "mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft", + "mouthPressRight", "mouthPucker", "mouthRight", "mouthRollLower", + "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper", "mouthSmileLeft", + "mouthSmileRight", "mouthStretchLeft", "mouthStretchRight", + "mouthUpperUpLeft", "mouthUpperUpRight", "noseSneerLeft", "noseSneerRight", +] + +# MediaPipe pose landmark index -> body part (33 real landmarks; the model +# emits 39, the last 6 are auxiliary and dropped). +POSE_LANDMARK_NAMES = [ + "nose", "left_eye_inner", "left_eye", "left_eye_outer", "right_eye_inner", + "right_eye", "right_eye_outer", "left_ear", "right_ear", "mouth_left", + "mouth_right", "left_shoulder", "right_shoulder", "left_elbow", + "right_elbow", "left_wrist", "right_wrist", "left_pinky", "right_pinky", + "left_index", "right_index", "left_thumb", "right_thumb", "left_hip", + "right_hip", "left_knee", "right_knee", "left_ankle", "right_ankle", + "left_heel", "right_heel", "left_foot_index", "right_foot_index", +] + + +# --------------------------------------------------------------------------- +# TFLite -> ONNX conversion +# --------------------------------------------------------------------------- + +def densify_tflite(src, dst): + """Rewrite a TFLite flatbuffer with all DENSIFY'd sparse constants dense. + + tf2onnx cannot parse TFLite block-sparse tensors. The TFLite interpreter + can (XNNPACK densifies at prepare time), so we run it once with + experimental_preserve_all_tensors, read back each DENSIFY output, store it + as a dense buffer on the op's input tensor, rewire consumers and drop the + DENSIFY ops. Returns the number of ops removed (0 = no rewrite needed). + """ + import tensorflow as tf + from tensorflow.lite.tools import flatbuffer_utils + from tensorflow.lite.python import schema_py_generated as schema + + names = {v: k for k, v in schema.BuiltinOperator.__dict__.items() + if isinstance(v, int)} + model = flatbuffer_utils.read_model(src) + sg = model.subgraphs[0] + + def is_densify(op): + code = model.operatorCodes[op.opcodeIndex] + bc = max(code.builtinCode, getattr(code, "deprecatedBuiltinCode", 0)) + return names.get(bc) == "DENSIFY" + + if not any(is_densify(op) for op in sg.operators): + shutil.copyfile(src, dst) + return 0 + + interp = tf.lite.Interpreter( + model_path=src, + experimental_preserve_all_tensors=True, + # XNNPACK folds DENSIFY away and get_tensor() then reads garbage — + # run pure-builtin so every intermediate tensor is materialized. + experimental_op_resolver_type=tf.lite.experimental.OpResolverType + .BUILTIN_WITHOUT_DEFAULT_DELEGATES) + interp.allocate_tensors() + inp = interp.get_input_details()[0] + interp.set_tensor(inp["index"], np.zeros(inp["shape"], dtype=inp["dtype"])) + interp.invoke() + + kept = [] + removed = 0 + for op in sg.operators: + if not is_densify(op): + kept.append(op) + continue + removed += 1 + ti, to = int(op.inputs[0]), int(op.outputs[0]) + dense = interp.get_tensor(to) + tensor = sg.tensors[ti] + # type 1 == FLOAT16 in the tflite schema; keep the storage dtype. + data = np.ascontiguousarray( + dense.astype(np.float16 if tensor.type == 1 else dense.dtype) + ).tobytes() + buf = schema.BufferT() + buf.data = np.frombuffer(data, dtype=np.uint8) + model.buffers.append(buf) + tensor.buffer = len(model.buffers) - 1 + tensor.sparsity = None + for other in sg.operators: + if other is not op: + other.inputs = [ti if int(x) == to else int(x) + for x in other.inputs] + sg.operators = kept + flatbuffer_utils.write_model(model, dst) + return removed + + +def convert_tflite_to_onnx(tflite_path, onnx_path, optimize=True): + """tf2onnx conversion via the python API so the optimizer can be skipped.""" + import onnx + from tf2onnx.tfonnx import process_tf_graph + from tf2onnx import optimizer as tf2onnx_optimizer + + g = process_tf_graph(None, opset=18, tflite_path=tflite_path) + if optimize: + g = tf2onnx_optimizer.optimize_graph(g, catch_errors=True) + model_proto = g.make_model(os.path.basename(onnx_path)) + onnx.save(model_proto, onnx_path) + + +def ort_session(path): + import onnxruntime as ort + return ort.InferenceSession(path, providers=["CPUExecutionProvider"]) + + +def convert_with_fallback(tflite_path, onnx_path): + """Convert; if onnxruntime rejects the optimized graph, retry unoptimized.""" + work = tflite_path + dense = onnx_path + ".dense.tflite" + removed = densify_tflite(tflite_path, dense) + if removed: + print(f" densified {removed} sparse tensors") + work = dense + try: + convert_tflite_to_onnx(work, onnx_path, optimize=True) + ort_session(onnx_path) + mode = "optimized" + except Exception as exc: # noqa: BLE001 - any ORT load failure + print(f" optimized graph rejected ({type(exc).__name__}); " + "retrying with tf2onnx optimizers disabled") + convert_tflite_to_onnx(work, onnx_path, optimize=False) + ort_session(onnx_path) + mode = "unoptimized" + if os.path.exists(dense): + os.remove(dense) + print(f" -> {onnx_path} ({mode}, " + f"{os.path.getsize(onnx_path) / 1e6:.1f} MB)") + + +# --------------------------------------------------------------------------- +# shared pipeline math (this is the contract Slice C reimplements in C++) +# --------------------------------------------------------------------------- + +def letterbox(img, size): + """Keep-aspect centre letterbox to size x size (BORDER_ZERO, plain + bilinear — cv2, NOT an antialiased resampler; matches MediaPipe's + ImageToTensor). Returns (uint8 HxWx3 array, pad). + + pad = (pad_x, pad_y, fx, fy): normalized padding on each leading side of + the letterboxed square and the fraction of the square the image occupies. + """ + import cv2 + arr = np.asarray(img) + h, w = arr.shape[:2] + s = size / max(w, h) + ox, oy = (size - w * s) / 2.0, (size - h * s) / 2.0 + m = np.array([[s, 0, ox], [0, s, oy]], dtype=np.float64) + out = cv2.warpAffine(arr, m, (size, size), flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, borderValue=0) + return out, (ox / size, oy / size, w * s / size, h * s / size) + + +def unletterbox_points(pts, pad): + """Map normalized letterbox-square coords -> normalized original coords.""" + pad_x, pad_y, fx, fy = pad + out = pts.copy() + out[..., 0] = (pts[..., 0] - pad_x) / fx + out[..., 1] = (pts[..., 1] - pad_y) / fy + return out + + +def gen_ssd_anchors(input_size, strides, min_scale=0.1484375, max_scale=0.75, + anchor_offset=0.5, interpolated_scale_aspect_ratio=1.0): + """MediaPipe SsdAnchorsCalculator with aspect_ratios=[1.0], fixed size.""" + def scale_for(i, n): + if n == 1: + return (min_scale + max_scale) * 0.5 + return min_scale + (max_scale - min_scale) * i / (n - 1) + + anchors = [] + n = len(strides) + layer = 0 + while layer < n: + # merge consecutive identical strides into one layer + last = layer + scales = [] + while last < n and strides[last] == strides[layer]: + scales.append(scale_for(last, n)) + if interpolated_scale_aspect_ratio > 0.0: + nxt = 1.0 if last == n - 1 else scale_for(last + 1, n) + scales.append(math.sqrt(scales[-1] * nxt)) + last += 1 + stride = strides[layer] + fm = int(math.ceil(input_size / stride)) + for y in range(fm): + for x in range(fm): + for _ in scales: + anchors.append(((x + anchor_offset) / fm, + (y + anchor_offset) / fm)) + layer = last + return np.array(anchors, dtype=np.float32) + + +def decode_detections(raw_boxes, raw_scores, anchors, input_size, num_keypoints, + min_score=0.5): + """MediaPipe TensorsToDetections: fixed anchor size, no box flip.""" + scores = sigmoid(raw_scores[..., 0]) + keep = scores >= min_score + boxes = raw_boxes[keep] + scores = scores[keep] + anc = anchors[keep] + if boxes.shape[0] == 0: + return [] + cx = boxes[:, 0] / input_size + anc[:, 0] + cy = boxes[:, 1] / input_size + anc[:, 1] + w = boxes[:, 2] / input_size + h = boxes[:, 3] / input_size + kps = np.stack([boxes[:, 4 + 2 * k:6 + 2 * k] / input_size + anc + for k in range(num_keypoints)], axis=1) + dets = [] + for i in range(boxes.shape[0]): + dets.append({ + "score": float(scores[i]), + "box": (float(cx[i] - w[i] / 2), float(cy[i] - h[i] / 2), + float(w[i]), float(h[i])), + "keypoints": kps[i].astype(float), + }) + return weighted_nms(dets) + + +def weighted_nms(dets, min_suppression_threshold=0.3): + """MediaPipe weighted non-max-suppression.""" + def iou(a, b): + ax, ay, aw, ah = a + bx, by, bw, bh = b + x1, y1 = max(ax, bx), max(ay, by) + x2, y2 = min(ax + aw, bx + bw), min(ay + ah, by + bh) + inter = max(0.0, x2 - x1) * max(0.0, y2 - y1) + union = aw * ah + bw * bh - inter + return inter / union if union > 0 else 0.0 + + remaining = sorted(dets, key=lambda d: -d["score"]) + out = [] + while remaining: + best = remaining[0] + cluster = [d for d in remaining if iou(d["box"], best["box"]) + > min_suppression_threshold] + remaining = [d for d in remaining if d not in cluster] + total = sum(d["score"] for d in cluster) + box = np.zeros(4) + kps = np.zeros_like(np.asarray(best["keypoints"])) + for d in cluster: + wgt = d["score"] / total + box += wgt * np.array(d["box"]) + kps += wgt * np.asarray(d["keypoints"]) + merged = dict(best) + merged["box"] = tuple(box) + merged["keypoints"] = kps + out.append(merged) + return out + + +def rect_from_face_detection(det, img_w, img_h): + """DetectionsToRects (kp0 right eye -> kp1 left eye, target 0 deg) + + RectTransformation(scale 1.5, square_long). rotation follows MediaPipe: + theta = normalize(target - atan2(-(dy), dx)), y down, CCW-positive in the + R(theta) = [[c,-s],[s,c]] image-coordinate sense.""" + x, y, w, h = det["box"] + cx, cy = x + w / 2, y + h / 2 + kp = det["keypoints"] + dx = (kp[1][0] - kp[0][0]) * img_w + dy = (kp[1][1] - kp[0][1]) * img_h + angle = normalize_angle(-math.atan2(-dy, dx)) # target angle 0 + side = max(w * img_w, h * img_h) * 1.5 + return cx * img_w, cy * img_h, side, side, angle + + +def rect_from_pose_detection(det, img_w, img_h): + """AlignmentPointsRects (centre kp0, radius |kp1-kp0|, target 90 deg) + + RectTransformation(scale 1.25, square_long). Same rotation convention as + rect_from_face_detection.""" + kp = det["keypoints"] + cx, cy = kp[0][0] * img_w, kp[0][1] * img_h + dx = kp[1][0] * img_w - cx + dy = kp[1][1] * img_h - cy + radius = math.sqrt(dx * dx + dy * dy) + angle = normalize_angle(math.pi / 2 - math.atan2(-dy, dx)) + side = 2 * radius * 1.25 + return cx, cy, side, side, angle + + +def normalize_angle(a): + return a - 2 * math.pi * math.floor((a + math.pi) / (2 * math.pi)) + + +def crop_rotated_rect(img, rect, out_size): + """Sample the rotated rect (cx,cy,w,h px + MediaPipe rotation) into an + out_size x out_size RGB uint8 array. Output (u,v) maps to image point + centre + R(theta) . ((u/out - 0.5) * w, (v/out - 0.5) * h); plain bilinear, + BORDER_ZERO — MediaPipe's GetRotatedSubRectToRectTransformMatrix.""" + import cv2 + arr = np.asarray(img) + cx, cy, w, h, angle = rect + ca, sa = math.cos(angle), math.sin(angle) + sx, sy = w / out_size, h / out_size + # input <- output affine, then invert for warpAffine's dst<-src semantics + m = np.array([ + [ca * sx, -sa * sy, cx - (ca * sx + -sa * sy) * (out_size / 2.0)], + [sa * sx, ca * sy, cy - (sa * sx + ca * sy) * (out_size / 2.0)], + ], dtype=np.float64) + return cv2.warpAffine(arr, m, (out_size, out_size), + flags=cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP, + borderMode=cv2.BORDER_CONSTANT, borderValue=0) + + +def project_landmarks(pts_norm, rect, img_w, img_h): + """LandmarkProjection: normalized crop coords -> image pixel coords + (x' = cx + dx*w*cos - dy*h*sin, same R(theta) as the crop).""" + cx, cy, w, h, angle = rect + x = pts_norm[:, 0] - 0.5 + y = pts_norm[:, 1] - 0.5 + ca, sa = math.cos(angle), math.sin(angle) + px = cx + x * w * ca - y * h * sa + py = cy + x * w * sa + y * h * ca + out = pts_norm.copy() + out[:, 0] = px + out[:, 1] = py + if out.shape[1] > 2: + out[:, 2] = pts_norm[:, 2] * w + return out + + +def sigmoid(x): + x = np.clip(x, -100, 100) + return np.where(x >= 0, 1.0 / (1.0 + np.exp(-x)), + np.exp(x) / (1.0 + np.exp(x))) + + +def umeyama_rotation(src, dst, weights): + """Weighted Kabsch: rotation R (+ translation) with dst ~= s*R*src + t. + Returns (R 3x3, t 3, s). Scale solved then discarded by callers that only + want the pose.""" + w = weights / weights.sum() + mu_s = (src * w[:, None]).sum(axis=0) + mu_d = (dst * w[:, None]).sum(axis=0) + sc = src - mu_s + dc = dst - mu_d + cov = (dc * w[:, None]).T @ sc + U, S, Vt = np.linalg.svd(cov) + d = np.sign(np.linalg.det(U @ Vt)) + D = np.diag([1.0, 1.0, d]) + R = U @ D @ Vt + var = (w[:, None] * sc * sc).sum() + s = (S * [1, 1, d]).sum() / var if var > 0 else 1.0 + t = mu_d - s * (R @ mu_s) + return R, t, s + + +# --------------------------------------------------------------------------- +# ONNX pipelines (ours) — mirror of what Slice C does in C++ +# --------------------------------------------------------------------------- + +class OnnxFacePipeline: + def __init__(self, out_dir, canonical_obj, procrustes_pbtxt): + self.det = ort_session(os.path.join(out_dir, "face", "face_detector.onnx")) + self.lmk = ort_session(os.path.join(out_dir, "face", "face_landmarks.onnx")) + self.bs = ort_session(os.path.join(out_dir, "face", "face_blendshapes.onnx")) + self.det_size = self.det.get_inputs()[0].shape[1] + self.lmk_size = self.lmk.get_inputs()[0].shape[1] + self.anchors = gen_ssd_anchors(self.det_size, [8, 16, 16, 16]) + assert self.anchors.shape[0] == self.det.get_outputs()[0].shape[1], \ + f"anchor count {self.anchors.shape[0]} != model {self.det.get_outputs()[0].shape[1]}" + self.canonical = load_canonical_obj(canonical_obj) + ids, wgt = load_procrustes_basis(procrustes_pbtxt) + self.basis_ids = ids + self.basis_w = wgt + + def __call__(self, img): + from PIL import Image + img = img.convert("RGB") + W, H = img.size + # --- stage 1: detector + lb, pad = letterbox(img, self.det_size) + x = lb.astype(np.float32)[None] / 127.5 - 1.0 + raw_box, raw_score = self.det.run(None, {self.det.get_inputs()[0].name: x}) + dets = decode_detections(raw_box[0], raw_score[0], self.anchors, + self.det_size, num_keypoints=6) + if not dets: + return None + det = dets[0] + det["box"] = tuple(unletterbox_points( + np.array([[det["box"][0], det["box"][1]], + [det["box"][0] + det["box"][2], det["box"][1] + det["box"][3]]]), + pad).reshape(-1)) + det["box"] = (det["box"][0], det["box"][1], + det["box"][2] - det["box"][0], det["box"][3] - det["box"][1]) + det["keypoints"] = unletterbox_points(np.asarray(det["keypoints"]), pad) + rect = rect_from_face_detection(det, W, H) + # --- stage 2: landmarks + crop = crop_rotated_rect(img, rect, self.lmk_size) + x = crop.astype(np.float32)[None] / 255.0 + outs = self.lmk.run(None, {self.lmk.get_inputs()[0].name: x}) + lmk_raw = outs[0].reshape(-1, 3) / self.lmk_size + presence = float(sigmoid(outs[1].reshape(-1)[0])) + pts = project_landmarks(lmk_raw, rect, W, H) # px + # --- stage 3: blendshapes (146 subset, pixel coords, 2D) + sub = pts[BLENDSHAPE_LANDMARK_SUBSET][:, :2].astype(np.float32) + bs = self.bs.run(None, {self.bs.get_inputs()[0].name: sub[None]})[0] + bs = bs.reshape(-1) + # --- head pose: weighted Kabsch canonical(cm) -> camera-ish frame + pose_pts = np.stack([pts[:468, 0], -pts[:468, 1], -pts[:468, 2]], axis=1) + weights = np.zeros(468, dtype=np.float64) + weights[self.basis_ids] = self.basis_w + R, t, s = umeyama_rotation(self.canonical, pose_pts, weights) + return { + "landmarks": pts, # 478 x 3, px (z in px, w-scaled) + "presence": presence, + "blendshapes": bs, # 52 + "rotation": R, # canonical -> camera + } + + +class OnnxPosePipeline: + def __init__(self, out_dir): + self.det = ort_session(os.path.join(out_dir, "pose", "pose_detector.onnx")) + self.lmk = ort_session(os.path.join(out_dir, "pose", "pose_landmarks.onnx")) + self.det_size = self.det.get_inputs()[0].shape[1] + self.lmk_size = self.lmk.get_inputs()[0].shape[1] + self.anchors = gen_ssd_anchors(self.det_size, [8, 16, 32, 32, 32]) + assert self.anchors.shape[0] == self.det.get_outputs()[0].shape[1] + + def __call__(self, img): + img = img.convert("RGB") + W, H = img.size + lb, pad = letterbox(img, self.det_size) + x = lb.astype(np.float32)[None] / 127.5 - 1.0 + raw_box, raw_score = self.det.run(None, {self.det.get_inputs()[0].name: x}) + dets = decode_detections(raw_box[0], raw_score[0], self.anchors, + self.det_size, num_keypoints=4) + if not dets: + return None + det = dets[0] + det["keypoints"] = unletterbox_points(np.asarray(det["keypoints"]), pad) + rect = rect_from_pose_detection(det, W, H) + crop = crop_rotated_rect(img, rect, self.lmk_size) + x = crop.astype(np.float32)[None] / 255.0 + outs = self.lmk.run(None, {self.lmk.get_inputs()[0].name: x}) + raw = outs[0].reshape(-1, 5) # 39 x (x,y,z,vis,presence) + # unlike the face model (logit), the pose flag is already a probability + pose_flag = float(outs[1].reshape(-1)[0]) + world = outs[4].reshape(-1, 3) + pts_norm = raw[:, :3] / self.lmk_size + pts = project_landmarks(pts_norm, rect, W, H) + vis = sigmoid(raw[:, 3]) + # world landmarks: rotate by the rect rotation only (no scale) + cx, cy, w, h, angle = rect + ca, sa = math.cos(angle), math.sin(angle) + wx = world[:, 0] * ca - world[:, 1] * sa + wy = world[:, 0] * sa + world[:, 1] * ca + world_rot = np.stack([wx, wy, world[:, 2]], axis=1) + return { + "landmarks": pts[:33], + "visibility": vis[:33], + "world": world_rot[:33], + "presence": pose_flag, + } + + +def load_canonical_obj(path): + """468 canonical face model vertices (cm), mediapipe face_geometry data.""" + verts = [] + with open(path) as f: + for line in f: + if line.startswith("v "): + verts.append([float(v) for v in line.split()[1:4]]) + v = np.array(verts, dtype=np.float64) + assert v.shape == (468, 3), v.shape + return v + + +def load_procrustes_basis(path): + ids, weights = [], [] + with open(path) as f: + for line in f: + line = line.strip() + if line.startswith("procrustes_landmark_basis"): + parts = line.replace("{", " ").replace("}", " ").split() + ids.append(int(parts[parts.index("landmark_id:") + 1])) + weights.append(float(parts[parts.index("weight:") + 1])) + return np.array(ids), np.array(weights, dtype=np.float64) + + +# --------------------------------------------------------------------------- +# parity vs the python mediapipe reference +# --------------------------------------------------------------------------- + +def run_face_parity(face_task, out_dir, images, canonical_obj, procrustes_pbtxt): + import mediapipe as mp + from mediapipe.tasks import python as mp_python + from mediapipe.tasks.python import vision as mp_vision + from PIL import Image + + ours = OnnxFacePipeline(out_dir, canonical_obj, procrustes_pbtxt) + options = mp_vision.FaceLandmarkerOptions( + base_options=mp_python.BaseOptions(model_asset_path=face_task), + output_face_blendshapes=True, + output_facial_transformation_matrixes=True, + num_faces=1) + ref = mp_vision.FaceLandmarker.create_from_options(options) + + print("\n== FACE PARITY (ONNX pipeline vs python mediapipe) ==") + worst_bs, worst_px, worst_deg = 0.0, 0.0, 0.0 + n_ok = 0 + for path in images: + img = Image.open(path) + mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, + data=np.asarray(img.convert("RGB"))) + res = ref.detect(mp_img) + mine = ours(img) + if not res.face_landmarks: + print(f" {os.path.basename(path)}: no face (reference); " + f"ours={'none' if mine is None else 'found'}") + continue + if mine is None: + print(f" {os.path.basename(path)}: REFERENCE found a face, we did NOT") + continue + W, H = img.size + ref_pts = np.array([[l.x * W, l.y * H] for l in res.face_landmarks[0]]) + d_px = np.linalg.norm(ref_pts - mine["landmarks"][:, :2], axis=1) + ref_bs = np.array([b.score for b in res.face_blendshapes[0]]) + d_bs = np.abs(ref_bs - mine["blendshapes"]) + deg = np.nan + if res.facial_transformation_matrixes: + Rref = np.array(res.facial_transformation_matrixes[0])[:3, :3] + # our R maps canonical -> camera, so compare directly + dR = Rref.T @ mine["rotation"] + deg = math.degrees(math.acos(np.clip((np.trace(dR) - 1) / 2, -1, 1))) + print(f" {os.path.basename(path)}: landmarks mean {d_px.mean():.2f}px " + f"max {d_px.max():.2f}px | blendshapes mean {d_bs.mean():.4f} " + f"max {d_bs.max():.4f} | head-pose delta {deg:.2f} deg " + f"| presence {mine['presence']:.3f}") + worst_bs = max(worst_bs, d_bs.max()) + worst_px = max(worst_px, d_px.max()) + if not math.isnan(deg): + worst_deg = max(worst_deg, deg) + n_ok += 1 + ref.close() + assert n_ok > 0, "no face image produced a reference result" + print(f" WORST: blendshapes {worst_bs:.4f} (target <= 0.02), " + f"landmarks {worst_px:.2f}px (target <= 2px on the tracked face), " + f"head-pose {worst_deg:.2f} deg (informational)") + return {"worst_blendshape_abs": worst_bs, "worst_landmark_px": worst_px, + "worst_headpose_deg": worst_deg, "images": n_ok} + + +def run_pose_parity(pose_task, out_dir, images): + import mediapipe as mp + from mediapipe.tasks import python as mp_python + from mediapipe.tasks.python import vision as mp_vision + from PIL import Image + + ours = OnnxPosePipeline(out_dir) + options = mp_vision.PoseLandmarkerOptions( + base_options=mp_python.BaseOptions(model_asset_path=pose_task), + num_poses=1) + ref = mp_vision.PoseLandmarker.create_from_options(options) + + print("\n== POSE PARITY (ONNX pipeline vs python mediapipe) ==") + worst_world = 0.0 + n_ok = 0 + for path in images: + img = Image.open(path) + mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, + data=np.asarray(img.convert("RGB"))) + res = ref.detect(mp_img) + mine = ours(img) + if not res.pose_world_landmarks: + print(f" {os.path.basename(path)}: no pose (reference)") + continue + if mine is None: + print(f" {os.path.basename(path)}: REFERENCE found a pose, we did NOT") + continue + ref_world = np.array([[l.x, l.y, l.z] + for l in res.pose_world_landmarks[0]]) + d = np.linalg.norm(ref_world - mine["world"], axis=1) + print(f" {os.path.basename(path)}: world landmarks mean " + f"{d.mean() * 100:.2f}cm max {d.max() * 100:.2f}cm " + f"| presence {mine['presence']:.3f}") + worst_world = max(worst_world, d.max()) + n_ok += 1 + ref.close() + assert n_ok > 0, "no pose image produced a reference result" + print(f" WORST: world {worst_world * 100:.2f}cm (target <= 2cm)") + return {"worst_world_m": worst_world, "images": n_ok} + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- + +def extract_task(task_path, dest): + os.makedirs(dest, exist_ok=True) + with zipfile.ZipFile(task_path) as z: + z.extractall(dest) + return {os.path.splitext(n)[0]: os.path.join(dest, n) + for n in os.listdir(dest) if n.endswith(".tflite")} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--face-task", help="face_landmarker.task path") + ap.add_argument("--pose-task", help="pose_landmarker_full.task path") + ap.add_argument("--images", required=True, + help="dir with test images (face parity uses images whose " + "name doesn't start with 'pose', pose parity the rest)") + ap.add_argument("--out-dir", default=".mocap_work/out") + ap.add_argument("--canonical-obj", default=".mocap_work/canonical_face_model.obj") + ap.add_argument("--procrustes", default=".mocap_work/geometry_pipeline_metadata_landmarks.pbtxt") + ap.add_argument("--verify-only", action="store_true", + help="skip conversion, only run parity on existing out-dir") + args = ap.parse_args() + + imgs = sorted(os.path.join(args.images, f) for f in os.listdir(args.images) + if f.lower().endswith((".jpg", ".jpeg", ".png"))) + face_imgs = [p for p in imgs if not os.path.basename(p).startswith("pose")] + pose_imgs = [p for p in imgs if os.path.basename(p).startswith("pose")] + + report = {} + if args.face_task: + face_out = os.path.join(args.out_dir, "face") + os.makedirs(face_out, exist_ok=True) + if not args.verify_only: + print("== converting face models ==") + tflites = extract_task(args.face_task, os.path.join(args.out_dir, "_face_tflite")) + convert_with_fallback(tflites["face_detector"], + os.path.join(face_out, "face_detector.onnx")) + convert_with_fallback(tflites["face_landmarks_detector"], + os.path.join(face_out, "face_landmarks.onnx")) + convert_with_fallback(tflites["face_blendshapes"], + os.path.join(face_out, "face_blendshapes.onnx")) + report["face"] = run_face_parity(args.face_task, args.out_dir, face_imgs, + args.canonical_obj, args.procrustes) + assert report["face"]["worst_blendshape_abs"] <= 0.02, "blendshape parity FAILED" + + if args.pose_task: + pose_out = os.path.join(args.out_dir, "pose") + os.makedirs(pose_out, exist_ok=True) + if not args.verify_only: + print("== converting pose models ==") + tflites = extract_task(args.pose_task, os.path.join(args.out_dir, "_pose_tflite")) + convert_with_fallback(tflites["pose_detector"], + os.path.join(pose_out, "pose_detector.onnx")) + convert_with_fallback(tflites["pose_landmarks_detector"], + os.path.join(pose_out, "pose_landmarks.onnx")) + report["pose"] = run_pose_parity(args.pose_task, args.out_dir, pose_imgs) + assert report["pose"]["worst_world_m"] <= 0.02, "pose world parity FAILED" + + # contract dump for the C++ side / spike doc + contract = { + "blendshape_names": BLENDSHAPE_NAMES, + "blendshape_landmark_subset": BLENDSHAPE_LANDMARK_SUBSET, + "pose_landmark_names": POSE_LANDMARK_NAMES, + "face_detector": {"input": 128, "range": [-1, 1], + "anchors": {"strides": [8, 16, 16, 16], "count": 896}, + "keypoints": ["right_eye", "left_eye", "nose", "mouth", + "right_ear", "left_ear"], + "roi": {"rotation_kp": [0, 1], "target_angle_deg": 0, + "scale": 1.5, "square_long": True}}, + "face_landmarks": {"input": 256, "range": [0, 1], + "outputs": "1434=478*xyz in 256-space; presence logit"}, + "face_blendshapes": {"input": "[1,146,2] (x*W, y*H) px", "output": 52}, + "pose_detector": {"input": 224, "range": [-1, 1], + "anchors": {"strides": [8, 16, 32, 32, 32], "count": 2254}, + "roi": {"center_kp": 0, "radius_kp": 1, + "target_angle_deg": 90, "scale": 1.25}}, + "pose_landmarks": {"input": 256, "range": [0, 1], + "outputs": "195=39*(x,y,z,vis,presence); world 117=39*xyz metres"}, + "parity": report, + } + os.makedirs(args.out_dir, exist_ok=True) + with open(os.path.join(args.out_dir, "mocap_contract.json"), "w") as f: + json.dump(contract, f, indent=2) + print(f"\ncontract + parity report -> {os.path.join(args.out_dir, 'mocap_contract.json')}") + print("ALL PARITY CHECKS PASSED") + + +if __name__ == "__main__": + sys.exit(main()) From 858b97385824700d7e1f9b6764b142cf2653ee85 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 12 Jul 2026 21:29:17 -0300 Subject: [PATCH 02/31] feat(#871): ENABLE_MOCAP build infra + VideoFrameSource (Qt Multimedia) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice B of epic #869 (performance capture). - New ENABLE_MOCAP CMake option (default OFF; requires ENABLE_ONNX, errors otherwise) pulling in Qt6::Multimedia for the app + UnitTests. - src/Mocap/VideoFrameSource.{h,cpp}: one frame-source abstraction delivering timestamped RGB888 MocapFrames from (a) a video file (QMediaPlayer + QVideoSink, targetFps decimation, playback-driven — faster-than-realtime decode is a documented follow-up), (b) a live camera (QCamera + QMediaCaptureSession, device enumeration for the GUI picker / MCP, permission-denied mapped to a human-readable error, latest-wins FrameMailbox for a slower inference consumer), (c) an image sequence (the synchronous headless test double / CLI --frames-dir path). - FrameDecimator + FrameMailbox are pure data and headless-tested; 13 new tests pass (emission order/timestamps, 60->30 decimation, latest-wins drop semantics incl. cross-thread, open() failure paths, device enumeration). - test_main.cpp gains QTMESH_TESTS_SKIP_OGRE_PREFLIGHT so pure-data suites can run on machines with no GL/WindowServer at all; CI never sets it. - NSCameraUsageDescription added to Info.plist.in. - CI: Linux unit-test lane installs the qtmultimedia module and configures with -DENABLE_MOCAP=ON. Co-Authored-By: Claude Fable 5 --- .github/workflows/deploy.yml | 5 +- CMakeLists.txt | 23 +++ src/CMakeLists.txt | 11 ++ src/Info.plist.in | 3 + src/Mocap/VideoFrameSource.cpp | 282 ++++++++++++++++++++++++++++ src/Mocap/VideoFrameSource.h | 216 +++++++++++++++++++++ src/Mocap/VideoFrameSource_test.cpp | 200 ++++++++++++++++++++ src/test_main.cpp | 13 +- 8 files changed, 750 insertions(+), 3 deletions(-) create mode 100644 src/Mocap/VideoFrameSource.cpp create mode 100644 src/Mocap/VideoFrameSource.h create mode 100644 src/Mocap/VideoFrameSource_test.cpp diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5d721cd73..4c9717a6c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -958,6 +958,8 @@ jobs: host: 'linux' target: 'desktop' arch: 'linux_gcc_64' + # qtmultimedia: performance capture (ENABLE_MOCAP, epic #869) + modules: 'qtmultimedia' - name: change folder permissions run: | @@ -1074,7 +1076,8 @@ jobs: -DBUILD_QT_MESH_EDITOR=OFF \ -DENABLE_SENTRY=OFF \ -DENABLE_PS1_RIP=ON \ - -DENABLE_ONNX=ON + -DENABLE_ONNX=ON \ + -DENABLE_MOCAP=ON - name: Run build-wrapper env: diff --git a/CMakeLists.txt b/CMakeLists.txt index f04b48990..d68fb1c48 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -324,6 +324,29 @@ else() message(STATUS "Alembic vertex-animation import disabled (VAT_POSE playback still available)") endif() ############################################################## +# Performance capture — video/webcam mocap (epic #869) +############################################################## +# Slice B (#871): brings in Qt Multimedia (camera + video decode) and the +# src/Mocap/ frame-source layer. Default OFF: Qt Multimedia is a new runtime +# dependency (FFmpeg backend) that packaging has to carry per platform. The +# ONNX predictors (Slices C/E) additionally require ENABLE_ONNX; requiring it +# here keeps a single "mocap build" configuration instead of a half-working +# one. Non-mocap builds print "rebuild with -DENABLE_MOCAP" on every surface. +option(ENABLE_MOCAP "Enable performance capture (video/webcam mocap)" OFF) + +if(ENABLE_MOCAP) + if(NOT ENABLE_ONNX) + message(FATAL_ERROR "ENABLE_MOCAP requires ENABLE_ONNX (the face/pose " + "predictors run on ONNX Runtime). Configure with " + "-DENABLE_ONNX=ON -DENABLE_MOCAP=ON.") + endif() + find_package(Qt6 REQUIRED COMPONENTS Multimedia) + add_definitions(-DENABLE_MOCAP) + message(STATUS "Performance capture enabled (Qt Multimedia ${Qt6Multimedia_VERSION})") +else() + message(STATUS "Performance capture disabled (rebuild with -DENABLE_MOCAP=ON -DENABLE_ONNX=ON)") +endif() +############################################################## # PS1 runtime geometry extraction (experimental) ############################################################## option(ENABLE_PS1_RIP "Enable experimental PS1 runtime geometry extraction" OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a06f61f83..81a3aa044 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -161,6 +161,7 @@ NodeAnimationManager.cpp PoseLibrary.cpp VertexAnimationManager.cpp AlembicImporter.cpp +Mocap/VideoFrameSource.cpp ApplyAtlas.cpp EmbeddedTextureCache.cpp NormalMapGenerator.cpp @@ -753,6 +754,11 @@ if(ENABLE_PS1_RIP AND TARGET Qt6::Gamepad) target_link_libraries(${CMAKE_PROJECT_NAME} Qt6::Gamepad) endif() +# Performance capture (epic #869): Qt Multimedia for the frame sources +if(ENABLE_MOCAP) + target_link_libraries(${CMAKE_PROJECT_NAME} Qt6::Multimedia) +endif() + if(ENABLE_AUTO_UPDATER AND TARGET qtmesh-relauncher) add_dependencies(${CMAKE_PROJECT_NAME} qtmesh-relauncher) if(APPLE) @@ -898,6 +904,11 @@ if(BUILD_TESTS) ADD_DEPENDENCIES(UnitTests ui) + # Performance capture (epic #869): Qt Multimedia for the frame sources + if(ENABLE_MOCAP) + target_link_libraries(UnitTests Qt6::Multimedia) + endif() + if(ENABLE_PS1_RIP) add_dependencies(UnitTests qtmesh_ps1core_stub) if(TARGET qtmesh_ps1core_libretro) diff --git a/src/Info.plist.in b/src/Info.plist.in index 886eab2ca..ba8de8099 100644 --- a/src/Info.plist.in +++ b/src/Info.plist.in @@ -43,6 +43,9 @@ NSSupportsAutomaticGraphicsSwitching + NSCameraUsageDescription + QtMeshEditor uses the camera for live performance capture. + CFBundleDocumentTypes diff --git a/src/Mocap/VideoFrameSource.cpp b/src/Mocap/VideoFrameSource.cpp new file mode 100644 index 000000000..3769ad417 --- /dev/null +++ b/src/Mocap/VideoFrameSource.cpp @@ -0,0 +1,282 @@ +#ifdef ENABLE_MOCAP + +#include "VideoFrameSource.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +// Queued frameReady connections (capture thread -> worker) need the metatype. +struct MocapMetaTypeRegistrar { + MocapMetaTypeRegistrar() { qRegisterMetaType("MocapFrame"); } +}; +const MocapMetaTypeRegistrar registrar; +} // namespace + +QImage mocapFrameToRgb888(const QImage& image) +{ + if (image.format() == QImage::Format_RGB888) + return image; + return image.convertToFormat(QImage::Format_RGB888); +} + +// --------------------------------------------------------------------------- +// ImageSequenceFrameSource +// --------------------------------------------------------------------------- + +ImageSequenceFrameSource::ImageSequenceFrameSource(const QStringList& imagePaths, + double fps, double targetFps, + QObject* parent) + : VideoFrameSource(parent), + m_paths(imagePaths), + m_fps(fps > 0.0 ? fps : 30.0), + m_decimator(targetFps) +{ +} + +bool ImageSequenceFrameSource::open(QString* error) +{ + if (m_paths.isEmpty()) { + if (error) *error = tr("image sequence is empty"); + return false; + } + for (const QString& p : m_paths) { + if (!QFileInfo::exists(p)) { + if (error) *error = tr("image not found: %1").arg(p); + return false; + } + } + return true; +} + +void ImageSequenceFrameSource::start() +{ + m_stopped = false; + m_decimator.reset(); + for (qint64 i = 0; i < m_paths.size() && !m_stopped; ++i) { + const double t = static_cast(i) / m_fps; + if (!m_decimator.shouldEmit(t)) + continue; + QImage img(m_paths.at(i)); + if (img.isNull()) { + emit errorOccurred(tr("failed to load image: %1").arg(m_paths.at(i))); + return; + } + MocapFrame frame; + frame.image = mocapFrameToRgb888(img); + frame.timeSec = t; + frame.frameIndex = i; + emit frameReady(frame); + } + if (!m_stopped) + emit finished(); +} + +void ImageSequenceFrameSource::stop() +{ + m_stopped = true; +} + +// --------------------------------------------------------------------------- +// FileFrameSource +// --------------------------------------------------------------------------- + +FileFrameSource::FileFrameSource(const QString& filePath, double targetFps, + QObject* parent) + : VideoFrameSource(parent), m_path(filePath), m_decimator(targetFps) +{ +} + +FileFrameSource::~FileFrameSource() +{ + stop(); +} + +bool FileFrameSource::open(QString* error) +{ + if (!QFileInfo::exists(m_path)) { + if (error) *error = tr("video file not found: %1").arg(m_path); + return false; + } + m_player = std::make_unique(); + m_sink = std::make_unique(); + m_player->setVideoSink(m_sink.get()); + + connect(m_sink.get(), &QVideoSink::videoFrameChanged, this, + &FileFrameSource::handleVideoFrame); + connect(m_player.get(), &QMediaPlayer::mediaStatusChanged, this, + [this](QMediaPlayer::MediaStatus status) { + if (status == QMediaPlayer::EndOfMedia && !m_finishedEmitted) { + m_finishedEmitted = true; + emit finished(); + } + if (status == QMediaPlayer::LoadedMedia) { + const auto rate = m_player->metaData() + .value(QMediaMetaData::VideoFrameRate); + if (rate.isValid()) + m_nativeFps = rate.toDouble(); + } + }); + connect(m_player.get(), &QMediaPlayer::errorOccurred, this, + [this](QMediaPlayer::Error, const QString& message) { + emit errorOccurred(tr("video decode error: %1").arg(message)); + }); + + m_player->setSource(QUrl::fromLocalFile(m_path)); + return true; +} + +void FileFrameSource::start() +{ + if (!m_player) { + emit errorOccurred(tr("start() before open()")); + return; + } + m_decimator.reset(); + m_frameIndex = 0; + m_finishedEmitted = false; + m_player->play(); +} + +void FileFrameSource::stop() +{ + if (m_player) + m_player->stop(); +} + +void FileFrameSource::handleVideoFrame() +{ + const QVideoFrame vf = m_sink->videoFrame(); + if (!vf.isValid()) + return; + const qint64 index = m_frameIndex++; + // Prefer the frame's own timestamp; fall back to the player clock. + double t = vf.startTime() >= 0 + ? vf.startTime() / 1e6 + : (m_player ? m_player->position() / 1e3 : 0.0); + if (!m_decimator.shouldEmit(t)) + return; + MocapFrame frame; + frame.image = mocapFrameToRgb888(vf.toImage()); + frame.timeSec = t; + frame.frameIndex = index; + if (!frame.image.isNull()) + emit frameReady(frame); +} + +// --------------------------------------------------------------------------- +// CameraFrameSource +// --------------------------------------------------------------------------- + +QList CameraFrameSource::availableDevices() +{ + QList out; + const auto devices = QMediaDevices::videoInputs(); + for (const QCameraDevice& d : devices) + out.append({QString::fromUtf8(d.id()), d.description()}); + return out; +} + +CameraFrameSource::CameraFrameSource(const QString& deviceId, QObject* parent) + : VideoFrameSource(parent), m_deviceId(deviceId) +{ +} + +CameraFrameSource::~CameraFrameSource() +{ + stop(); +} + +bool CameraFrameSource::open(QString* error) +{ + QCameraDevice device; + const auto devices = QMediaDevices::videoInputs(); + if (m_deviceId.isEmpty()) { + device = QMediaDevices::defaultVideoInput(); + } else { + for (const QCameraDevice& d : devices) { + if (QString::fromUtf8(d.id()) == m_deviceId) { + device = d; + break; + } + } + } + if (device.isNull()) { + if (error) + *error = devices.isEmpty() + ? tr("no camera available") + : tr("camera not found: %1").arg(m_deviceId); + return false; + } + + m_camera = std::make_unique(device); + m_session = std::make_unique(); + m_sink = std::make_unique(); + m_session->setCamera(m_camera.get()); + m_session->setVideoSink(m_sink.get()); + m_clock = std::make_unique(); + + const auto formats = device.videoFormats(); + if (!formats.isEmpty()) + m_nativeFps = formats.first().maxFrameRate(); + + connect(m_sink.get(), &QVideoSink::videoFrameChanged, this, + &CameraFrameSource::handleVideoFrame); + connect(m_camera.get(), &QCamera::errorOccurred, this, + [this](QCamera::Error err, const QString& message) { + if (err == QCamera::CameraError && message.contains( + QStringLiteral("permission"), Qt::CaseInsensitive)) { + emit errorOccurred(tr("camera permission denied — allow " + "camera access for QtMeshEditor in " + "the system settings")); + } else { + emit errorOccurred(tr("camera error: %1").arg(message)); + } + }); + return true; +} + +void CameraFrameSource::start() +{ + if (!m_camera) { + emit errorOccurred(tr("start() before open()")); + return; + } + m_frameIndex = 0; + m_clock->start(); + m_camera->start(); +} + +void CameraFrameSource::stop() +{ + if (m_camera) + m_camera->stop(); +} + +void CameraFrameSource::handleVideoFrame() +{ + const QVideoFrame vf = m_sink->videoFrame(); + if (!vf.isValid()) + return; + MocapFrame frame; + frame.image = mocapFrameToRgb888(vf.toImage()); + if (frame.image.isNull()) + return; + frame.timeSec = m_clock->isValid() ? m_clock->elapsed() / 1e3 : 0.0; + frame.frameIndex = m_frameIndex++; + // Latest-wins for the inference consumer; the signal serves lightweight + // observers (preview HUD) that keep up with the camera. + m_mailbox.put(frame); + emit frameReady(frame); +} + +#endif // ENABLE_MOCAP diff --git a/src/Mocap/VideoFrameSource.h b/src/Mocap/VideoFrameSource.h new file mode 100644 index 000000000..eab852d06 --- /dev/null +++ b/src/Mocap/VideoFrameSource.h @@ -0,0 +1,216 @@ +#ifndef VIDEOFRAMESOURCE_H +#define VIDEOFRAMESOURCE_H + +// Performance capture frame sources (epic #869, Slice B #871). +// +// One abstraction delivers timestamped RGB888 frames from (a) a video file, +// (b) a live camera, (c) an image sequence (the headless test double / the +// CLI --frames-dir debug path) — so the predictors and controllers never +// touch Qt Multimedia directly. Everything in this header is compiled only +// under ENABLE_MOCAP (src/CMakeLists.txt adds the .cpp behind the flag). + +#ifdef ENABLE_MOCAP + +#include +#include +#include +#include + +#include +#include + +class QCamera; +class QElapsedTimer; +class QMediaCaptureSession; +class QMediaPlayer; +class QVideoSink; + +// One decoded frame. image is guaranteed Format_RGB888. +struct MocapFrame { + QImage image; + double timeSec = 0.0; // media timestamp (file/sequence) or wall-clock since start (camera) + qint64 frameIndex = 0; // source frame counter (pre-decimation) +}; +Q_DECLARE_METATYPE(MocapFrame) + +// Drops frames so a source delivering at native fps emits ~targetFps. +// Pure data — unit-tested headless. targetFps <= 0 disables decimation. +class FrameDecimator { +public: + explicit FrameDecimator(double targetFps = 0.0) : m_targetFps(targetFps) {} + + // Called with each frame's timestamp (monotonically increasing); returns + // true when the frame should be emitted. The first frame always passes. + // Tolerates timestamps landing half a source-frame early so 60 -> 30 fps + // emits exactly every other frame instead of every third. + bool shouldEmit(double timeSec) + { + if (m_targetFps <= 0.0) + return true; + const double interval = 1.0 / m_targetFps; + if (m_hasEmitted && timeSec - m_lastEmitted < interval * 0.75) + return false; + m_hasEmitted = true; + m_lastEmitted = timeSec; + return true; + } + + void reset() { m_hasEmitted = false; m_lastEmitted = 0.0; } + +private: + double m_targetFps; + double m_lastEmitted = 0.0; + bool m_hasEmitted = false; +}; + +// Latest-wins single-slot mailbox between the capture thread and a (slower) +// inference consumer: a new frame REPLACES any undelivered pending frame, so +// live inference never falls behind the camera. Thread-safe; unit-tested. +class FrameMailbox { +public: + void put(const MocapFrame& frame) + { + std::lock_guard lock(m_mutex); + if (m_hasPending) + ++m_dropped; + m_pending = frame; + m_hasPending = true; + } + + // Takes the newest pending frame, if any. Returns false when empty. + bool take(MocapFrame* out) + { + std::lock_guard lock(m_mutex); + if (!m_hasPending) + return false; + *out = m_pending; + m_pending = MocapFrame{}; // release the QImage + m_hasPending = false; + return true; + } + + // Frames overwritten before a consumer took them (diagnostics/HUD). + qint64 droppedCount() const + { + std::lock_guard lock(m_mutex); + return m_dropped; + } + +private: + mutable std::mutex m_mutex; + MocapFrame m_pending; + bool m_hasPending = false; + qint64 m_dropped = 0; +}; + +// Guarantees Format_RGB888 (the input contract of every mocap predictor). +QImage mocapFrameToRgb888(const QImage& image); + +class VideoFrameSource : public QObject { + Q_OBJECT +public: + explicit VideoFrameSource(QObject* parent = nullptr) : QObject(parent) {} + ~VideoFrameSource() override = default; + + // Prepare the source. Returns false and fills *error on failure. + virtual bool open(QString* error) = 0; + virtual void start() = 0; + virtual void stop() = 0; + virtual bool isLive() const = 0; + virtual double nativeFps() const = 0; // 0 if unknown + +signals: + void frameReady(const MocapFrame& frame); + void finished(); // file/sequence sources: end of media + void errorOccurred(const QString& message); +}; + +// (c) Image sequence — synchronous test double. Emits every image on start() +// with timestamps i/fps, honouring targetFps decimation, then finished(). +class ImageSequenceFrameSource : public VideoFrameSource { + Q_OBJECT +public: + ImageSequenceFrameSource(const QStringList& imagePaths, double fps, + double targetFps = 0.0, QObject* parent = nullptr); + + bool open(QString* error) override; + void start() override; + void stop() override; + bool isLive() const override { return false; } + double nativeFps() const override { return m_fps; } + +private: + QStringList m_paths; + double m_fps; + FrameDecimator m_decimator; + bool m_stopped = false; +}; + +// (a) Video file — QMediaPlayer + QVideoSink. Playback-driven (real-time; +// faster-than-realtime decode is a known follow-up, QVideoSink is fed by the +// player clock). targetFps decimates delivery for offline capture. +class FileFrameSource : public VideoFrameSource { + Q_OBJECT +public: + explicit FileFrameSource(const QString& filePath, double targetFps = 0.0, + QObject* parent = nullptr); + ~FileFrameSource() override; + + bool open(QString* error) override; + void start() override; + void stop() override; + bool isLive() const override { return false; } + double nativeFps() const override { return m_nativeFps; } + +private: + void handleVideoFrame(); + + QString m_path; + FrameDecimator m_decimator; + std::unique_ptr m_player; + std::unique_ptr m_sink; + double m_nativeFps = 0.0; + qint64 m_frameIndex = 0; + bool m_finishedEmitted = false; +}; + +// (b) Live camera — QCamera + QMediaCaptureSession + QVideoSink. Frames are +// emitted as frameReady AND written into mailbox() (latest-wins) for a +// slower inference consumer. deviceId empty = default camera. +class CameraFrameSource : public VideoFrameSource { + Q_OBJECT +public: + struct DeviceInfo { + QString id; + QString description; + }; + // Enumerates video inputs (feeds the GUI picker + MCP list_capture_devices). + static QList availableDevices(); + + explicit CameraFrameSource(const QString& deviceId = {}, + QObject* parent = nullptr); + ~CameraFrameSource() override; + + bool open(QString* error) override; + void start() override; + void stop() override; + bool isLive() const override { return true; } + double nativeFps() const override { return m_nativeFps; } + + FrameMailbox& mailbox() { return m_mailbox; } + +private: + void handleVideoFrame(); + + QString m_deviceId; + std::unique_ptr m_camera; + std::unique_ptr m_session; + std::unique_ptr m_sink; + std::unique_ptr m_clock; + FrameMailbox m_mailbox; + double m_nativeFps = 0.0; + qint64 m_frameIndex = 0; +}; + +#endif // ENABLE_MOCAP +#endif // VIDEOFRAMESOURCE_H diff --git a/src/Mocap/VideoFrameSource_test.cpp b/src/Mocap/VideoFrameSource_test.cpp new file mode 100644 index 000000000..e8161599c --- /dev/null +++ b/src/Mocap/VideoFrameSource_test.cpp @@ -0,0 +1,200 @@ +#ifdef ENABLE_MOCAP + +#include + +#include +#include +#include +#include + +#include +#include + +#include "Mocap/VideoFrameSource.h" + +namespace { + +QStringList writeTestImages(const QString& dir, int count, int size = 8) +{ + QStringList paths; + for (int i = 0; i < count; ++i) { + QImage img(size, size, QImage::Format_ARGB32); + img.fill(QColor(i * 10 % 255, 0, 0)); + const QString p = dir + QStringLiteral("/frame_%1.png").arg(i, 3, 10, QChar('0')); + img.save(p); + paths << p; + } + return paths; +} + +} // namespace + +TEST(FrameDecimator, PassesEverythingWhenDisabled) +{ + FrameDecimator d(0.0); + for (int i = 0; i < 10; ++i) + EXPECT_TRUE(d.shouldEmit(i / 60.0)); +} + +TEST(FrameDecimator, HalvesSixtyToThirty) +{ + FrameDecimator d(30.0); + int emitted = 0; + for (int i = 0; i < 60; ++i) + if (d.shouldEmit(i / 60.0)) + ++emitted; + EXPECT_GE(emitted, 28); + EXPECT_LE(emitted, 32); +} + +TEST(FrameDecimator, FirstFrameAlwaysPasses) +{ + FrameDecimator d(1.0); + EXPECT_TRUE(d.shouldEmit(0.0)); + EXPECT_FALSE(d.shouldEmit(0.1)); + d.reset(); + EXPECT_TRUE(d.shouldEmit(0.1)); +} + +TEST(FrameMailbox, LatestWinsDropsIntermediates) +{ + FrameMailbox box; + for (int i = 0; i < 3; ++i) { + MocapFrame f; + f.frameIndex = i; + box.put(f); + } + MocapFrame out; + ASSERT_TRUE(box.take(&out)); + EXPECT_EQ(out.frameIndex, 2); // only the newest survives + EXPECT_FALSE(box.take(&out)); // and only once + EXPECT_EQ(box.droppedCount(), 2); // the two overwritten frames +} + +TEST(FrameMailbox, ThreadSafePutTake) +{ + FrameMailbox box; + std::thread producer([&box] { + for (int i = 0; i < 1000; ++i) { + MocapFrame f; + f.frameIndex = i; + box.put(f); + } + }); + qint64 last = -1; + for (int i = 0; i < 2000; ++i) { + MocapFrame out; + if (box.take(&out)) { + EXPECT_GT(out.frameIndex, last); // monotone: never re-deliver older + last = out.frameIndex; + } + } + producer.join(); + MocapFrame out; + while (box.take(&out)) + last = out.frameIndex; + EXPECT_EQ(last, 999); // the final frame is never lost +} + +TEST(ImageSequenceFrameSource, EmitsAllFramesInOrderWithTimestamps) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QStringList paths = writeTestImages(tmp.path(), 5); + + ImageSequenceFrameSource src(paths, 10.0); + QString error; + ASSERT_TRUE(src.open(&error)) << error.toStdString(); + + std::vector frames; + bool done = false; + QObject::connect(&src, &VideoFrameSource::frameReady, + [&frames](const MocapFrame& f) { frames.push_back(f); }); + QObject::connect(&src, &VideoFrameSource::finished, [&done] { done = true; }); + src.start(); + + ASSERT_TRUE(done); + ASSERT_EQ(frames.size(), 5u); + for (size_t i = 0; i < frames.size(); ++i) { + EXPECT_EQ(frames[i].frameIndex, static_cast(i)); + EXPECT_DOUBLE_EQ(frames[i].timeSec, i / 10.0); + EXPECT_EQ(frames[i].image.format(), QImage::Format_RGB888); + EXPECT_FALSE(frames[i].image.isNull()); + } +} + +TEST(ImageSequenceFrameSource, DecimatesToTargetFps) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QStringList paths = writeTestImages(tmp.path(), 60); + + ImageSequenceFrameSource src(paths, 60.0, /*targetFps=*/30.0); + QString error; + ASSERT_TRUE(src.open(&error)) << error.toStdString(); + + int emitted = 0; + QObject::connect(&src, &VideoFrameSource::frameReady, + [&emitted](const MocapFrame&) { ++emitted; }); + src.start(); + EXPECT_GE(emitted, 28); + EXPECT_LE(emitted, 32); +} + +TEST(ImageSequenceFrameSource, OpenFailsOnMissingFile) +{ + ImageSequenceFrameSource src({QStringLiteral("/nonexistent/frame.png")}, 30.0); + QString error; + EXPECT_FALSE(src.open(&error)); + EXPECT_FALSE(error.isEmpty()); +} + +TEST(ImageSequenceFrameSource, OpenFailsOnEmptyList) +{ + ImageSequenceFrameSource src({}, 30.0); + QString error; + EXPECT_FALSE(src.open(&error)); + EXPECT_FALSE(error.isEmpty()); +} + +TEST(FileFrameSource, OpenFailsOnMissingFile) +{ + FileFrameSource src(QStringLiteral("/nonexistent/video.mp4")); + QString error; + EXPECT_FALSE(src.open(&error)); + EXPECT_FALSE(error.isEmpty()); +} + +// Real camera tests are impossible in CI; enumeration must not crash headless. +TEST(CameraFrameSource, AvailableDevicesDoesNotCrash) +{ + const auto devices = CameraFrameSource::availableDevices(); + for (const auto& d : devices) { + EXPECT_FALSE(d.id.isEmpty()); + } +} + +TEST(CameraFrameSource, OpenFailsOnBogusDeviceId) +{ + if (!qEnvironmentVariableIsSet("QTMESH_MOCAP_CAMERA_TESTS") + && CameraFrameSource::availableDevices().isEmpty()) { + // headless CI: also exercises the no-camera error path + } + CameraFrameSource src(QStringLiteral("definitely-not-a-camera-id")); + QString error; + EXPECT_FALSE(src.open(&error)); + EXPECT_FALSE(error.isEmpty()); +} + +TEST(MocapFrameToRgb888, ConvertsAndPassesThrough) +{ + QImage argb(4, 4, QImage::Format_ARGB32); + argb.fill(Qt::green); + const QImage converted = mocapFrameToRgb888(argb); + EXPECT_EQ(converted.format(), QImage::Format_RGB888); + + const QImage same = mocapFrameToRgb888(converted); + EXPECT_EQ(same.format(), QImage::Format_RGB888); +} + +#endif // ENABLE_MOCAP diff --git a/src/test_main.cpp b/src/test_main.cpp index 8ec3dd95b..23df3646c 100644 --- a/src/test_main.cpp +++ b/src/test_main.cpp @@ -139,12 +139,21 @@ int main(int argc, char **argv) // Prove headless GL works on this runner, then tear down: many suites // (Assimp processors, etc.) construct their own Ogre::Root and cannot // coexist with a live Manager singleton from a prior init. - if (!tryInitOgre()) { + // QTMESH_TESTS_SKIP_OGRE_PREFLIGHT=1 skips the proof so PURE-DATA suites + // can run (with --gtest_filter) on machines with no GL/WindowServer at + // all (remote shells, containers without Xvfb). Ogre-dependent fixtures + // will still fail under it — this only moves the failure from "no test + // ran" to per-fixture. CI never sets it. + if (qEnvironmentVariableIsSet("QTMESH_TESTS_SKIP_OGRE_PREFLIGHT")) { + fprintf(stderr, "UnitTests: skipping Ogre GL preflight " + "(QTMESH_TESTS_SKIP_OGRE_PREFLIGHT set)\n"); + } else if (!tryInitOgre()) { fprintf(stderr, "UnitTests FATAL: tryInitOgre() failed — need working DISPLAY / Xvfb for GL.\n"); return 1; + } else { + Manager::kill(); } - Manager::kill(); if (QCoreApplication::instance()) QCoreApplication::processEvents(); QThread::msleep(50); From 4b437600ca8297bf6764783b0455c242afa8458a Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 12 Jul 2026 21:33:59 -0300 Subject: [PATCH 03/31] feat(#872): FaceCapPredictor (ONNX consumer #9) + mocap pure-data core Slice C of epic #869 (performance capture). Implements the contract proven in docs/MOCAP_SPIKE.md (Slice A) in C++. - FaceCapPredictor: three-session ONNX pipeline (BlazeFace detector -> Face Mesh V2 landmarks -> MLP-Mixer blendshapes) -> FaceSample {52 weights, head pose quat+translation, confidence}. Detector-skip tracking (next-frame ROI from the previous landmarks; detector re-runs only when presence drops). Models download on first use to AppData/ai_models/mocap/face/ (QTMESH_MOCAP_MODEL_BASE_URL / ai/mocapModelBaseUrl / QTMESH_MOCAP_NO_DOWNLOAD; UniRig multi-file pattern). Runtime I/O discovery, anchor-count sanity check, graceful degradation without ONNX/models. - FaceCapGeom (pure, QtGui-only): letterbox + inverse, SSD anchor gen (896 face / 2254 pose), TensorsToDetections decode + weighted NMS, face/pose ROI rects, rotated-crop tensor sampling (plain bilinear, BORDER_ZERO, the cv2-integer-index convention the spike parity pinned), landmark projection. - FaceCapPose (pure): weighted rigid fit via Horn's quaternion method with a self-contained 4x4 Jacobi eigensolver (no linear-algebra dependency); solveHeadPose fits the embedded canonical face model with MediaPipe's Procrustes basis weights. Convention documented in the header. - FaceCapMapper (pure, QtCore-only): canonical-52 -> mesh morph-target names via side-token expansion + normalization + alias table + JSON override sidecar; unmatched channels always reported. - OneEuroFilter (pure): scalar + quaternion (hemisphere-aligned slerp) variants. - FaceCapCanonicalData.h: generated constants (52 names in model order, the 146-landmark blendshape subset, 468 canonical vertices, Procrustes basis). 33 headless tests pass; the env-gated real-inference test (models + QTMESH_MOCAP_MODELS_DIR/QTMESH_MOCAP_TEST_IMAGE) verifies the full pipeline on a real photo: mouthSmileLeft 0.96 on the smiling MediaPipe portrait, identity head pose, confidence 1.0, and the detector-skip assertion. Co-Authored-By: Claude Fable 5 --- src/CMakeLists.txt | 5 + src/Mocap/FaceCapCanonicalData.h | 305 ++++++++++++++++++++ src/Mocap/FaceCapGeom.cpp | 297 +++++++++++++++++++ src/Mocap/FaceCapGeom.h | 103 +++++++ src/Mocap/FaceCapGeom_test.cpp | 194 +++++++++++++ src/Mocap/FaceCapMapper.cpp | 164 +++++++++++ src/Mocap/FaceCapMapper.h | 53 ++++ src/Mocap/FaceCapMapper_test.cpp | 125 ++++++++ src/Mocap/FaceCapPose.cpp | 186 ++++++++++++ src/Mocap/FaceCapPose.h | 46 +++ src/Mocap/FaceCapPose_test.cpp | 165 +++++++++++ src/Mocap/FaceCapPredictor.cpp | 432 ++++++++++++++++++++++++++++ src/Mocap/FaceCapPredictor.h | 70 +++++ src/Mocap/FaceCapPredictor_test.cpp | 84 ++++++ src/Mocap/OneEuroFilter.cpp | 95 ++++++ src/Mocap/OneEuroFilter.h | 61 ++++ src/Mocap/OneEuroFilter_test.cpp | 122 ++++++++ 17 files changed, 2507 insertions(+) create mode 100644 src/Mocap/FaceCapCanonicalData.h create mode 100644 src/Mocap/FaceCapGeom.cpp create mode 100644 src/Mocap/FaceCapGeom.h create mode 100644 src/Mocap/FaceCapGeom_test.cpp create mode 100644 src/Mocap/FaceCapMapper.cpp create mode 100644 src/Mocap/FaceCapMapper.h create mode 100644 src/Mocap/FaceCapMapper_test.cpp create mode 100644 src/Mocap/FaceCapPose.cpp create mode 100644 src/Mocap/FaceCapPose.h create mode 100644 src/Mocap/FaceCapPose_test.cpp create mode 100644 src/Mocap/FaceCapPredictor.cpp create mode 100644 src/Mocap/FaceCapPredictor.h create mode 100644 src/Mocap/FaceCapPredictor_test.cpp create mode 100644 src/Mocap/OneEuroFilter.cpp create mode 100644 src/Mocap/OneEuroFilter.h create mode 100644 src/Mocap/OneEuroFilter_test.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81a3aa044..16c8fa8ee 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -162,6 +162,11 @@ PoseLibrary.cpp VertexAnimationManager.cpp AlembicImporter.cpp Mocap/VideoFrameSource.cpp +Mocap/OneEuroFilter.cpp +Mocap/FaceCapMapper.cpp +Mocap/FaceCapPose.cpp +Mocap/FaceCapGeom.cpp +Mocap/FaceCapPredictor.cpp ApplyAtlas.cpp EmbeddedTextureCache.cpp NormalMapGenerator.cpp diff --git a/src/Mocap/FaceCapCanonicalData.h b/src/Mocap/FaceCapCanonicalData.h new file mode 100644 index 000000000..1c57cb1c0 --- /dev/null +++ b/src/Mocap/FaceCapCanonicalData.h @@ -0,0 +1,305 @@ +#ifndef FACECAPCANONICALDATA_H +#define FACECAPCANONICALDATA_H + +// GENERATED from the MediaPipe reference data (do not hand-edit; see +// scripts/export-facecap-onnx.py and docs/MOCAP_SPIKE.md). +// Sources (Apache-2.0, Google MediaPipe): +// - canonical_face_model.obj (face_geometry module): 468 canonical +// face-mesh vertex positions in centimetres, +X right / +Y up / +Z +// toward the camera; the head-pose Procrustes reference. +// - geometry_pipeline_metadata_landmarks.pbtxt: the 33 weighted +// landmarks MediaPipe uses for its Procrustes fit. +// - face_blendshapes_graph.cc: the 146-landmark subset fed to the +// blendshape model and the canonical 52 blendshape names in the +// model output order. + +#include + +namespace FaceCap { + +inline constexpr int kBlendshapeCount = 52; +inline constexpr std::array kBlendshapeNames = { + "_neutral", "browDownLeft", "browDownRight", "browInnerUp", + "browOuterUpLeft", "browOuterUpRight", "cheekPuff", "cheekSquintLeft", + "cheekSquintRight", "eyeBlinkLeft", "eyeBlinkRight", "eyeLookDownLeft", + "eyeLookDownRight", "eyeLookInLeft", "eyeLookInRight", "eyeLookOutLeft", + "eyeLookOutRight", "eyeLookUpLeft", "eyeLookUpRight", "eyeSquintLeft", + "eyeSquintRight", "eyeWideLeft", "eyeWideRight", "jawForward", + "jawLeft", "jawOpen", "jawRight", "mouthClose", + "mouthDimpleLeft", "mouthDimpleRight", "mouthFrownLeft", "mouthFrownRight", + "mouthFunnel", "mouthLeft", "mouthLowerDownLeft", "mouthLowerDownRight", + "mouthPressLeft", "mouthPressRight", "mouthPucker", "mouthRight", + "mouthRollLower", "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper", + "mouthSmileLeft", "mouthSmileRight", "mouthStretchLeft", "mouthStretchRight", + "mouthUpperUpLeft", "mouthUpperUpRight", "noseSneerLeft", "noseSneerRight", +}; + +inline constexpr int kBlendshapeInputLandmarks = 146; +inline constexpr std::array kBlendshapeLandmarkSubset = { + 0, 1, 4, 5, 6, 7, 8, 10, 13, 14, 17, 21, 33, 37, 39, + 40, 46, 52, 53, 54, 55, 58, 61, 63, 65, 66, 67, 70, 78, 80, + 81, 82, 84, 87, 88, 91, 93, 95, 103, 105, 107, 109, 127, 132, 133, + 136, 144, 145, 146, 148, 149, 150, 152, 153, 154, 155, 157, 158, 159, 160, + 161, 162, 163, 168, 172, 173, 176, 178, 181, 185, 191, 195, 197, 234, 246, + 249, 251, 263, 267, 269, 270, 276, 282, 283, 284, 285, 288, 291, 293, 295, + 296, 297, 300, 308, 310, 311, 312, 314, 317, 318, 321, 323, 324, 332, 334, + 336, 338, 356, 361, 362, 365, 373, 374, 375, 377, 378, 379, 380, 381, 382, + 384, 385, 386, 387, 388, 389, 390, 397, 398, 400, 402, 405, 409, 415, 454, + 466, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, +}; + +inline constexpr int kCanonicalVertexCount = 468; +// x,y,z per vertex, centimetres. +inline constexpr std::array kCanonicalFaceModel = { + 0.000000f, -3.406404f, 5.979507f, 0.000000f, -1.126865f, 7.475604f, + 0.000000f, -2.089024f, 6.058267f, -0.463928f, 0.955357f, 6.633583f, + 0.000000f, -0.463170f, 7.586580f, 0.000000f, 0.365669f, 7.242870f, + 0.000000f, 2.473255f, 5.788627f, -4.253081f, 2.577646f, 3.279702f, + 0.000000f, 4.019042f, 5.284764f, 0.000000f, 4.885979f, 5.385258f, + 0.000000f, 8.261778f, 4.481535f, 0.000000f, -3.706811f, 5.864924f, + 0.000000f, -3.918301f, 5.569430f, 0.000000f, -3.994436f, 5.219482f, + 0.000000f, -4.542400f, 5.404754f, 0.000000f, -4.745577f, 5.529457f, + 0.000000f, -5.019567f, 5.601448f, 0.000000f, -5.365123f, 5.535441f, + 0.000000f, -6.149624f, 5.071372f, 0.000000f, -1.501095f, 7.112196f, + -0.416106f, -1.466449f, 6.447657f, -7.087960f, 5.434801f, 0.099620f, + -2.628639f, 2.035898f, 3.848121f, -3.198363f, 1.985815f, 3.796952f, + -3.775151f, 2.039402f, 3.646194f, -4.465819f, 2.422950f, 3.155168f, + -2.164289f, 2.189867f, 3.851822f, -3.208229f, 3.223926f, 4.115822f, + -2.673803f, 3.205337f, 4.092203f, -3.745193f, 3.165286f, 3.972409f, + -4.161018f, 3.059069f, 3.719554f, -5.062006f, 1.934418f, 2.776093f, + -2.266659f, -7.425768f, 4.389812f, -4.445859f, 2.663991f, 3.173422f, + -7.214530f, 2.263009f, 0.073150f, -5.799793f, 2.349546f, 2.204059f, + -2.844939f, -0.720868f, 4.433130f, -0.711452f, -3.329355f, 5.877044f, + -0.606033f, -3.924562f, 5.444923f, -1.431615f, -3.500953f, 5.496189f, + -1.914910f, -3.803146f, 5.028930f, -1.131043f, -3.973937f, 5.189648f, + -1.563548f, -4.082763f, 4.842263f, -2.650112f, -5.003649f, 4.188483f, + -0.427049f, -1.094134f, 7.360529f, -0.496396f, -0.475659f, 7.440358f, + -5.253307f, 3.881582f, 3.363159f, -1.718698f, 0.974609f, 4.558359f, + -1.608635f, -0.942516f, 5.814193f, -1.651267f, -0.610868f, 5.581319f, + -4.765501f, -0.701554f, 3.534632f, -0.478306f, 0.295766f, 7.101013f, + -3.734964f, 4.508230f, 4.550454f, -4.588603f, 4.302037f, 4.048484f, + -6.279331f, 6.615427f, 1.425850f, -1.220941f, 4.142165f, 5.106035f, + -2.193489f, 3.100317f, 4.000575f, -3.102642f, -4.352984f, 4.095905f, + -6.719682f, -4.788645f, -1.745401f, -1.193824f, -1.306795f, 5.737747f, + -0.729766f, -1.593712f, 5.833208f, -2.456206f, -4.342621f, 4.283884f, + -2.204823f, -4.304508f, 4.162499f, -4.985894f, 4.802461f, 3.751977f, + -1.592294f, -1.257709f, 5.456949f, -2.644548f, 4.524654f, 4.921559f, + -2.760292f, 5.100971f, 5.015990f, -3.523964f, 8.005976f, 3.729163f, + -5.599763f, 5.715470f, 2.724259f, -3.063932f, 6.566144f, 4.529981f, + -5.720968f, 4.254584f, 2.830852f, -6.374393f, 4.785590f, 1.591691f, + -0.672728f, -3.688016f, 5.737804f, -1.262560f, -3.787691f, 5.417779f, + -1.732553f, -3.952767f, 5.000579f, -1.043625f, -1.464973f, 5.662455f, + -2.321234f, -4.329069f, 4.258156f, -2.056846f, -4.477671f, 4.520883f, + -2.153084f, -4.276322f, 4.038093f, -0.946874f, -1.035249f, 6.512274f, + -1.469132f, -4.036351f, 4.604908f, -1.024340f, -3.989851f, 4.926693f, + -0.533422f, -3.993222f, 5.138202f, -0.769720f, -6.095394f, 4.985883f, + -0.699606f, -5.291850f, 5.448304f, -0.669687f, -4.949770f, 5.509612f, + -0.630947f, -4.695101f, 5.449371f, -0.583218f, -4.517982f, 5.339869f, + -1.537170f, -4.423206f, 4.745470f, -1.615600f, -4.475942f, 4.813632f, + -1.729053f, -4.618680f, 4.854463f, -1.838624f, -4.828746f, 4.823737f, + -2.368250f, -3.106237f, 4.868096f, -7.542244f, -1.049282f, -2.431321f, + 0.000000f, -1.724003f, 6.601390f, -1.826614f, -4.399531f, 4.399021f, + -1.929558f, -4.411831f, 4.497052f, -0.597442f, -2.013686f, 5.866456f, + -1.405627f, -1.714196f, 5.241087f, -0.662449f, -1.819321f, 5.863759f, + -2.342340f, 0.572222f, 4.294303f, -3.327324f, 0.104863f, 4.113860f, + -1.726175f, -0.919165f, 5.273355f, -5.133204f, 7.485602f, 2.660442f, + -4.538641f, 6.319907f, 3.683424f, -3.986562f, 5.109487f, 4.466315f, + -2.169681f, -5.440433f, 4.455874f, -1.395634f, 5.011963f, 5.316032f, + -1.619500f, 6.599217f, 4.921106f, -1.891399f, 8.236377f, 4.274997f, + -4.195832f, 2.235205f, 3.375099f, -5.733342f, 1.411738f, 2.431726f, + -1.859887f, 2.355757f, 3.843181f, -4.988612f, 3.074654f, 3.083858f, + -1.303263f, 1.416453f, 4.831091f, -1.305757f, -0.672779f, 6.415959f, + -6.465170f, 0.937119f, 1.689873f, -5.258659f, 0.945811f, 2.974312f, + -4.432338f, 0.722096f, 3.522615f, -3.300681f, 0.861641f, 3.872784f, + -2.430178f, 1.131492f, 4.039035f, -1.820731f, 1.467954f, 4.224124f, + -0.563221f, 2.307693f, 5.566789f, -6.338145f, -0.529279f, 1.881175f, + -5.587698f, 3.208071f, 2.687839f, -0.242624f, -1.462857f, 7.071491f, + -1.611251f, 0.339326f, 4.895421f, -7.743095f, 2.364999f, -2.005167f, + -1.391142f, 1.851048f, 4.448999f, -1.785794f, -0.978284f, 4.850470f, + -4.670959f, 2.664461f, 3.084075f, -1.333970f, -0.283761f, 6.097047f, + -7.270895f, -2.890917f, -2.252455f, -1.856432f, 2.585245f, 3.757904f, + -0.923388f, 0.073076f, 6.671944f, -5.000589f, -6.135128f, 1.892523f, + -5.085276f, -7.178590f, 0.714711f, -7.159291f, -0.811820f, -0.072044f, + -5.843051f, -5.248023f, 0.924091f, -6.847258f, 3.662916f, 0.724695f, + -2.412942f, -8.258853f, 4.119213f, -0.179909f, -1.689864f, 6.573301f, + -2.103655f, -0.163946f, 4.566119f, -6.407571f, 2.236021f, 1.560843f, + -3.670075f, 2.360153f, 3.635230f, -3.177186f, 2.294265f, 3.775704f, + -2.196121f, -4.598322f, 4.479786f, -6.234883f, -1.944430f, 1.663542f, + -1.292924f, -9.295920f, 4.094063f, -3.210651f, -8.533278f, 2.802001f, + -4.068926f, -7.993109f, 1.925119f, 0.000000f, 6.545390f, 5.027311f, + 0.000000f, -9.403378f, 4.264492f, -2.724032f, 2.315802f, 3.777151f, + -2.288460f, 2.398891f, 3.697603f, -1.998311f, 2.496547f, 3.689148f, + -6.130040f, 3.399261f, 2.038516f, -2.288460f, 2.886504f, 3.775031f, + -2.724032f, 2.961810f, 3.871767f, -3.177186f, 2.964136f, 3.876973f, + -3.670075f, 2.927714f, 3.724325f, -4.018389f, 2.857357f, 3.482983f, + -7.555811f, 4.106811f, -0.991917f, -4.018389f, 2.483695f, 3.440898f, + 0.000000f, -2.521945f, 5.932265f, -1.776217f, -2.683946f, 5.213116f, + -1.222237f, -1.182444f, 5.952465f, -0.731493f, -2.536683f, 5.815343f, + 0.000000f, 3.271027f, 5.236015f, -4.135272f, -6.996638f, 2.671970f, + -3.311811f, -7.660815f, 3.382963f, -1.313701f, -8.639995f, 4.702456f, + -5.940524f, -6.223629f, -0.631468f, -1.998311f, 2.743838f, 3.744030f, + -0.901447f, 1.236992f, 5.754256f, 0.000000f, -8.765243f, 4.891441f, + -2.308977f, -8.974196f, 3.609070f, -6.954154f, -2.439843f, -0.131163f, + -1.098819f, -4.458788f, 5.120727f, -1.181124f, -4.579996f, 5.189564f, + -1.255818f, -4.787901f, 5.237051f, -1.325085f, -5.106507f, 5.205010f, + -1.546388f, -5.819392f, 4.757893f, -1.953754f, -4.183892f, 4.431713f, + -2.117802f, -4.137093f, 4.555096f, -2.285339f, -4.051196f, 4.582438f, + -2.850160f, -3.665720f, 4.484994f, -5.278538f, -2.238942f, 2.861224f, + -0.946709f, 1.907628f, 5.196779f, -1.314173f, 3.104912f, 4.231404f, + -1.780000f, 2.860000f, 3.881555f, -1.845110f, -4.098880f, 4.247264f, + -5.436187f, -4.030482f, 2.109852f, -0.766444f, 3.182131f, 4.861453f, + -1.938616f, -6.614410f, 4.521085f, 0.000000f, 1.059413f, 6.774605f, + -0.516573f, 1.583572f, 6.148363f, 0.000000f, 1.728369f, 6.316750f, + -1.246815f, 0.230297f, 5.681036f, 0.000000f, -7.942194f, 5.181173f, + 0.000000f, -6.991499f, 5.153478f, -0.997827f, -6.930921f, 4.979576f, + -3.288807f, -5.382514f, 3.795752f, -2.311631f, -1.566237f, 4.590085f, + -2.680250f, -6.111567f, 4.096152f, -3.832928f, -1.537326f, 4.137731f, + -2.961860f, -2.274215f, 4.440943f, -4.386901f, -2.683286f, 3.643886f, + -1.217295f, -7.834465f, 4.969286f, -1.542374f, -0.136843f, 5.201008f, + -3.878377f, -6.041764f, 3.311079f, -3.084037f, -6.809842f, 3.814195f, + -3.747321f, -4.503545f, 3.726453f, -6.094129f, -3.205991f, 1.473482f, + -4.588995f, -4.728726f, 2.983221f, -6.583231f, -3.941269f, 0.070268f, + -3.492580f, -3.195820f, 4.130198f, -1.255543f, 0.802341f, 5.307551f, + -1.126122f, -0.933602f, 6.538785f, -1.443109f, -1.142774f, 5.905127f, + -0.923043f, -0.529042f, 7.003423f, -1.755386f, 3.529117f, 4.327696f, + -2.632589f, 3.713828f, 4.364629f, -3.388062f, 3.721976f, 4.309028f, + -4.075766f, 3.675413f, 4.076063f, -4.622910f, 3.474691f, 3.646321f, + -5.171755f, 2.535753f, 2.670867f, -7.297331f, 0.763172f, -0.048769f, + -4.706828f, 1.651000f, 3.109532f, -4.071712f, 1.476821f, 3.476944f, + -3.269817f, 1.470659f, 3.731945f, -2.527572f, 1.617311f, 3.865444f, + -1.970894f, 1.858505f, 3.961782f, -1.579543f, 2.097941f, 4.084996f, + -7.664182f, 0.673132f, -2.435867f, -1.397041f, -1.340139f, 5.630378f, + -0.884838f, 0.658740f, 6.233232f, -0.767097f, -0.968035f, 7.077932f, + -0.460213f, -1.334106f, 6.787447f, -0.748618f, -1.067994f, 6.798303f, + -1.236408f, -1.585568f, 5.480490f, -0.387306f, -1.409990f, 6.957705f, + -0.319925f, -1.607931f, 6.508676f, -1.639633f, 2.556298f, 3.863736f, + -1.255645f, 2.467144f, 4.203800f, -1.031362f, 2.382663f, 4.615849f, + -4.253081f, 2.772296f, 3.315305f, -4.530000f, 2.910000f, 3.339685f, + 0.463928f, 0.955357f, 6.633583f, 4.253081f, 2.577646f, 3.279702f, + 0.416106f, -1.466449f, 6.447657f, 7.087960f, 5.434801f, 0.099620f, + 2.628639f, 2.035898f, 3.848121f, 3.198363f, 1.985815f, 3.796952f, + 3.775151f, 2.039402f, 3.646194f, 4.465819f, 2.422950f, 3.155168f, + 2.164289f, 2.189867f, 3.851822f, 3.208229f, 3.223926f, 4.115822f, + 2.673803f, 3.205337f, 4.092203f, 3.745193f, 3.165286f, 3.972409f, + 4.161018f, 3.059069f, 3.719554f, 5.062006f, 1.934418f, 2.776093f, + 2.266659f, -7.425768f, 4.389812f, 4.445859f, 2.663991f, 3.173422f, + 7.214530f, 2.263009f, 0.073150f, 5.799793f, 2.349546f, 2.204059f, + 2.844939f, -0.720868f, 4.433130f, 0.711452f, -3.329355f, 5.877044f, + 0.606033f, -3.924562f, 5.444923f, 1.431615f, -3.500953f, 5.496189f, + 1.914910f, -3.803146f, 5.028930f, 1.131043f, -3.973937f, 5.189648f, + 1.563548f, -4.082763f, 4.842263f, 2.650112f, -5.003649f, 4.188483f, + 0.427049f, -1.094134f, 7.360529f, 0.496396f, -0.475659f, 7.440358f, + 5.253307f, 3.881582f, 3.363159f, 1.718698f, 0.974609f, 4.558359f, + 1.608635f, -0.942516f, 5.814193f, 1.651267f, -0.610868f, 5.581319f, + 4.765501f, -0.701554f, 3.534632f, 0.478306f, 0.295766f, 7.101013f, + 3.734964f, 4.508230f, 4.550454f, 4.588603f, 4.302037f, 4.048484f, + 6.279331f, 6.615427f, 1.425850f, 1.220941f, 4.142165f, 5.106035f, + 2.193489f, 3.100317f, 4.000575f, 3.102642f, -4.352984f, 4.095905f, + 6.719682f, -4.788645f, -1.745401f, 1.193824f, -1.306795f, 5.737747f, + 0.729766f, -1.593712f, 5.833208f, 2.456206f, -4.342621f, 4.283884f, + 2.204823f, -4.304508f, 4.162499f, 4.985894f, 4.802461f, 3.751977f, + 1.592294f, -1.257709f, 5.456949f, 2.644548f, 4.524654f, 4.921559f, + 2.760292f, 5.100971f, 5.015990f, 3.523964f, 8.005976f, 3.729163f, + 5.599763f, 5.715470f, 2.724259f, 3.063932f, 6.566144f, 4.529981f, + 5.720968f, 4.254584f, 2.830852f, 6.374393f, 4.785590f, 1.591691f, + 0.672728f, -3.688016f, 5.737804f, 1.262560f, -3.787691f, 5.417779f, + 1.732553f, -3.952767f, 5.000579f, 1.043625f, -1.464973f, 5.662455f, + 2.321234f, -4.329069f, 4.258156f, 2.056846f, -4.477671f, 4.520883f, + 2.153084f, -4.276322f, 4.038093f, 0.946874f, -1.035249f, 6.512274f, + 1.469132f, -4.036351f, 4.604908f, 1.024340f, -3.989851f, 4.926693f, + 0.533422f, -3.993222f, 5.138202f, 0.769720f, -6.095394f, 4.985883f, + 0.699606f, -5.291850f, 5.448304f, 0.669687f, -4.949770f, 5.509612f, + 0.630947f, -4.695101f, 5.449371f, 0.583218f, -4.517982f, 5.339869f, + 1.537170f, -4.423206f, 4.745470f, 1.615600f, -4.475942f, 4.813632f, + 1.729053f, -4.618680f, 4.854463f, 1.838624f, -4.828746f, 4.823737f, + 2.368250f, -3.106237f, 4.868096f, 7.542244f, -1.049282f, -2.431321f, + 1.826614f, -4.399531f, 4.399021f, 1.929558f, -4.411831f, 4.497052f, + 0.597442f, -2.013686f, 5.866456f, 1.405627f, -1.714196f, 5.241087f, + 0.662449f, -1.819321f, 5.863759f, 2.342340f, 0.572222f, 4.294303f, + 3.327324f, 0.104863f, 4.113860f, 1.726175f, -0.919165f, 5.273355f, + 5.133204f, 7.485602f, 2.660442f, 4.538641f, 6.319907f, 3.683424f, + 3.986562f, 5.109487f, 4.466315f, 2.169681f, -5.440433f, 4.455874f, + 1.395634f, 5.011963f, 5.316032f, 1.619500f, 6.599217f, 4.921106f, + 1.891399f, 8.236377f, 4.274997f, 4.195832f, 2.235205f, 3.375099f, + 5.733342f, 1.411738f, 2.431726f, 1.859887f, 2.355757f, 3.843181f, + 4.988612f, 3.074654f, 3.083858f, 1.303263f, 1.416453f, 4.831091f, + 1.305757f, -0.672779f, 6.415959f, 6.465170f, 0.937119f, 1.689873f, + 5.258659f, 0.945811f, 2.974312f, 4.432338f, 0.722096f, 3.522615f, + 3.300681f, 0.861641f, 3.872784f, 2.430178f, 1.131492f, 4.039035f, + 1.820731f, 1.467954f, 4.224124f, 0.563221f, 2.307693f, 5.566789f, + 6.338145f, -0.529279f, 1.881175f, 5.587698f, 3.208071f, 2.687839f, + 0.242624f, -1.462857f, 7.071491f, 1.611251f, 0.339326f, 4.895421f, + 7.743095f, 2.364999f, -2.005167f, 1.391142f, 1.851048f, 4.448999f, + 1.785794f, -0.978284f, 4.850470f, 4.670959f, 2.664461f, 3.084075f, + 1.333970f, -0.283761f, 6.097047f, 7.270895f, -2.890917f, -2.252455f, + 1.856432f, 2.585245f, 3.757904f, 0.923388f, 0.073076f, 6.671944f, + 5.000589f, -6.135128f, 1.892523f, 5.085276f, -7.178590f, 0.714711f, + 7.159291f, -0.811820f, -0.072044f, 5.843051f, -5.248023f, 0.924091f, + 6.847258f, 3.662916f, 0.724695f, 2.412942f, -8.258853f, 4.119213f, + 0.179909f, -1.689864f, 6.573301f, 2.103655f, -0.163946f, 4.566119f, + 6.407571f, 2.236021f, 1.560843f, 3.670075f, 2.360153f, 3.635230f, + 3.177186f, 2.294265f, 3.775704f, 2.196121f, -4.598322f, 4.479786f, + 6.234883f, -1.944430f, 1.663542f, 1.292924f, -9.295920f, 4.094063f, + 3.210651f, -8.533278f, 2.802001f, 4.068926f, -7.993109f, 1.925119f, + 2.724032f, 2.315802f, 3.777151f, 2.288460f, 2.398891f, 3.697603f, + 1.998311f, 2.496547f, 3.689148f, 6.130040f, 3.399261f, 2.038516f, + 2.288460f, 2.886504f, 3.775031f, 2.724032f, 2.961810f, 3.871767f, + 3.177186f, 2.964136f, 3.876973f, 3.670075f, 2.927714f, 3.724325f, + 4.018389f, 2.857357f, 3.482983f, 7.555811f, 4.106811f, -0.991917f, + 4.018389f, 2.483695f, 3.440898f, 1.776217f, -2.683946f, 5.213116f, + 1.222237f, -1.182444f, 5.952465f, 0.731493f, -2.536683f, 5.815343f, + 4.135272f, -6.996638f, 2.671970f, 3.311811f, -7.660815f, 3.382963f, + 1.313701f, -8.639995f, 4.702456f, 5.940524f, -6.223629f, -0.631468f, + 1.998311f, 2.743838f, 3.744030f, 0.901447f, 1.236992f, 5.754256f, + 2.308977f, -8.974196f, 3.609070f, 6.954154f, -2.439843f, -0.131163f, + 1.098819f, -4.458788f, 5.120727f, 1.181124f, -4.579996f, 5.189564f, + 1.255818f, -4.787901f, 5.237051f, 1.325085f, -5.106507f, 5.205010f, + 1.546388f, -5.819392f, 4.757893f, 1.953754f, -4.183892f, 4.431713f, + 2.117802f, -4.137093f, 4.555096f, 2.285339f, -4.051196f, 4.582438f, + 2.850160f, -3.665720f, 4.484994f, 5.278538f, -2.238942f, 2.861224f, + 0.946709f, 1.907628f, 5.196779f, 1.314173f, 3.104912f, 4.231404f, + 1.780000f, 2.860000f, 3.881555f, 1.845110f, -4.098880f, 4.247264f, + 5.436187f, -4.030482f, 2.109852f, 0.766444f, 3.182131f, 4.861453f, + 1.938616f, -6.614410f, 4.521085f, 0.516573f, 1.583572f, 6.148363f, + 1.246815f, 0.230297f, 5.681036f, 0.997827f, -6.930921f, 4.979576f, + 3.288807f, -5.382514f, 3.795752f, 2.311631f, -1.566237f, 4.590085f, + 2.680250f, -6.111567f, 4.096152f, 3.832928f, -1.537326f, 4.137731f, + 2.961860f, -2.274215f, 4.440943f, 4.386901f, -2.683286f, 3.643886f, + 1.217295f, -7.834465f, 4.969286f, 1.542374f, -0.136843f, 5.201008f, + 3.878377f, -6.041764f, 3.311079f, 3.084037f, -6.809842f, 3.814195f, + 3.747321f, -4.503545f, 3.726453f, 6.094129f, -3.205991f, 1.473482f, + 4.588995f, -4.728726f, 2.983221f, 6.583231f, -3.941269f, 0.070268f, + 3.492580f, -3.195820f, 4.130198f, 1.255543f, 0.802341f, 5.307551f, + 1.126122f, -0.933602f, 6.538785f, 1.443109f, -1.142774f, 5.905127f, + 0.923043f, -0.529042f, 7.003423f, 1.755386f, 3.529117f, 4.327696f, + 2.632589f, 3.713828f, 4.364629f, 3.388062f, 3.721976f, 4.309028f, + 4.075766f, 3.675413f, 4.076063f, 4.622910f, 3.474691f, 3.646321f, + 5.171755f, 2.535753f, 2.670867f, 7.297331f, 0.763172f, -0.048769f, + 4.706828f, 1.651000f, 3.109532f, 4.071712f, 1.476821f, 3.476944f, + 3.269817f, 1.470659f, 3.731945f, 2.527572f, 1.617311f, 3.865444f, + 1.970894f, 1.858505f, 3.961782f, 1.579543f, 2.097941f, 4.084996f, + 7.664182f, 0.673132f, -2.435867f, 1.397041f, -1.340139f, 5.630378f, + 0.884838f, 0.658740f, 6.233232f, 0.767097f, -0.968035f, 7.077932f, + 0.460213f, -1.334106f, 6.787447f, 0.748618f, -1.067994f, 6.798303f, + 1.236408f, -1.585568f, 5.480490f, 0.387306f, -1.409990f, 6.957705f, + 0.319925f, -1.607931f, 6.508676f, 1.639633f, 2.556298f, 3.863736f, + 1.255645f, 2.467144f, 4.203800f, 1.031362f, 2.382663f, 4.615849f, + 4.253081f, 2.772296f, 3.315305f, 4.530000f, 2.910000f, 3.339685f, +}; + +inline constexpr int kProcrustesBasisCount = 33; +inline constexpr std::array kProcrustesLandmarkIds = { + 4, 6, 10, 33, 54, 67, 117, 119, 121, 127, 129, 132, 133, 136, 143, 147, 198, 205, 263, 284, 297, 346, 348, 350, 356, 358, 361, 362, 365, 372, 376, 420, 425, +}; +inline constexpr std::array kProcrustesWeights = { + 0.070910f, 0.032100f, 0.008447f, 0.058724f, 0.007667f, 0.009078f, + 0.009792f, 0.014565f, 0.018591f, 0.005198f, 0.120625f, 0.005560f, + 0.053286f, 0.066890f, 0.014817f, 0.014263f, 0.025462f, 0.047252f, + 0.058724f, 0.007667f, 0.009078f, 0.009792f, 0.014565f, 0.018591f, + 0.005198f, 0.120625f, 0.005560f, 0.053286f, 0.066890f, 0.014817f, + 0.014263f, 0.025462f, 0.047252f, +}; + +} // namespace FaceCap + +#endif // FACECAPCANONICALDATA_H diff --git a/src/Mocap/FaceCapGeom.cpp b/src/Mocap/FaceCapGeom.cpp new file mode 100644 index 000000000..320c36c73 --- /dev/null +++ b/src/Mocap/FaceCapGeom.cpp @@ -0,0 +1,297 @@ +#include "FaceCapGeom.h" + +#include +#include + +namespace FaceCapGeom { + +namespace { + +constexpr float kMinScale = 0.1484375f; +constexpr float kMaxScale = 0.75f; +constexpr float kAnchorOffset = 0.5f; + +float sigmoid(float x) +{ + x = std::clamp(x, -100.f, 100.f); + return x >= 0.f ? 1.f / (1.f + std::exp(-x)) + : std::exp(x) / (1.f + std::exp(x)); +} + +float normalizeAngle(float a) +{ + return a - 2.f * static_cast(M_PI) + * std::floor((a + static_cast(M_PI)) + / (2.f * static_cast(M_PI))); +} + +// Bilinear sample of an RGB888 image with zero border (MediaPipe BORDER_ZERO, +// non-antialiased — the parity requirement from the spike). +inline void sampleBilinear(const uchar* bits, int w, int h, qsizetype stride, + float x, float y, float rgb[3]) +{ + const int x0 = static_cast(std::floor(x)); + const int y0 = static_cast(std::floor(y)); + const float fx = x - x0; + const float fy = y - y0; + float acc[3] = {0.f, 0.f, 0.f}; + for (int dy = 0; dy < 2; ++dy) { + const int yy = y0 + dy; + if (yy < 0 || yy >= h) + continue; + const float wy = dy ? fy : 1.f - fy; + const uchar* row = bits + yy * stride; + for (int dx = 0; dx < 2; ++dx) { + const int xx = x0 + dx; + if (xx < 0 || xx >= w) + continue; + const float wgt = wy * (dx ? fx : 1.f - fx); + const uchar* px = row + xx * 3; + acc[0] += wgt * px[0]; + acc[1] += wgt * px[1]; + acc[2] += wgt * px[2]; + } + } + rgb[0] = acc[0]; + rgb[1] = acc[1]; + rgb[2] = acc[2]; +} + +// Generic affine resample: out(u,v) <- img(origin + u*du + v*dv), the shared +// core of letterbox and rotated-crop. Indices follow cv2.warpAffine's +// integer-index convention — the one the spike's parity proof pinned down +// (docs/MOCAP_SPIKE.md); do NOT switch to pixel-centre sampling. +void resampleToTensor(const QImage& rgb888, int size, float lo, float hi, + float originX, float originY, float dux, float duy, + float dvx, float dvy, float* out) +{ + const uchar* bits = rgb888.constBits(); + const int w = rgb888.width(); + const int h = rgb888.height(); + const qsizetype stride = rgb888.bytesPerLine(); + const float scale = (hi - lo) / 255.f; + float* dst = out; + for (int v = 0; v < size; ++v) { + for (int u = 0; u < size; ++u) { + const float sx = originX + u * dux + v * dvx; + const float sy = originY + u * duy + v * dvy; + float rgb[3]; + sampleBilinear(bits, w, h, stride, sx, sy, rgb); + *dst++ = rgb[0] * scale + lo; + *dst++ = rgb[1] * scale + lo; + *dst++ = rgb[2] * scale + lo; + } + } +} + +float iou(const std::array& a, const std::array& b) +{ + const float x1 = std::max(a[0], b[0]); + const float y1 = std::max(a[1], b[1]); + const float x2 = std::min(a[0] + a[2], b[0] + b[2]); + const float y2 = std::min(a[1] + a[3], b[1] + b[3]); + const float inter = std::max(0.f, x2 - x1) * std::max(0.f, y2 - y1); + const float uni = a[2] * a[3] + b[2] * b[3] - inter; + return uni > 0.f ? inter / uni : 0.f; +} + +} // namespace + +Letterbox letterboxToTensor(const QImage& rgb888, int size, float lo, float hi, + float* out) +{ + Letterbox lb; + const int w = rgb888.width(); + const int h = rgb888.height(); + if (w <= 0 || h <= 0) + return lb; + const float s = static_cast(size) / std::max(w, h); + const float ox = (size - w * s) / 2.f; + const float oy = (size - h * s) / 2.f; + lb.padX = ox / size; + lb.padY = oy / size; + lb.fracX = w * s / size; + lb.fracY = h * s / size; + // output (u,v) -> input ((u - ox) / s, (v - oy) / s) + resampleToTensor(rgb888, size, lo, hi, -ox / s, -oy / s, 1.f / s, 0.f, 0.f, + 1.f / s, out); + return lb; +} + +std::vector> genSsdAnchors(int inputSize, + const std::vector& strides) +{ + std::vector> anchors; + const int n = static_cast(strides.size()); + auto scaleFor = [&](int i) { + return n == 1 ? (kMinScale + kMaxScale) * 0.5f + : kMinScale + (kMaxScale - kMinScale) * i / (n - 1); + }; + int layer = 0; + while (layer < n) { + int last = layer; + int scalesInLayer = 0; + while (last < n && strides[last] == strides[layer]) { + scalesInLayer += 2; // scale + interpolated scale (aspect 1.0) + (void)scaleFor(last); + ++last; + } + const int stride = strides[layer]; + const int fm = (inputSize + stride - 1) / stride; + for (int y = 0; y < fm; ++y) + for (int x = 0; x < fm; ++x) + for (int k = 0; k < scalesInLayer; ++k) + anchors.push_back({(x + kAnchorOffset) / fm, + (y + kAnchorOffset) / fm}); + layer = last; + } + return anchors; +} + +std::vector decodeDetections( + const float* rawBoxes, const float* rawScores, + const std::vector>& anchors, int inputSize, + int numKeypoints, float minScore, float iouThreshold) +{ + const int boxDim = 4 + 2 * numKeypoints; + std::vector candidates; + for (size_t i = 0; i < anchors.size(); ++i) { + const float score = sigmoid(rawScores[i]); + if (score < minScore) + continue; + const float* box = rawBoxes + i * boxDim; + Detection d; + d.score = score; + const float cx = box[0] / inputSize + anchors[i][0]; + const float cy = box[1] / inputSize + anchors[i][1]; + const float w = box[2] / inputSize; + const float h = box[3] / inputSize; + d.box = {cx - w / 2.f, cy - h / 2.f, w, h}; + d.keypoints.reserve(numKeypoints); + for (int k = 0; k < numKeypoints; ++k) + d.keypoints.push_back({box[4 + 2 * k] / inputSize + anchors[i][0], + box[5 + 2 * k] / inputSize + anchors[i][1]}); + candidates.push_back(std::move(d)); + } + + // weighted NMS (MediaPipe): clusters above the IoU threshold merge into a + // score-weighted average instead of being discarded. + std::sort(candidates.begin(), candidates.end(), + [](const Detection& a, const Detection& b) { return a.score > b.score; }); + std::vector out; + std::vector used(candidates.size(), false); + for (size_t i = 0; i < candidates.size(); ++i) { + if (used[i]) + continue; + std::vector cluster{i}; + for (size_t j = i + 1; j < candidates.size(); ++j) { + if (!used[j] + && iou(candidates[i].box, candidates[j].box) > iouThreshold) + cluster.push_back(j); + } + float total = 0.f; + Detection merged = candidates[i]; + std::fill(merged.box.begin(), merged.box.end(), 0.f); + for (auto& kp : merged.keypoints) + kp = {0.f, 0.f}; + for (size_t j : cluster) { + used[j] = true; + total += candidates[j].score; + } + for (size_t j : cluster) { + const float w = candidates[j].score / total; + for (int k = 0; k < 4; ++k) + merged.box[k] += w * candidates[j].box[k]; + for (size_t k = 0; k < merged.keypoints.size(); ++k) { + merged.keypoints[k][0] += w * candidates[j].keypoints[k][0]; + merged.keypoints[k][1] += w * candidates[j].keypoints[k][1]; + } + } + out.push_back(std::move(merged)); + } + return out; +} + +RotatedRect rectFromFaceDetection(const Detection& det, int imgW, int imgH) +{ + RotatedRect r; + r.cx = (det.box[0] + det.box[2] / 2.f) * imgW; + r.cy = (det.box[1] + det.box[3] / 2.f) * imgH; + const float dx = (det.keypoints[1][0] - det.keypoints[0][0]) * imgW; + const float dy = (det.keypoints[1][1] - det.keypoints[0][1]) * imgH; + r.angle = normalizeAngle(-std::atan2(-dy, dx)); // target angle 0 + const float side = std::max(det.box[2] * imgW, det.box[3] * imgH) * 1.5f; + r.w = r.h = side; + return r; +} + +RotatedRect rectFromFaceLandmarks(const float* landmarksXyz, int count, + int imgW, int imgH) +{ + RotatedRect r; + if (count < 264) + return r; + float minX = landmarksXyz[0], maxX = landmarksXyz[0]; + float minY = landmarksXyz[1], maxY = landmarksXyz[1]; + for (int i = 1; i < count; ++i) { + minX = std::min(minX, landmarksXyz[i * 3]); + maxX = std::max(maxX, landmarksXyz[i * 3]); + minY = std::min(minY, landmarksXyz[i * 3 + 1]); + maxY = std::max(maxY, landmarksXyz[i * 3 + 1]); + } + r.cx = (minX + maxX) / 2.f; + r.cy = (minY + maxY) / 2.f; + // eye outer corners: 33 (right), 263 (left) — MediaPipe tracking mode + const float dx = landmarksXyz[263 * 3] - landmarksXyz[33 * 3]; + const float dy = landmarksXyz[263 * 3 + 1] - landmarksXyz[33 * 3 + 1]; + r.angle = normalizeAngle(-std::atan2(-dy, dx)); + const float side = std::max(maxX - minX, maxY - minY) * 1.5f; + r.w = r.h = side; + (void)imgW; + (void)imgH; + return r; +} + +RotatedRect rectFromPoseDetection(const Detection& det, int imgW, int imgH) +{ + RotatedRect r; + r.cx = det.keypoints[0][0] * imgW; + r.cy = det.keypoints[0][1] * imgH; + const float dx = det.keypoints[1][0] * imgW - r.cx; + const float dy = det.keypoints[1][1] * imgH - r.cy; + const float radius = std::sqrt(dx * dx + dy * dy); + r.angle = normalizeAngle(static_cast(M_PI) / 2.f - std::atan2(-dy, dx)); + r.w = r.h = 2.f * radius * 1.25f; + return r; +} + +void cropRotatedRectToTensor(const QImage& rgb888, const RotatedRect& rect, + int size, float lo, float hi, float* out) +{ + const float ca = std::cos(rect.angle); + const float sa = std::sin(rect.angle); + const float sx = rect.w / size; + const float sy = rect.h / size; + // p_img = centre + R(angle) . ((u/size - 0.5) * w, (v/size - 0.5) * h) + const float originX = rect.cx + ca * (-rect.w / 2.f) - sa * (-rect.h / 2.f); + const float originY = rect.cy + sa * (-rect.w / 2.f) + ca * (-rect.h / 2.f); + resampleToTensor(rgb888, size, lo, hi, originX, originY, ca * sx, sa * sx, + -sa * sy, ca * sy, out); +} + +void projectLandmarks(float* pts, int count, int stride, const RotatedRect& rect) +{ + const float ca = std::cos(rect.angle); + const float sa = std::sin(rect.angle); + for (int i = 0; i < count; ++i) { + float* p = pts + i * stride; + const float x = p[0] - 0.5f; + const float y = p[1] - 0.5f; + p[0] = rect.cx + x * rect.w * ca - y * rect.h * sa; + p[1] = rect.cy + x * rect.w * sa + y * rect.h * ca; + if (stride > 2) + p[2] *= rect.w; + } +} + +} // namespace FaceCapGeom diff --git a/src/Mocap/FaceCapGeom.h b/src/Mocap/FaceCapGeom.h new file mode 100644 index 000000000..bdd5481df --- /dev/null +++ b/src/Mocap/FaceCapGeom.h @@ -0,0 +1,103 @@ +#ifndef FACECAPGEOM_H +#define FACECAPGEOM_H + +// Image-space geometry for the mocap predictors (epic #869, Slice C #872): +// letterbox transform + inverse, SSD anchor generation + detection decode + +// weighted NMS, detection->ROI rects, rotated-rect crop sampling and landmark +// re-projection. Every function mirrors the MediaPipe calculator semantics +// proven in scripts/export-facecap-onnx.py — see docs/MOCAP_SPIKE.md for the +// contract (incl. the "plain bilinear, never antialiased" requirement). +// QtGui-only (QImage); no Ogre/ONNX; headless-tested. + +#include + +#include +#include + +namespace FaceCapGeom { + +// ---- letterbox -------------------------------------------------------------- + +struct Letterbox { + float padX = 0.f; // normalized padding left of the image content + float padY = 0.f; // normalized padding above the image content + float fracX = 1.f; // fraction of the square the content occupies in x + float fracY = 1.f; +}; + +// Keep-aspect centre letterbox of an RGB888 image into a size x size float +// tensor, NHWC interleaved RGB, plain bilinear, zero border. Values are +// normalized to [lo, hi] ((v/255) * (hi-lo) + lo). out must hold +// size*size*3 floats. +Letterbox letterboxToTensor(const QImage& rgb888, int size, float lo, float hi, + float* out); + +// Map a point from letterboxed-square normalized coords back to original +// normalized image coords. +inline void unletterbox(const Letterbox& lb, float& x, float& y) +{ + x = (x - lb.padX) / lb.fracX; + y = (y - lb.padY) / lb.fracY; +} + +// ---- SSD detection decode --------------------------------------------------- + +struct Detection { + float score = 0.f; + // box in normalized coords of the DECODE frame (the letterboxed square + // until unletterboxed): x, y = top-left; w, h. + std::array box{0.f, 0.f, 0.f, 0.f}; + std::vector> keypoints; +}; + +// MediaPipe SsdAnchorsCalculator with aspect_ratios=[1.0], fixed anchor size, +// min/max scale 0.1484375/0.75, offset 0.5, interpolated aspect 1.0. +// Face detector: inputSize 128, strides {8,16,16,16} -> 896 anchors. +// Pose detector: inputSize 224, strides {8,16,32,32,32} -> 2254 anchors. +std::vector> genSsdAnchors(int inputSize, + const std::vector& strides); + +// TensorsToDetections + weighted NMS. rawBoxes is anchorCount x boxDim +// (boxDim = 4 + 2*numKeypoints), rawScores anchorCount x 1 (logits). +std::vector decodeDetections( + const float* rawBoxes, const float* rawScores, + const std::vector>& anchors, int inputSize, + int numKeypoints, float minScore = 0.5f, float iouThreshold = 0.3f); + +// ---- ROI rects -------------------------------------------------------------- + +struct RotatedRect { + float cx = 0.f, cy = 0.f; // centre, px + float w = 0.f, h = 0.f; // px + float angle = 0.f; // MediaPipe rotation, rad: R = [[c,-s],[s,c]] y-down +}; + +// Face: rotation from detector keypoint 0 (right eye) -> 1 (left eye), target +// angle 0; box made square-long then scaled 1.5x. Detection coords normalized +// to the ORIGINAL image (unletterbox first). +RotatedRect rectFromFaceDetection(const Detection& det, int imgW, int imgH); + +// Tracking mode: next-frame ROI from the previous frame's 478 landmarks (px): +// bounding box -> square-long * 1.5, rotation from eye outer corners 33->263. +RotatedRect rectFromFaceLandmarks(const float* landmarksXyz, int count, + int imgW, int imgH); + +// Pose: centre keypoint 0 (mid-hip), radius |kp1-kp0|, box 2*radius square, +// target angle 90 deg, scaled 1.25x. +RotatedRect rectFromPoseDetection(const Detection& det, int imgW, int imgH); + +// ---- crop + projection ------------------------------------------------------ + +// Sample the rotated rect into a size x size float tensor (NHWC RGB, range +// [lo, hi], plain bilinear, zero border): +// p_img = centre + R(angle) . ((u/size - 0.5) * w, (v/size - 0.5) * h) +void cropRotatedRectToTensor(const QImage& rgb888, const RotatedRect& rect, + int size, float lo, float hi, float* out); + +// LandmarkProjection: landmark (x,y[,z]) normalized to the crop -> image px +// (z scaled by rect.w). pts is count x stride floats, mutated in place. +void projectLandmarks(float* pts, int count, int stride, const RotatedRect& rect); + +} // namespace FaceCapGeom + +#endif // FACECAPGEOM_H diff --git a/src/Mocap/FaceCapGeom_test.cpp b/src/Mocap/FaceCapGeom_test.cpp new file mode 100644 index 000000000..39366bd14 --- /dev/null +++ b/src/Mocap/FaceCapGeom_test.cpp @@ -0,0 +1,194 @@ +#ifdef ENABLE_MOCAP + +#include + +#include + +#include +#include + +#include "Mocap/FaceCapGeom.h" + +using namespace FaceCapGeom; + +TEST(FaceCapGeom, AnchorCountsMatchTheModels) +{ + EXPECT_EQ(genSsdAnchors(128, {8, 16, 16, 16}).size(), 896u); // BlazeFace + EXPECT_EQ(genSsdAnchors(224, {8, 16, 32, 32, 32}).size(), 2254u); // BlazePose +} + +TEST(FaceCapGeom, LetterboxPadsTheShortSideAndUnmapsBack) +{ + QImage img(200, 100, QImage::Format_RGB888); + img.fill(Qt::white); + std::vector tensor(64 * 64 * 3); + const Letterbox lb = letterboxToTensor(img, 64, 0.f, 1.f, tensor.data()); + + EXPECT_NEAR(lb.padX, 0.f, 1e-5); + EXPECT_NEAR(lb.padY, 0.25f, 1e-5); // 100/200 -> half height, centred + EXPECT_NEAR(lb.fracX, 1.f, 1e-5); + EXPECT_NEAR(lb.fracY, 0.5f, 1e-5); + + // the letterbox bands are zero, the content is white + auto at = [&](int x, int y) { return tensor[(y * 64 + x) * 3]; }; + EXPECT_FLOAT_EQ(at(32, 2), 0.f); // top band + EXPECT_FLOAT_EQ(at(32, 61), 0.f); // bottom band + EXPECT_NEAR(at(32, 32), 1.f, 0.02f); // centre content + + // centre of the letterboxed square unmaps to the image centre + float x = 0.5f, y = 0.5f; + unletterbox(lb, x, y); + EXPECT_NEAR(x, 0.5f, 1e-5); + EXPECT_NEAR(y, 0.5f, 1e-5); + // top of the content band unmaps to y = 0 + x = 0.5f; + y = 0.25f; + unletterbox(lb, x, y); + EXPECT_NEAR(y, 0.f, 1e-5); +} + +TEST(FaceCapGeom, LetterboxNormalizationRange) +{ + QImage img(10, 10, QImage::Format_RGB888); + img.fill(Qt::white); + std::vector tensor(16 * 16 * 3); + letterboxToTensor(img, 16, -1.f, 1.f, tensor.data()); + float mx = -2.f; + for (float v : tensor) + mx = std::max(mx, v); + EXPECT_NEAR(mx, 1.f, 0.02f); // white -> +1 in [-1,1] +} + +TEST(FaceCapGeom, DecodeDetectionsFindsTheSyntheticBox) +{ + const auto anchors = genSsdAnchors(128, {8, 16, 16, 16}); + std::vector boxes(anchors.size() * 16, 0.f); + std::vector scores(anchors.size(), -10.f); // sigmoid ~ 0 + + // plant one detection on anchor 100: centred on the anchor, 32px box, + // keypoints 4px right of centre + const size_t i = 100; + scores[i] = 10.f; // sigmoid ~ 1 + boxes[i * 16 + 2] = 32.f; + boxes[i * 16 + 3] = 32.f; + for (int k = 0; k < 6; ++k) + boxes[i * 16 + 4 + 2 * k] = 4.f; + + const auto dets = + decodeDetections(boxes.data(), scores.data(), anchors, 128, 6); + ASSERT_EQ(dets.size(), 1u); + const auto& d = dets.front(); + EXPECT_NEAR(d.score, 1.f, 1e-3); + EXPECT_NEAR(d.box[0] + d.box[2] / 2, anchors[i][0], 1e-5); // centre = anchor + EXPECT_NEAR(d.box[2], 0.25f, 1e-5); // 32/128 + ASSERT_EQ(d.keypoints.size(), 6u); + EXPECT_NEAR(d.keypoints[0][0], anchors[i][0] + 4.f / 128.f, 1e-5); +} + +TEST(FaceCapGeom, WeightedNmsMergesOverlappingCluster) +{ + const auto anchors = genSsdAnchors(128, {8, 16, 16, 16}); + std::vector boxes(anchors.size() * 16, 0.f); + std::vector scores(anchors.size(), -10.f); + // two strongly overlapping detections on nearby anchors -> ONE merged out + for (size_t i : {200u, 201u}) { + scores[i] = 5.f; + boxes[i * 16 + 2] = 64.f; + boxes[i * 16 + 3] = 64.f; + } + const auto dets = + decodeDetections(boxes.data(), scores.data(), anchors, 128, 6); + EXPECT_EQ(dets.size(), 1u); +} + +TEST(FaceCapGeom, FaceRectIsSquareLongAndScaled) +{ + Detection det; + det.score = 1.f; + det.box = {0.4f, 0.4f, 0.2f, 0.1f}; // in a 100x100 image: 20 x 10 box + det.keypoints = {{0.45f, 0.45f}, {0.55f, 0.45f}}; // level eyes + const RotatedRect r = rectFromFaceDetection(det, 100, 100); + EXPECT_NEAR(r.cx, 50.f, 1e-3); + EXPECT_NEAR(r.cy, 45.f, 1e-3); + EXPECT_NEAR(r.w, 30.f, 1e-3); // max(20,10) * 1.5 + EXPECT_NEAR(r.h, 30.f, 1e-3); + EXPECT_NEAR(r.angle, 0.f, 1e-5); // level eyes -> no rotation +} + +TEST(FaceCapGeom, FaceRectRotationFollowsTheEyeLine) +{ + Detection det; + det.score = 1.f; + det.box = {0.4f, 0.4f, 0.2f, 0.2f}; + // left eye 45 degrees BELOW the right eye in image coords (y down) + det.keypoints = {{0.45f, 0.45f}, {0.55f, 0.55f}}; + const RotatedRect r = rectFromFaceDetection(det, 100, 100); + EXPECT_NEAR(r.angle, static_cast(M_PI) / 4.f, 1e-4); +} + +TEST(FaceCapGeom, CropIdentityRoundTrip) +{ + // a gradient image cropped with an axis-aligned rect the size of the + // image reproduces the image + QImage img(64, 64, QImage::Format_RGB888); + for (int y = 0; y < 64; ++y) + for (int x = 0; x < 64; ++x) + img.setPixel(x, y, qRgb(x * 4, y * 4, 0)); + RotatedRect rect; + rect.cx = 32.f; + rect.cy = 32.f; + rect.w = 64.f; + rect.h = 64.f; + rect.angle = 0.f; + std::vector tensor(64 * 64 * 3); + cropRotatedRectToTensor(img, rect, 64, 0.f, 1.f, tensor.data()); + for (int i : {5, 20, 40, 60}) { + EXPECT_NEAR(tensor[(i * 64 + i) * 3 + 0], i * 4 / 255.f, 0.03f); + EXPECT_NEAR(tensor[(i * 64 + i) * 3 + 1], i * 4 / 255.f, 0.03f); + } +} + +TEST(FaceCapGeom, ProjectLandmarksInvertsTheCropMapping) +{ + RotatedRect rect; + rect.cx = 100.f; + rect.cy = 80.f; + rect.w = 50.f; + rect.h = 50.f; + rect.angle = 0.5f; + + // crop-space corners + centre with z + std::vector pts = { + 0.5f, 0.5f, 2.f, // centre + 0.f, 0.f, 0.f, // top-left + 1.f, 1.f, 0.f, // bottom-right + }; + projectLandmarks(pts.data(), 3, 3, rect); + EXPECT_NEAR(pts[0], 100.f, 1e-4); // centre -> rect centre + EXPECT_NEAR(pts[1], 80.f, 1e-4); + EXPECT_NEAR(pts[2], 100.f, 1e-4); // z scaled by rect.w + // the two corners are diagonal through the centre: their midpoint is it + EXPECT_NEAR((pts[3] + pts[6]) / 2.f, 100.f, 1e-3); + EXPECT_NEAR((pts[4] + pts[7]) / 2.f, 80.f, 1e-3); + // and the diagonal length is |(w, h)| rotated = sqrt(50^2 + 50^2) + const float dx = pts[6] - pts[3]; + const float dy = pts[7] - pts[4]; + EXPECT_NEAR(std::sqrt(dx * dx + dy * dy), std::sqrt(2.f) * 50.f, 1e-2); +} + +TEST(FaceCapGeom, PoseRectFromAlignmentKeypoints) +{ + Detection det; + det.score = 1.f; + det.box = {0.f, 0.f, 1.f, 1.f}; + // mid-hip at centre, alignment point straight ABOVE it (y up in image = + // smaller y): target angle 90 deg means this is rotation 0 + det.keypoints = {{0.5f, 0.5f}, {0.5f, 0.3f}}; + const RotatedRect r = rectFromPoseDetection(det, 100, 100); + EXPECT_NEAR(r.cx, 50.f, 1e-3); + EXPECT_NEAR(r.cy, 50.f, 1e-3); + EXPECT_NEAR(r.w, 2.f * 20.f * 1.25f, 1e-3); // 2 * radius * 1.25 + EXPECT_NEAR(r.angle, 0.f, 1e-4); +} + +#endif // ENABLE_MOCAP diff --git a/src/Mocap/FaceCapMapper.cpp b/src/Mocap/FaceCapMapper.cpp new file mode 100644 index 000000000..fa33f1e44 --- /dev/null +++ b/src/Mocap/FaceCapMapper.cpp @@ -0,0 +1,164 @@ +#include "FaceCapMapper.h" + +#include "FaceCapCanonicalData.h" + +#include +#include +#include +#include +#include + +namespace FaceCapMapper { + +namespace { + +const QHash& aliasTable() +{ + // normalized third-party name -> normalized canonical name. Only entries + // the side-expansion + normalization pass can NOT already resolve. + static const QHash table = { + // CC/iClone + {QStringLiteral("mouthopen"), QStringLiteral("jawopen")}, + {QStringLiteral("browraiseinnerleft"), QStringLiteral("browinnerup")}, + {QStringLiteral("browraiseinnerright"), QStringLiteral("browinnerup")}, + {QStringLiteral("browraiseouterleft"), QStringLiteral("browouterupleft")}, + {QStringLiteral("browraiseouterright"), QStringLiteral("browouterupright")}, + {QStringLiteral("browdropleft"), QStringLiteral("browdownleft")}, + {QStringLiteral("browdropright"), QStringLiteral("browdownright")}, + {QStringLiteral("eyeswideleft"), QStringLiteral("eyewideleft")}, + {QStringLiteral("eyeswideright"), QStringLiteral("eyewideright")}, + // common shorthand + {QStringLiteral("blinkleft"), QStringLiteral("eyeblinkleft")}, + {QStringLiteral("blinkright"), QStringLiteral("eyeblinkright")}, + {QStringLiteral("smileleft"), QStringLiteral("mouthsmileleft")}, + {QStringLiteral("smileright"), QStringLiteral("mouthsmileright")}, + }; + return table; +} + +// "Mouth_Smile_L" -> "mouth smile left" token stream -> normalized join. +QString expandSideTokens(const QString& name) +{ + // split on separator characters AND camelCase boundaries + QStringList tokens; + QString current; + for (const QChar c : name) { + if (c == QLatin1Char('_') || c == QLatin1Char('-') + || c == QLatin1Char('.') || c.isSpace()) { + if (!current.isEmpty()) tokens << current; + current.clear(); + } else if (c.isUpper() && !current.isEmpty() + && current.back().isLower()) { + tokens << current; + current = c; + } else { + current += c; + } + } + if (!current.isEmpty()) tokens << current; + if (!tokens.isEmpty()) { + const QString last = tokens.last().toLower(); + if (last == QLatin1String("l")) + tokens.last() = QStringLiteral("left"); + else if (last == QLatin1String("r")) + tokens.last() = QStringLiteral("right"); + } + return tokens.join(QString()).toLower(); +} + +} // namespace + +QString normalizedName(const QString& name) +{ + return expandSideTokens(name); +} + +Mapping build(const QStringList& meshTargetNames, const QString& overrideJsonPath) +{ + Mapping mapping; + + QHash overrideMap; // canonical name -> mesh name + QSet ignore; + if (!overrideJsonPath.isEmpty()) { + QFile f(overrideJsonPath); + if (!f.open(QIODevice::ReadOnly)) { + mapping.error = QStringLiteral("cannot open mapping override: %1") + .arg(overrideJsonPath); + } else { + QJsonParseError parseError; + const QJsonDocument doc = QJsonDocument::fromJson(f.readAll(), &parseError); + if (doc.isNull()) { + mapping.error = QStringLiteral("mapping override parse error: %1") + .arg(parseError.errorString()); + } else { + const QJsonObject root = doc.object(); + const QJsonObject map = root.value(QLatin1String("map")).toObject(); + for (auto it = map.begin(); it != map.end(); ++it) + overrideMap.insert(it.key(), it.value().toString()); + const QJsonArray ign = root.value(QLatin1String("ignore")).toArray(); + for (const auto& v : ign) + ignore.insert(v.toString()); + } + } + } + + // normalized mesh name -> exact mesh name (first wins on collision) + QHash meshByNorm; + for (const QString& mesh : meshTargetNames) { + const QString norm = normalizedName(mesh); + if (!meshByNorm.contains(norm)) + meshByNorm.insert(norm, mesh); + } + + QSet usedMeshTargets; + for (int i = 0; i < FaceCap::kBlendshapeCount; ++i) { + const QString canonical = QString::fromLatin1(FaceCap::kBlendshapeNames[i]); + if (canonical == QLatin1String("_neutral")) + continue; // rest weight; no mesh equivalent by design + if (ignore.contains(canonical)) { + mapping.ignored << canonical; + continue; + } + QString meshName; + if (overrideMap.contains(canonical)) { + const QString target = overrideMap.value(canonical); + if (meshTargetNames.contains(target)) { + meshName = target; + } else { + mapping.error = mapping.error.isEmpty() + ? QStringLiteral("override target '%1' (for %2) not on mesh") + .arg(target, canonical) + : mapping.error; + } + } + if (meshName.isEmpty()) { + const QString norm = normalizedName(canonical); + meshName = meshByNorm.value(norm); + } + if (meshName.isEmpty()) { + // alias table: some mesh naming convention that normalizes to a + // different string but means this canonical channel + const QString canonNorm = normalizedName(canonical); + for (auto it = meshByNorm.begin(); it != meshByNorm.end(); ++it) { + if (aliasTable().value(it.key()) == canonNorm) { + meshName = it.value(); + break; + } + } + } + if (meshName.isEmpty()) { + mapping.unmatchedCanonical << canonical; + } else { + mapping.channels.append({i, meshName}); + usedMeshTargets.insert(meshName); + } + } + + for (const QString& mesh : meshTargetNames) + if (!usedMeshTargets.contains(mesh)) + mapping.unmatchedMesh << mesh; + + return mapping; +} + +} // namespace FaceCapMapper diff --git a/src/Mocap/FaceCapMapper.h b/src/Mocap/FaceCapMapper.h new file mode 100644 index 000000000..9bd8d93b5 --- /dev/null +++ b/src/Mocap/FaceCapMapper.h @@ -0,0 +1,53 @@ +#ifndef FACECAPMAPPER_H +#define FACECAPMAPPER_H + +// Maps the 52 canonical MediaPipe/ARKit blendshape names onto the target +// mesh's actual morph-target names (epic #869, Slice C #872). Pure data — +// QtCore only — headless-tested. +// +// Matching, in priority order per canonical channel: +// 1. explicit override from the JSON sidecar ("map"), +// 2. normalized name equality: lowercase, separators ('_', '-', '.', ' ') +// stripped, with trailing side tokens expanded first so "Mouth_Smile_L", +// "mouthSmile.R" and "MouthSmileLeft" all normalize the same way, +// 3. alias table for common third-party conventions. +// +// Override JSON sidecar format: +// { +// "map": { "jawOpen": "MyJawTarget", ... }, // canonical -> mesh name +// "ignore": [ "tongueOut", ... ] // canonical names to skip +// } +// +// Unmatched channels are REPORTED (unmatchedCanonical / unmatchedMesh), +// never silently dropped. "_neutral" (index 0) is a rest-weight channel with +// no mesh equivalent — it is excluded from mapping and reporting. + +#include +#include + +namespace FaceCapMapper { + +struct Channel { + int canonicalIndex = -1; // into FaceCap::kBlendshapeNames + QString meshTargetName; // exact morph-target name on the mesh +}; + +struct Mapping { + QList channels; + QStringList unmatchedCanonical; // canonical names with no mesh target + QStringList unmatchedMesh; // mesh targets no channel drives + QStringList ignored; // canonical names ignored via override + QString error; // override-file problem (mapping still built) + + bool isEmpty() const { return channels.isEmpty(); } +}; + +Mapping build(const QStringList& meshTargetNames, + const QString& overrideJsonPath = {}); + +// exposed for tests: "Mouth_Smile_L" -> "mouthsmileleft" +QString normalizedName(const QString& name); + +} // namespace FaceCapMapper + +#endif // FACECAPMAPPER_H diff --git a/src/Mocap/FaceCapMapper_test.cpp b/src/Mocap/FaceCapMapper_test.cpp new file mode 100644 index 000000000..2f69590e0 --- /dev/null +++ b/src/Mocap/FaceCapMapper_test.cpp @@ -0,0 +1,125 @@ +#ifdef ENABLE_MOCAP + +#include + +#include +#include +#include +#include +#include + +#include "Mocap/FaceCapCanonicalData.h" +#include "Mocap/FaceCapMapper.h" + +namespace { + +QString findMesh(const FaceCapMapper::Mapping& m, const QString& canonical) +{ + for (const auto& ch : m.channels) { + if (QString::fromLatin1(FaceCap::kBlendshapeNames[ch.canonicalIndex]) + == canonical) + return ch.meshTargetName; + } + return {}; +} + +} // namespace + +TEST(FaceCapMapper, ExactArkitNamesAllMatch) +{ + QStringList mesh; + for (int i = 1; i < FaceCap::kBlendshapeCount; ++i) // skip _neutral + mesh << QString::fromLatin1(FaceCap::kBlendshapeNames[i]); + const auto m = FaceCapMapper::build(mesh); + EXPECT_EQ(m.channels.size(), FaceCap::kBlendshapeCount - 1); + EXPECT_TRUE(m.unmatchedCanonical.isEmpty()) + << m.unmatchedCanonical.join(", ").toStdString(); + EXPECT_TRUE(m.unmatchedMesh.isEmpty()); +} + +TEST(FaceCapMapper, NormalizedMatchingHandlesSeparatorsAndSides) +{ + const QStringList mesh{ + QStringLiteral("Jaw_Open"), // CC style + QStringLiteral("mouth_smile_l"), // snake + side letter + QStringLiteral("MouthSmile.R"), // dot side + QStringLiteral("eye-blink-left"), // kebab + QStringLiteral("EyeBlink_R"), + }; + const auto m = FaceCapMapper::build(mesh); + EXPECT_EQ(findMesh(m, QStringLiteral("jawOpen")), QStringLiteral("Jaw_Open")); + EXPECT_EQ(findMesh(m, QStringLiteral("mouthSmileLeft")), + QStringLiteral("mouth_smile_l")); + EXPECT_EQ(findMesh(m, QStringLiteral("mouthSmileRight")), + QStringLiteral("MouthSmile.R")); + EXPECT_EQ(findMesh(m, QStringLiteral("eyeBlinkLeft")), + QStringLiteral("eye-blink-left")); + EXPECT_EQ(findMesh(m, QStringLiteral("eyeBlinkRight")), + QStringLiteral("EyeBlink_R")); + EXPECT_TRUE(m.unmatchedMesh.isEmpty()); +} + +TEST(FaceCapMapper, AliasTableResolvesCommonConventions) +{ + const QStringList mesh{QStringLiteral("Blink_L"), QStringLiteral("MouthOpen")}; + const auto m = FaceCapMapper::build(mesh); + EXPECT_EQ(findMesh(m, QStringLiteral("eyeBlinkLeft")), QStringLiteral("Blink_L")); + EXPECT_EQ(findMesh(m, QStringLiteral("jawOpen")), QStringLiteral("MouthOpen")); +} + +TEST(FaceCapMapper, UnmatchedAreReportedNeverDropped) +{ + const QStringList mesh{QStringLiteral("jawOpen"), QStringLiteral("SomeCustomTarget")}; + const auto m = FaceCapMapper::build(mesh); + EXPECT_EQ(m.channels.size(), 1); + // 50 canonical channels unmatched (52 - _neutral - jawOpen) + EXPECT_EQ(m.unmatchedCanonical.size(), FaceCap::kBlendshapeCount - 2); + ASSERT_EQ(m.unmatchedMesh.size(), 1); + EXPECT_EQ(m.unmatchedMesh.first(), QStringLiteral("SomeCustomTarget")); + // _neutral is by-design absent from both lists + EXPECT_FALSE(m.unmatchedCanonical.contains(QStringLiteral("_neutral"))); +} + +TEST(FaceCapMapper, OverrideMapAndIgnore) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QString overridePath = tmp.path() + QStringLiteral("/map.json"); + { + QJsonObject map; + map.insert(QStringLiteral("jawOpen"), QStringLiteral("WeirdJaw")); + QJsonObject root; + root.insert(QStringLiteral("map"), map); + root.insert(QStringLiteral("ignore"), + QJsonArray{QStringLiteral("cheekPuff")}); + QFile f(overridePath); + ASSERT_TRUE(f.open(QIODevice::WriteOnly)); + f.write(QJsonDocument(root).toJson()); + } + const QStringList mesh{QStringLiteral("WeirdJaw"), QStringLiteral("cheekPuff")}; + const auto m = FaceCapMapper::build(mesh, overridePath); + EXPECT_EQ(findMesh(m, QStringLiteral("jawOpen")), QStringLiteral("WeirdJaw")); + // ignored channel is not mapped even though the mesh name would match + EXPECT_TRUE(findMesh(m, QStringLiteral("cheekPuff")).isEmpty()); + EXPECT_TRUE(m.ignored.contains(QStringLiteral("cheekPuff"))); + EXPECT_FALSE(m.unmatchedCanonical.contains(QStringLiteral("cheekPuff"))); + EXPECT_TRUE(m.error.isEmpty()) << m.error.toStdString(); +} + +TEST(FaceCapMapper, BadOverridePathReportsErrorButStillMaps) +{ + const QStringList mesh{QStringLiteral("jawOpen")}; + const auto m = + FaceCapMapper::build(mesh, QStringLiteral("/nonexistent/map.json")); + EXPECT_FALSE(m.error.isEmpty()); + EXPECT_EQ(m.channels.size(), 1); +} + +TEST(FaceCapMapper, EmptyMeshTargetList) +{ + const auto m = FaceCapMapper::build({}); + EXPECT_TRUE(m.channels.isEmpty()); + EXPECT_EQ(m.unmatchedCanonical.size(), FaceCap::kBlendshapeCount - 1); +} + +#endif // ENABLE_MOCAP diff --git a/src/Mocap/FaceCapPose.cpp b/src/Mocap/FaceCapPose.cpp new file mode 100644 index 000000000..951423eb3 --- /dev/null +++ b/src/Mocap/FaceCapPose.cpp @@ -0,0 +1,186 @@ +#include "FaceCapPose.h" + +#include "FaceCapCanonicalData.h" + +#include + +namespace FaceCapPose { + +namespace { + +// Largest-eigenvalue eigenvector of a symmetric 4x4 matrix via cyclic Jacobi. +// Deterministic, ~1e-12 accurate after a handful of sweeps. +std::array maxEigenvector4(double m[4][4]) +{ + double v[4][4] = {{1, 0, 0, 0}, {0, 1, 0, 0}, {0, 0, 1, 0}, {0, 0, 0, 1}}; + for (int sweep = 0; sweep < 32; ++sweep) { + double off = 0.0; + for (int p = 0; p < 4; ++p) + for (int q = p + 1; q < 4; ++q) + off += m[p][q] * m[p][q]; + if (off < 1e-24) + break; + for (int p = 0; p < 4; ++p) { + for (int q = p + 1; q < 4; ++q) { + if (std::abs(m[p][q]) < 1e-30) + continue; + const double theta = (m[q][q] - m[p][p]) / (2.0 * m[p][q]); + const double t = (theta >= 0 ? 1.0 : -1.0) + / (std::abs(theta) + std::sqrt(theta * theta + 1.0)); + const double c = 1.0 / std::sqrt(t * t + 1.0); + const double s = t * c; + for (int k = 0; k < 4; ++k) { + const double mkp = m[k][p], mkq = m[k][q]; + m[k][p] = c * mkp - s * mkq; + m[k][q] = s * mkp + c * mkq; + } + for (int k = 0; k < 4; ++k) { + const double mpk = m[p][k], mqk = m[q][k]; + m[p][k] = c * mpk - s * mqk; + m[q][k] = s * mpk + c * mqk; + } + for (int k = 0; k < 4; ++k) { + const double vkp = v[k][p], vkq = v[k][q]; + v[k][p] = c * vkp - s * vkq; + v[k][q] = s * vkp + c * vkq; + } + } + } + } + int best = 0; + for (int i = 1; i < 4; ++i) + if (m[i][i] > m[best][best]) + best = i; + return {v[0][best], v[1][best], v[2][best], v[3][best]}; +} + +void rotate(const std::array& q, const double in[3], double out[3]) +{ + // (x,y,z,w) quaternion rotation of a vector + const double x = q[0], y = q[1], z = q[2], w = q[3]; + const double tx = 2.0 * (y * in[2] - z * in[1]); + const double ty = 2.0 * (z * in[0] - x * in[2]); + const double tz = 2.0 * (x * in[1] - y * in[0]); + out[0] = in[0] + w * tx + (y * tz - z * ty); + out[1] = in[1] + w * ty + (z * tx - x * tz); + out[2] = in[2] + w * tz + (x * ty - y * tx); +} + +} // namespace + +Result solve(const float* src, const float* dst, const float* weights, int count) +{ + Result r; + if (count < 3) + return r; + + double wSum = 0.0; + double muS[3] = {0, 0, 0}, muD[3] = {0, 0, 0}; + for (int i = 0; i < count; ++i) { + const double w = weights ? weights[i] : 1.0; + if (w <= 0.0) + continue; + wSum += w; + for (int k = 0; k < 3; ++k) { + muS[k] += w * src[i * 3 + k]; + muD[k] += w * dst[i * 3 + k]; + } + } + if (wSum <= 0.0) + return r; + for (int k = 0; k < 3; ++k) { + muS[k] /= wSum; + muD[k] /= wSum; + } + + // weighted covariance src x dst (Horn's S = sum s.d^T) + source variance + // for the scale. Order matters: S[a][b] = s[a]*d[b] yields the rotation + // taking SRC into DST; the transpose yields its inverse. + double cov[3][3] = {{0}}; + double varS = 0.0; + for (int i = 0; i < count; ++i) { + const double w = weights ? weights[i] : 1.0; + if (w <= 0.0) + continue; + double s[3], d[3]; + for (int k = 0; k < 3; ++k) { + s[k] = src[i * 3 + k] - muS[k]; + d[k] = dst[i * 3 + k] - muD[k]; + } + for (int a = 0; a < 3; ++a) + for (int b = 0; b < 3; ++b) + cov[a][b] += w * s[a] * d[b]; + varS += w * (s[0] * s[0] + s[1] * s[1] + s[2] * s[2]); + } + if (varS <= 0.0) + return r; + + // Horn's method: max eigenvector of the 4x4 built from the covariance + const double sxx = cov[0][0], sxy = cov[0][1], sxz = cov[0][2]; + const double syx = cov[1][0], syy = cov[1][1], syz = cov[1][2]; + const double szx = cov[2][0], szy = cov[2][1], szz = cov[2][2]; + double n[4][4] = { + {sxx + syy + szz, syz - szy, szx - sxz, sxy - syx}, + {syz - szy, sxx - syy - szz, sxy + syx, szx + sxz}, + {szx - sxz, sxy + syx, -sxx + syy - szz, syz + szy}, + {sxy - syx, szx + sxz, syz + szy, -sxx - syy + szz}, + }; + const std::array e = maxEigenvector4(n); // (w, x, y, z) + double norm = std::sqrt(e[0] * e[0] + e[1] * e[1] + e[2] * e[2] + e[3] * e[3]); + if (norm <= 0.0) + return r; + // store as (x,y,z,w), w kept positive for a canonical representation + const double sign = e[0] >= 0.0 ? 1.0 : -1.0; + r.rotation = {static_cast(sign * e[1] / norm), + static_cast(sign * e[2] / norm), + static_cast(sign * e[3] / norm), + static_cast(sign * e[0] / norm)}; + + // scale = sum(w * d . (R s)) / sum(w * |s|^2) + double dot = 0.0; + for (int i = 0; i < count; ++i) { + const double w = weights ? weights[i] : 1.0; + if (w <= 0.0) + continue; + double s[3], rs[3], d[3]; + for (int k = 0; k < 3; ++k) { + s[k] = src[i * 3 + k] - muS[k]; + d[k] = dst[i * 3 + k] - muD[k]; + } + rotate(r.rotation, s, rs); + dot += w * (d[0] * rs[0] + d[1] * rs[1] + d[2] * rs[2]); + } + r.scale = static_cast(dot / varS); + + // t = muD - scale * R * muS (scale participates: the fit is a similarity; + // the caller treats rotation as the pose and translation as informational) + double rMu[3]; + rotate(r.rotation, muS, rMu); + for (int k = 0; k < 3; ++k) + r.translation[k] = static_cast(muD[k] - r.scale * rMu[k]); + r.ok = true; + return r; +} + +Result solveHeadPose(const float* landmarksXyz, int landmarkCount) +{ + if (landmarkCount < FaceCap::kCanonicalVertexCount) + return {}; + // gather the weighted Procrustes basis subset in both frames + float src[FaceCap::kProcrustesBasisCount * 3]; + float dst[FaceCap::kProcrustesBasisCount * 3]; + float wgt[FaceCap::kProcrustesBasisCount]; + for (int i = 0; i < FaceCap::kProcrustesBasisCount; ++i) { + const int id = FaceCap::kProcrustesLandmarkIds[i]; + for (int k = 0; k < 3; ++k) + src[i * 3 + k] = FaceCap::kCanonicalFaceModel[id * 3 + k]; + // image frame -> pose frame: +X right stays, y and z flip + dst[i * 3 + 0] = landmarksXyz[id * 3 + 0]; + dst[i * 3 + 1] = -landmarksXyz[id * 3 + 1]; + dst[i * 3 + 2] = -landmarksXyz[id * 3 + 2]; + wgt[i] = FaceCap::kProcrustesWeights[i]; + } + return solve(src, dst, wgt, FaceCap::kProcrustesBasisCount); +} + +} // namespace FaceCapPose diff --git a/src/Mocap/FaceCapPose.h b/src/Mocap/FaceCapPose.h new file mode 100644 index 000000000..279d1607c --- /dev/null +++ b/src/Mocap/FaceCapPose.h @@ -0,0 +1,46 @@ +#ifndef FACECAPPOSE_H +#define FACECAPPOSE_H + +// Head-pose solve for face capture (epic #869, Slice C #872). Pure data — +// no Qt/Ogre/ONNX — headless-tested. +// +// Weighted rigid fit (Horn's closed-form quaternion method — the weighted +// Kabsch/Umeyama equivalent, solved with a 4x4 Jacobi eigensolver so no +// linear-algebra dependency is needed) of the MediaPipe canonical face model +// onto the detected 3D screen landmarks. Scale is solved and DISCARDED for +// the pose; translation is returned in the destination units. +// +// Convention (documented contract): the destination frame is +X right, +// +Y up, camera looking down -Z; identity rotation = the face looking +// straight at the camera. Callers build destination points from screen +// landmarks as (x_px, -y_px, -z_px) — image y grows downward and the +// landmark z grows away from the camera, so both flip. This intentionally +// replaces MediaPipe's perspective geometry pipeline (delta vs its facial +// transformation matrix measured at 4-9 degrees, systematic; the recorder +// neutral-calibrates the first confident frame anyway). See docs/MOCAP_SPIKE.md. + +#include + +namespace FaceCapPose { + +struct Result { + std::array rotation{0.f, 0.f, 0.f, 1.f}; // (x,y,z,w), src -> dst + std::array translation{0.f, 0.f, 0.f}; // dst units + float scale = 1.f; // solved, informational + bool ok = false; +}; + +// General weighted rigid fit: dst ~= scale * R * src + t. +// src/dst are count x 3 (xyz interleaved); weights length count (>= 0, at +// least 3 non-zero non-collinear points required). +Result solve(const float* src, const float* dst, const float* weights, int count); + +// Face-capture convenience: fits the embedded canonical face model +// (FaceCapCanonicalData.h, MediaPipe Procrustes basis weights) onto 478 (or +// 468) screen landmarks given as xyz triples in PIXELS (image coords, y down, +// z from the landmark model). Applies the (x, -y, -z) frame flip internally. +Result solveHeadPose(const float* landmarksXyz, int landmarkCount); + +} // namespace FaceCapPose + +#endif // FACECAPPOSE_H diff --git a/src/Mocap/FaceCapPose_test.cpp b/src/Mocap/FaceCapPose_test.cpp new file mode 100644 index 000000000..6d6b7e267 --- /dev/null +++ b/src/Mocap/FaceCapPose_test.cpp @@ -0,0 +1,165 @@ +#ifdef ENABLE_MOCAP + +#include + +#include +#include +#include + +#include "Mocap/FaceCapCanonicalData.h" +#include "Mocap/FaceCapPose.h" + +namespace { + +using Quat = std::array; // (x,y,z,w) + +Quat axisAngle(float ax, float ay, float az, float rad) +{ + const float n = std::sqrt(ax * ax + ay * ay + az * az); + const float s = std::sin(rad / 2) / n; + return {ax * s, ay * s, az * s, std::cos(rad / 2)}; +} + +void rotate(const Quat& q, const float in[3], float out[3]) +{ + const float x = q[0], y = q[1], z = q[2], w = q[3]; + const float tx = 2 * (y * in[2] - z * in[1]); + const float ty = 2 * (z * in[0] - x * in[2]); + const float tz = 2 * (x * in[1] - y * in[0]); + out[0] = in[0] + w * tx + (y * tz - z * ty); + out[1] = in[1] + w * ty + (z * tx - x * tz); + out[2] = in[2] + w * tz + (x * ty - y * tx); +} + +double quatAngle(const Quat& a, const Quat& b) +{ + double dot = 0; + for (int i = 0; i < 4; ++i) + dot += a[i] * b[i]; + return 2.0 * std::acos(std::fmin(1.0, std::fabs(dot))); +} + +// asymmetric, non-coplanar point cloud +std::vector makeCloud() +{ + return {0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 1, 1, 0.5f, -2, 0.5f, 1}; +} + +} // namespace + +TEST(FaceCapPose, RecoversSyntheticRigidTransform) +{ + const std::vector src = makeCloud(); + const int n = static_cast(src.size() / 3); + const Quat q = axisAngle(0.3f, 1.f, 0.2f, 0.7f); + const float t[3] = {5.f, -3.f, 2.f}; + + std::vector dst(src.size()); + for (int i = 0; i < n; ++i) { + float r[3]; + rotate(q, &src[i * 3], r); + for (int k = 0; k < 3; ++k) + dst[i * 3 + k] = r[k] + t[k]; + } + const auto res = FaceCapPose::solve(src.data(), dst.data(), nullptr, n); + ASSERT_TRUE(res.ok); + EXPECT_LT(quatAngle(res.rotation, q), 1e-3); + EXPECT_NEAR(res.scale, 1.f, 1e-4); + for (int k = 0; k < 3; ++k) + EXPECT_NEAR(res.translation[k], t[k], 1e-3); +} + +TEST(FaceCapPose, RecoversScaleAndDiscardsItFromRotation) +{ + const std::vector src = makeCloud(); + const int n = static_cast(src.size() / 3); + const Quat q = axisAngle(0.f, 1.f, 0.f, -0.4f); + + std::vector dst(src.size()); + for (int i = 0; i < n; ++i) { + float r[3]; + rotate(q, &src[i * 3], r); + for (int k = 0; k < 3; ++k) + dst[i * 3 + k] = 2.5f * r[k] + 1.f; + } + const auto res = FaceCapPose::solve(src.data(), dst.data(), nullptr, n); + ASSERT_TRUE(res.ok); + EXPECT_LT(quatAngle(res.rotation, q), 1e-3); + EXPECT_NEAR(res.scale, 2.5f, 1e-3); +} + +TEST(FaceCapPose, WeightsFocusTheFit) +{ + const std::vector src = makeCloud(); + const int n = static_cast(src.size() / 3); + const Quat q = axisAngle(0.f, 0.f, 1.f, 0.5f); + + std::vector dst(src.size()); + for (int i = 0; i < n; ++i) { + float r[3]; + rotate(q, &src[i * 3], r); + for (int k = 0; k < 3; ++k) + dst[i * 3 + k] = r[k]; + } + // corrupt the last point badly, but give it zero weight + dst[(n - 1) * 3 + 0] += 100.f; + std::vector w(n, 1.f); + w[n - 1] = 0.f; + const auto res = FaceCapPose::solve(src.data(), dst.data(), w.data(), n); + ASSERT_TRUE(res.ok); + EXPECT_LT(quatAngle(res.rotation, q), 1e-3); +} + +TEST(FaceCapPose, DegenerateInputsReturnNotOk) +{ + const std::vector src = makeCloud(); + EXPECT_FALSE(FaceCapPose::solve(src.data(), src.data(), nullptr, 2).ok); + // all-zero weights + std::vector w(src.size() / 3, 0.f); + EXPECT_FALSE( + FaceCapPose::solve(src.data(), src.data(), w.data(), + static_cast(src.size() / 3)).ok); + // coincident points (zero variance) + std::vector same(src.size(), 1.f); + EXPECT_FALSE(FaceCapPose::solve(same.data(), same.data(), nullptr, + static_cast(same.size() / 3)).ok); +} + +TEST(FaceCapPose, HeadPoseIdentityForCanonicalLandmarks) +{ + // feed the canonical model itself as "screen landmarks" with the + // documented (x, -y, -z) flip already inverted: identity comes back + std::vector landmarks(FaceCap::kCanonicalVertexCount * 3); + for (int i = 0; i < FaceCap::kCanonicalVertexCount; ++i) { + landmarks[i * 3 + 0] = FaceCap::kCanonicalFaceModel[i * 3 + 0]; + landmarks[i * 3 + 1] = -FaceCap::kCanonicalFaceModel[i * 3 + 1]; + landmarks[i * 3 + 2] = -FaceCap::kCanonicalFaceModel[i * 3 + 2]; + } + const auto res = FaceCapPose::solveHeadPose( + landmarks.data(), FaceCap::kCanonicalVertexCount); + ASSERT_TRUE(res.ok); + const Quat identity{0.f, 0.f, 0.f, 1.f}; + EXPECT_LT(quatAngle(res.rotation, identity), 1e-3); + EXPECT_NEAR(res.scale, 1.f, 1e-4); +} + +TEST(FaceCapPose, HeadPoseRecoversYaw) +{ + // rotate the canonical model 30 degrees about +Y (turn left), project to + // the image frame (y,z flip), expect the same rotation back + const Quat q = axisAngle(0.f, 1.f, 0.f, static_cast(M_PI) / 6.f); + std::vector landmarks(FaceCap::kCanonicalVertexCount * 3); + for (int i = 0; i < FaceCap::kCanonicalVertexCount; ++i) { + float r[3]; + rotate(q, &FaceCap::kCanonicalFaceModel[i * 3], r); + landmarks[i * 3 + 0] = r[0]; + landmarks[i * 3 + 1] = -r[1]; + landmarks[i * 3 + 2] = -r[2]; + } + const auto res = FaceCapPose::solveHeadPose( + landmarks.data(), FaceCap::kCanonicalVertexCount); + ASSERT_TRUE(res.ok); + EXPECT_LT(quatAngle(res.rotation, q), 1e-3); +} + +#endif // ENABLE_MOCAP diff --git a/src/Mocap/FaceCapPredictor.cpp b/src/Mocap/FaceCapPredictor.cpp new file mode 100644 index 000000000..8d59dffae --- /dev/null +++ b/src/Mocap/FaceCapPredictor.cpp @@ -0,0 +1,432 @@ +#ifdef ENABLE_MOCAP + +#include "FaceCapPredictor.h" + +#include "FaceCapCanonicalData.h" +#include "FaceCapGeom.h" +#include "FaceCapPose.h" +#include "../ModelDownloader.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef ENABLE_ONNX +#include +#endif + +namespace { + +constexpr const char* kDetectorFile = "face_detector.onnx"; +constexpr const char* kLandmarksFile = "face_landmarks.onnx"; +constexpr const char* kBlendshapesFile = "face_blendshapes.onnx"; +constexpr const char* kDefaultModelBaseUrl = + "https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/mocap/face/"; +constexpr const char* kBaseUrlSettingsKey = "ai/mocapModelBaseUrl"; + +constexpr float kPresenceThreshold = 0.5f; + +} // namespace + +// --------------------------------------------------------------------------- +// static model management +// --------------------------------------------------------------------------- + +QString FaceCapPredictor::modelDir() +{ + const QString dataPath = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + return QDir(dataPath).filePath(QStringLiteral("ai_models/mocap/face")); +} + +bool FaceCapPredictor::modelsPresent() +{ + const QDir dir(modelDir()); + return QFileInfo::exists(dir.filePath(QLatin1String(kDetectorFile))) + && QFileInfo::exists(dir.filePath(QLatin1String(kLandmarksFile))) + && QFileInfo::exists(dir.filePath(QLatin1String(kBlendshapesFile))); +} + +QString FaceCapPredictor::ensureModelsBlocking() +{ +#ifndef ENABLE_ONNX + return {}; +#else + if (modelsPresent()) + return modelDir(); + if (!qEnvironmentVariableIsEmpty("QTMESH_MOCAP_NO_DOWNLOAD")) + return {}; + + QString base; + { + QSettings s; + base = s.value(QString::fromLatin1(kBaseUrlSettingsKey)).toString(); + if (base.isEmpty()) { + const QByteArray env = qgetenv("QTMESH_MOCAP_MODEL_BASE_URL"); + base = env.isEmpty() ? QString::fromLatin1(kDefaultModelBaseUrl) + : QString::fromUtf8(env); + } + } + if (base.isEmpty()) + return {}; + if (!base.endsWith('/')) + base += '/'; + + auto* dl = ModelDownloader::instance(); + if (!dl) + return {}; + + const QDir dir(modelDir()); + auto downloadOne = [&](const char* fileName) -> bool { + const QString dest = dir.filePath(QLatin1String(fileName)); + if (QFileInfo::exists(dest)) + return true; + QDir().mkpath(QFileInfo(dest).absolutePath()); + const QString label = + QStringLiteral("Face capture model (%1)").arg(QLatin1String(fileName)); + QEventLoop loop; + bool ok = false, timedOut = false; + auto onDone = QObject::connect( + dl, &ModelDownloader::downloadCompleted, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = true; loop.quit(); } + }); + auto onErr = QObject::connect( + dl, &ModelDownloader::downloadError, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = false; loop.quit(); } + }); + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&timeout, &QTimer::timeout, &loop, + [&]() { timedOut = true; loop.quit(); }); + timeout.start(300000); // 5 min — the three graphs total ~6.4 MB + dl->startDownload(base + QLatin1String(fileName), dest, label); + loop.exec(); + QObject::disconnect(onDone); + QObject::disconnect(onErr); + if (timedOut && dl) + dl->cancelDownload(); + return ok && !timedOut && QFileInfo::exists(dest); + }; + + if (!downloadOne(kDetectorFile) || !downloadOne(kLandmarksFile) + || !downloadOne(kBlendshapesFile)) + return {}; + return modelDir(); +#endif +} + +// --------------------------------------------------------------------------- +// non-ONNX stub (ENABLE_MOCAP requires ENABLE_ONNX in CMake, but keep the +// TU compilable standalone) +// --------------------------------------------------------------------------- + +#ifndef ENABLE_ONNX + +struct FaceCapPredictor::Impl { + QString error = QStringLiteral( + "face capture predictor unavailable (built without ENABLE_ONNX)"); +}; + +FaceCapPredictor::FaceCapPredictor() : d(new Impl) {} +FaceCapPredictor::~FaceCapPredictor() = default; +bool FaceCapPredictor::load(const QString&) { return false; } +bool FaceCapPredictor::isAvailable() const { return false; } +QString FaceCapPredictor::lastError() const { return d->error; } +FaceSample FaceCapPredictor::predict(const QImage&, double timeSec) +{ + FaceSample s; + s.timeSec = timeSec; + return s; +} +void FaceCapPredictor::resetTracking() {} +int FaceCapPredictor::detectorRuns() const { return 0; } + +#else // ENABLE_ONNX + +struct FaceCapPredictor::Impl { + Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "qtmesh_facecap"}; + std::unique_ptr detector; + std::unique_ptr landmarks; + std::unique_ptr blendshapes; + std::vector> anchors; + int detectorSize = 128; + int landmarksSize = 256; + QString error; + bool available = false; + + // detector-skip tracking state + bool tracking = false; + FaceCapGeom::RotatedRect trackedRect; + int detectorRuns = 0; + + // scratch buffers (avoid per-frame allocation in live mode) + std::vector detInput; + std::vector lmkInput; + + std::unique_ptr openSession(const QString& path) + { + Ort::SessionOptions so; + so.SetIntraOpNumThreads(2); +#ifdef Q_OS_MACOS + // CoreML EP where it helps; CPU fallback is always registered. + try { + std::unordered_map opts; + so.AppendExecutionProvider("CoreML", opts); + } catch (const Ort::Exception&) {} +#endif +#ifdef Q_OS_WIN + const std::wstring wpath = path.toStdWString(); + return std::make_unique(env, wpath.c_str(), so); +#else + const QByteArray p = path.toUtf8(); + return std::make_unique(env, p.constData(), so); +#endif + } +}; + +FaceCapPredictor::FaceCapPredictor() : d(new Impl) {} +FaceCapPredictor::~FaceCapPredictor() = default; + +bool FaceCapPredictor::load(const QString& dirIn) +{ + const QDir dir(dirIn.isEmpty() ? modelDir() : dirIn); + const QString det = dir.filePath(QLatin1String(kDetectorFile)); + const QString lmk = dir.filePath(QLatin1String(kLandmarksFile)); + const QString bs = dir.filePath(QLatin1String(kBlendshapesFile)); + if (!QFileInfo::exists(det) || !QFileInfo::exists(lmk) + || !QFileInfo::exists(bs)) { + d->error = QStringLiteral( + "face capture models not found in %1 — they download on first " + "use, or set QTMESH_MOCAP_MODEL_BASE_URL").arg(dir.absolutePath()); + return false; + } + try { + d->detector = d->openSession(det); + d->landmarks = d->openSession(lmk); + d->blendshapes = d->openSession(bs); + + // runtime shape discovery — never hardcode graph tensor names + { + const auto info = d->detector->GetInputTypeInfo(0); + const auto shape = info.GetTensorTypeAndShapeInfo().GetShape(); + if (shape.size() == 4 && shape[1] > 0) + d->detectorSize = static_cast(shape[1]); + } + { + const auto info = d->landmarks->GetInputTypeInfo(0); + const auto shape = info.GetTensorTypeAndShapeInfo().GetShape(); + if (shape.size() == 4 && shape[1] > 0) + d->landmarksSize = static_cast(shape[1]); + } + d->anchors = FaceCapGeom::genSsdAnchors(d->detectorSize, {8, 16, 16, 16}); + { + const auto info = d->detector->GetOutputTypeInfo(0); + const auto shape = info.GetTensorTypeAndShapeInfo().GetShape(); + if (shape.size() == 3 + && shape[1] != static_cast(d->anchors.size())) { + d->error = QStringLiteral( + "face detector anchor count mismatch (model %1, ours %2)") + .arg(shape[1]).arg(d->anchors.size()); + d->available = false; + return false; + } + } + d->detInput.resize(static_cast(d->detectorSize) + * d->detectorSize * 3); + d->lmkInput.resize(static_cast(d->landmarksSize) + * d->landmarksSize * 3); + d->available = true; + d->error.clear(); + return true; + } catch (const Ort::Exception& e) { + d->error = QStringLiteral("failed to load face capture models: %1") + .arg(QString::fromUtf8(e.what())); + d->available = false; + return false; + } +} + +bool FaceCapPredictor::isAvailable() const { return d->available; } +QString FaceCapPredictor::lastError() const { return d->error; } +void FaceCapPredictor::resetTracking() { d->tracking = false; } +int FaceCapPredictor::detectorRuns() const { return d->detectorRuns; } + +FaceSample FaceCapPredictor::predict(const QImage& image, double timeSec) +{ + FaceSample sample; + sample.timeSec = timeSec; + if (!d->available) { + if (d->error.isEmpty()) + d->error = QStringLiteral("predictor not loaded — call load()"); + return sample; + } + QImage rgb = image.format() == QImage::Format_RGB888 + ? image + : image.convertToFormat(QImage::Format_RGB888); + const int W = rgb.width(); + const int H = rgb.height(); + if (W <= 0 || H <= 0) + return sample; + + try { + Ort::MemoryInfo mem = + Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + Ort::AllocatorWithDefaultOptions alloc; + + auto runSession = [&](Ort::Session* session, Ort::Value input) + -> std::vector { + const Ort::AllocatedStringPtr inName = + session->GetInputNameAllocated(0, alloc); + std::vector outHolders; + std::vector outNames; + const size_t n = session->GetOutputCount(); + outHolders.reserve(n); + for (size_t i = 0; i < n; ++i) { + outHolders.push_back(session->GetOutputNameAllocated(i, alloc)); + outNames.push_back(outHolders.back().get()); + } + const char* in = inName.get(); + return session->Run(Ort::RunOptions{nullptr}, &in, &input, 1, + outNames.data(), outNames.size()); + }; + + // --- stage 1: face ROI (tracked, or detector) + FaceCapGeom::RotatedRect rect; + if (d->tracking) { + rect = d->trackedRect; + } else { + const int ds = d->detectorSize; + FaceCapGeom::Letterbox lb = FaceCapGeom::letterboxToTensor( + rgb, ds, -1.f, 1.f, d->detInput.data()); + const std::array shape{1, ds, ds, 3}; + Ort::Value input = Ort::Value::CreateTensor( + mem, d->detInput.data(), d->detInput.size(), shape.data(), + shape.size()); + auto outs = runSession(d->detector.get(), std::move(input)); + ++d->detectorRuns; + + // identify regressors vs scores by the last dimension + const float* rawBoxes = nullptr; + const float* rawScores = nullptr; + for (auto& o : outs) { + const auto s = o.GetTensorTypeAndShapeInfo().GetShape(); + if (s.size() == 3 && s[2] > 1) + rawBoxes = o.GetTensorData(); + else if (s.size() == 3 && s[2] == 1) + rawScores = o.GetTensorData(); + } + if (!rawBoxes || !rawScores) { + d->error = QStringLiteral("unexpected face detector outputs"); + return sample; + } + auto dets = FaceCapGeom::decodeDetections( + rawBoxes, rawScores, d->anchors, ds, /*numKeypoints=*/6); + if (dets.empty()) + return sample; // confidence 0 — no face this frame + FaceCapGeom::Detection& best = dets.front(); + FaceCapGeom::unletterbox(lb, best.box[0], best.box[1]); + // box w/h scale by the content fraction + best.box[2] /= lb.fracX; + best.box[3] /= lb.fracY; + for (auto& kp : best.keypoints) + FaceCapGeom::unletterbox(lb, kp[0], kp[1]); + rect = FaceCapGeom::rectFromFaceDetection(best, W, H); + } + + // --- stage 2: landmarks + const int ls = d->landmarksSize; + FaceCapGeom::cropRotatedRectToTensor(rgb, rect, ls, 0.f, 1.f, + d->lmkInput.data()); + const std::array lshape{1, ls, ls, 3}; + Ort::Value linput = Ort::Value::CreateTensor( + mem, d->lmkInput.data(), d->lmkInput.size(), lshape.data(), + lshape.size()); + auto louts = runSession(d->landmarks.get(), std::move(linput)); + + const float* rawLandmarks = nullptr; + size_t landmarkFloats = 0; + float presenceLogit = 0.f; + bool presenceFound = false; + for (auto& o : louts) { + const auto info = o.GetTensorTypeAndShapeInfo(); + const size_t count = info.GetElementCount(); + if (count >= 468 * 3) { + rawLandmarks = o.GetTensorData(); + landmarkFloats = count; + } else if (count == 1 && !presenceFound + && info.GetShape().size() == 4) { + presenceLogit = o.GetTensorData()[0]; + presenceFound = true; + } + } + if (!rawLandmarks) { + d->error = QStringLiteral("unexpected face landmark outputs"); + return sample; + } + const float presence = + 1.f / (1.f + std::exp(-std::clamp(presenceLogit, -50.f, 50.f))); + if (presence < kPresenceThreshold) { + // lost the face: fall back to the detector next frame + if (d->tracking) { + d->tracking = false; + return predict(image, timeSec); + } + return sample; + } + + const int landmarkCount = static_cast(landmarkFloats / 3); + std::vector pts(rawLandmarks, rawLandmarks + landmarkFloats); + for (auto& v : pts) + v /= ls; // 256-space px -> normalized crop coords (z: /256 then *w) + FaceCapGeom::projectLandmarks(pts.data(), landmarkCount, 3, rect); + + // --- stage 3: blendshapes (146-subset pixel coords) + std::vector bsInput(FaceCap::kBlendshapeInputLandmarks * 2); + for (int i = 0; i < FaceCap::kBlendshapeInputLandmarks; ++i) { + const int id = FaceCap::kBlendshapeLandmarkSubset[i]; + bsInput[i * 2 + 0] = pts[id * 3 + 0]; + bsInput[i * 2 + 1] = pts[id * 3 + 1]; + } + const std::array bshape{1, FaceCap::kBlendshapeInputLandmarks, 2}; + Ort::Value binput = Ort::Value::CreateTensor( + mem, bsInput.data(), bsInput.size(), bshape.data(), bshape.size()); + auto bouts = runSession(d->blendshapes.get(), std::move(binput)); + const float* scores = bouts.front().GetTensorData(); + const size_t scoreCount = + bouts.front().GetTensorTypeAndShapeInfo().GetElementCount(); + for (size_t i = 0; i < sample.weights.size() && i < scoreCount; ++i) + sample.weights[i] = scores[i]; + + // --- head pose (weighted Kabsch vs the canonical face model) + const FaceCapPose::Result pose = + FaceCapPose::solveHeadPose(pts.data(), landmarkCount); + if (pose.ok) { + sample.headRotation = pose.rotation; + sample.headTranslation = pose.translation; + } + sample.confidence = presence; + + // --- next-frame ROI from these landmarks (detector-skip) + d->trackedRect = + FaceCapGeom::rectFromFaceLandmarks(pts.data(), landmarkCount, W, H); + d->tracking = d->trackedRect.w > 4.f && d->trackedRect.h > 4.f; + return sample; + } catch (const Ort::Exception& e) { + d->error = QStringLiteral("face capture inference failed: %1") + .arg(QString::fromUtf8(e.what())); + d->tracking = false; + return sample; + } +} + +#endif // ENABLE_ONNX +#endif // ENABLE_MOCAP diff --git a/src/Mocap/FaceCapPredictor.h b/src/Mocap/FaceCapPredictor.h new file mode 100644 index 000000000..8099df09d --- /dev/null +++ b/src/Mocap/FaceCapPredictor.h @@ -0,0 +1,70 @@ +#ifndef FACECAPPREDICTOR_H +#define FACECAPPREDICTOR_H + +// Face capture predictor (epic #869, Slice C #872) — ONNX consumer #9. +// One RGB888 frame in -> 52 blendshape weights + head pose + confidence out, +// via the three MediaPipe graphs converted by scripts/export-facecap-onnx.py +// (Apache-2.0; contract in docs/MOCAP_SPIKE.md): +// face_detector.onnx BlazeFace short-range (letterboxed 128, [-1,1]) +// face_landmarks.onnx Face Mesh V2 (rotated 256 crop, [0,1] -> 478 xyz) +// face_blendshapes.onnx MLP-Mixer (146-landmark subset px -> 52 scores) +// +// Detector-skip tracking: once a face is tracked, the next frame's ROI comes +// from the previous frame's landmarks and the detector is skipped until the +// landmark presence drops below threshold (MediaPipe's own strategy). +// +// Models download on first use to AppData/ai_models/mocap/face/ (base URL +// override: QSettings ai/mocapModelBaseUrl or QTMESH_MOCAP_MODEL_BASE_URL; +// offline guard: QTMESH_MOCAP_NO_DOWNLOAD). Without ENABLE_ONNX (implied by +// ENABLE_MOCAP) or without models, isAvailable() stays false and predict() +// returns confidence 0 with lastError() explaining why — never crashes. + +#ifdef ENABLE_MOCAP + +#include +#include + +#include +#include + +// One predicted sample. Pure data; shared vocabulary with the mapper, +// recorder and GUI is FaceCap::kBlendshapeNames (FaceCapCanonicalData.h). +struct FaceSample { + double timeSec = 0.0; + std::array weights{}; // [0,1] each + std::array headRotation{0.f, 0.f, 0.f, 1.f}; // (x,y,z,w); + // +X right, +Y up, camera looks -Z; identity = facing the camera + std::array headTranslation{0.f, 0.f, 0.f}; // px units + float confidence = 0.f; // 0 = no face +}; + +class FaceCapPredictor { +public: + FaceCapPredictor(); + ~FaceCapPredictor(); + + static QString modelDir(); // AppData/ai_models/mocap/face + static bool modelsPresent(); + // Blocking first-use download of the three graphs (~6.4 MB total). + // Returns the model dir, or empty on failure/offline-guard/non-ONNX. + static QString ensureModelsBlocking(); + + // Create the ONNX sessions from dir (default: modelDir()). + bool load(const QString& dir = {}); + bool isAvailable() const; + QString lastError() const; + + // rgb888: any QImage (converted internally if needed). + FaceSample predict(const QImage& image, double timeSec); + + void resetTracking(); + // diagnostics: how many frames actually ran the detector (tracking test) + int detectorRuns() const; + +private: + struct Impl; + std::unique_ptr d; +}; + +#endif // ENABLE_MOCAP +#endif // FACECAPPREDICTOR_H diff --git a/src/Mocap/FaceCapPredictor_test.cpp b/src/Mocap/FaceCapPredictor_test.cpp new file mode 100644 index 000000000..fa822f67d --- /dev/null +++ b/src/Mocap/FaceCapPredictor_test.cpp @@ -0,0 +1,84 @@ +#ifdef ENABLE_MOCAP + +#include + +#include +#include + +#include "Mocap/FaceCapCanonicalData.h" +#include "Mocap/FaceCapPredictor.h" + +// Without models on disk the predictor must degrade gracefully (the +// UniRig/MeshSegmenter pattern): load() false, isAvailable() false, a clear +// lastError, and predict() returning confidence 0 without crashing. +TEST(FaceCapPredictor, GracefulWithoutModels) +{ + FaceCapPredictor p; + EXPECT_FALSE(p.isAvailable()); + const bool loaded = p.load(QStringLiteral("/nonexistent/model/dir")); + EXPECT_FALSE(loaded); + EXPECT_FALSE(p.isAvailable()); + EXPECT_FALSE(p.lastError().isEmpty()); + + QImage img(64, 64, QImage::Format_RGB888); + img.fill(Qt::gray); + const FaceSample s = p.predict(img, 1.5); + EXPECT_DOUBLE_EQ(s.timeSec, 1.5); + EXPECT_FLOAT_EQ(s.confidence, 0.f); +} + +TEST(FaceCapPredictor, BlendshapeVocabularyIsStable) +{ + // the shared vocabulary contract: 52 names, _neutral first, jawOpen at 25 + EXPECT_EQ(FaceCap::kBlendshapeCount, 52); + EXPECT_STREQ(FaceCap::kBlendshapeNames[0], "_neutral"); + EXPECT_STREQ(FaceCap::kBlendshapeNames[25], "jawOpen"); + EXPECT_STREQ(FaceCap::kBlendshapeNames[9], "eyeBlinkLeft"); + EXPECT_STREQ(FaceCap::kBlendshapeNames[44], "mouthSmileLeft"); + EXPECT_EQ(static_cast(FaceCap::kBlendshapeLandmarkSubset.size()), 146); +} + +// Real-inference test, env-gated (models are not in CI): set +// QTMESH_MOCAP_MODELS_DIR to a dir containing the three face graphs and +// QTMESH_MOCAP_TEST_IMAGE to a clear frontal-face photo. +TEST(FaceCapPredictor, EnvGatedRealInference) +{ + const QByteArray dir = qgetenv("QTMESH_MOCAP_MODELS_DIR"); + const QByteArray imagePath = qgetenv("QTMESH_MOCAP_TEST_IMAGE"); + if (dir.isEmpty() || imagePath.isEmpty()) + GTEST_SKIP() << "set QTMESH_MOCAP_MODELS_DIR + QTMESH_MOCAP_TEST_IMAGE"; + + FaceCapPredictor p; + ASSERT_TRUE(p.load(QString::fromUtf8(dir))) << p.lastError().toStdString(); + ASSERT_TRUE(p.isAvailable()); + + QImage img(QString::fromUtf8(imagePath)); + ASSERT_FALSE(img.isNull()); + + const FaceSample s = p.predict(img, 0.0); + EXPECT_GT(s.confidence, 0.9f); + for (float w : s.weights) { + EXPECT_GE(w, 0.f); + EXPECT_LE(w, 1.f); + } + // a unit quaternion came back + const auto& q = s.headRotation; + EXPECT_NEAR(q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3], 1.f, 1e-3); + + // detector-skip: a second predict on the same frame must NOT re-run the + // detector (tracking ROI from the first frame's landmarks) + const int runsAfterFirst = p.detectorRuns(); + const FaceSample s2 = p.predict(img, 1.0 / 30.0); + EXPECT_GT(s2.confidence, 0.9f); + EXPECT_EQ(p.detectorRuns(), runsAfterFirst); + + // spot-check values against docs/MOCAP_SPIKE.md's parity run + for (int i : {9, 25, 44, 45}) { + printf(" %s = %.4f (first) / %.4f (tracked)\n", + FaceCap::kBlendshapeNames[i], s.weights[i], s2.weights[i]); + } + printf(" headRotation = (%.3f, %.3f, %.3f, %.3f) confidence=%.3f\n", + q[0], q[1], q[2], q[3], s.confidence); +} + +#endif // ENABLE_MOCAP diff --git a/src/Mocap/OneEuroFilter.cpp b/src/Mocap/OneEuroFilter.cpp new file mode 100644 index 000000000..a3e4c0555 --- /dev/null +++ b/src/Mocap/OneEuroFilter.cpp @@ -0,0 +1,95 @@ +#include "OneEuroFilter.h" + +#include + +namespace { + +double smoothingAlpha(double cutoffHz, double dt) +{ + const double tau = 1.0 / (2.0 * M_PI * cutoffHz); + return 1.0 / (1.0 + tau / dt); +} + +} // namespace + +double OneEuroFilter::filter(double value, double timeSec) +{ + if (!m_initialized) { + m_initialized = true; + m_lastTime = timeSec; + m_lastValue = value; + m_lastDerivative = 0.0; + return value; + } + double dt = timeSec - m_lastTime; + if (dt <= 0.0) + return m_lastValue; + m_lastTime = timeSec; + + const double rawDerivative = (value - m_lastValue) / dt; + const double aD = smoothingAlpha(m_params.dCutoff, dt); + m_lastDerivative = aD * rawDerivative + (1.0 - aD) * m_lastDerivative; + + const double cutoff = m_params.minCutoff + + m_params.beta * std::abs(m_lastDerivative); + const double a = smoothingAlpha(cutoff, dt); + m_lastValue = a * value + (1.0 - a) * m_lastValue; + return m_lastValue; +} + +std::array OneEuroQuatFilter::filter(const std::array& quat, + double timeSec) +{ + // hemisphere alignment: q and -q are the same rotation; keep the + // representation continuous against the previous OUTPUT. + std::array q = quat; + if (m_initialized) { + const double dot = q[0] * m_lastQuat[0] + q[1] * m_lastQuat[1] + + q[2] * m_lastQuat[2] + q[3] * m_lastQuat[3]; + if (dot < 0.0) + for (auto& c : q) c = -c; + } + + if (!m_initialized) { + m_initialized = true; + m_lastTime = timeSec; + m_lastSpeed = 0.0; + m_lastQuat = q; + return q; + } + double dt = timeSec - m_lastTime; + if (dt <= 0.0) + return m_lastQuat; + m_lastTime = timeSec; + + // angular distance previous-output -> sample + double dot = q[0] * m_lastQuat[0] + q[1] * m_lastQuat[1] + + q[2] * m_lastQuat[2] + q[3] * m_lastQuat[3]; + dot = std::fmin(1.0, std::fmax(-1.0, dot)); + const double angle = 2.0 * std::acos(std::abs(dot)); + + const double aD = smoothingAlpha(m_params.dCutoff, dt); + m_lastSpeed = aD * (angle / dt) + (1.0 - aD) * m_lastSpeed; + + const double cutoff = m_params.minCutoff + m_params.beta * m_lastSpeed; + const double a = smoothingAlpha(cutoff, dt); + + // slerp(m_lastQuat, q, a) + std::array out; + if (angle < 1e-6) { + out = q; + } else { + const double theta = std::acos(std::abs(dot)); + const double s = std::sin(theta); + const double w0 = std::sin((1.0 - a) * theta) / s; + const double w1 = std::sin(a * theta) / s; + for (int i = 0; i < 4; ++i) + out[i] = static_cast(w0 * m_lastQuat[i] + w1 * q[i]); + double n = std::sqrt(out[0] * out[0] + out[1] * out[1] + + out[2] * out[2] + out[3] * out[3]); + if (n > 0.0) + for (auto& c : out) c = static_cast(c / n); + } + m_lastQuat = out; + return out; +} diff --git a/src/Mocap/OneEuroFilter.h b/src/Mocap/OneEuroFilter.h new file mode 100644 index 000000000..74121d322 --- /dev/null +++ b/src/Mocap/OneEuroFilter.h @@ -0,0 +1,61 @@ +#ifndef ONEEUROFILTER_H +#define ONEEUROFILTER_H + +// One-Euro filter (Casiez, Roussel & Vogel, CHI 2012) for mocap channel +// smoothing (epic #869, Slice C #872). Pure data — no Qt/Ogre/ONNX — so the +// recorder, the live controller and the tests all share it headless. +// +// Defaults are tuned for 30 fps face capture: minCutoff 1.0 Hz keeps slow +// drift smooth, beta 0.05 lets fast expressions track without lag. + +#include + +class OneEuroFilter { +public: + struct Params { + double minCutoff = 1.0; // Hz — smoothing at rest (lower = smoother) + double beta = 0.05; // speed coefficient (higher = faster tracking) + double dCutoff = 1.0; // Hz — derivative smoothing + }; + + OneEuroFilter() : m_params() {} + explicit OneEuroFilter(const Params& params) : m_params(params) {} + + // value at timeSec (monotonically increasing). The first sample passes + // through unchanged. + double filter(double value, double timeSec); + + void reset() { m_initialized = false; } + const Params& params() const { return m_params; } + +private: + Params m_params; + bool m_initialized = false; + double m_lastTime = 0.0; + double m_lastValue = 0.0; + double m_lastDerivative = 0.0; +}; + +// Quaternion variant: hemisphere-aligns each sample against the previous +// output (dot >= 0) and slerps toward the new sample with a One-Euro-derived +// alpha where "speed" is the angular velocity between consecutive samples. +// Quaternions are (x, y, z, w), unit length. +class OneEuroQuatFilter { +public: + OneEuroQuatFilter() : m_params() {} + explicit OneEuroQuatFilter(const OneEuroFilter::Params& params) + : m_params(params) {} + + std::array filter(const std::array& quat, double timeSec); + + void reset() { m_initialized = false; } + +private: + OneEuroFilter::Params m_params; + bool m_initialized = false; + double m_lastTime = 0.0; + double m_lastSpeed = 0.0; // rad/s, low-passed with dCutoff + std::array m_lastQuat{0.f, 0.f, 0.f, 1.f}; +}; + +#endif // ONEEUROFILTER_H diff --git a/src/Mocap/OneEuroFilter_test.cpp b/src/Mocap/OneEuroFilter_test.cpp new file mode 100644 index 000000000..1c057b71d --- /dev/null +++ b/src/Mocap/OneEuroFilter_test.cpp @@ -0,0 +1,122 @@ +#ifdef ENABLE_MOCAP + +#include + +#include + +#include "Mocap/OneEuroFilter.h" + +TEST(OneEuroFilter, FirstSamplePassesThrough) +{ + OneEuroFilter f; + EXPECT_DOUBLE_EQ(f.filter(0.7, 0.0), 0.7); +} + +TEST(OneEuroFilter, StepResponseSmooths) +{ + OneEuroFilter f(OneEuroFilter::Params{1.0, 0.0, 1.0}); // beta 0: pure low-pass + f.filter(0.0, 0.0); + const double stepped = f.filter(1.0, 1.0 / 30.0); + EXPECT_GT(stepped, 0.0); + EXPECT_LT(stepped, 0.6); // strongly smoothed at 30 fps with 1 Hz cutoff + + // converges toward the target + double v = stepped; + for (int i = 2; i < 90; ++i) + v = f.filter(1.0, i / 30.0); + EXPECT_GT(v, 0.95); +} + +TEST(OneEuroFilter, BetaLetsFastMotionTrack) +{ + OneEuroFilter slow(OneEuroFilter::Params{1.0, 0.0, 1.0}); + OneEuroFilter fast(OneEuroFilter::Params{1.0, 5.0, 1.0}); + slow.filter(0.0, 0.0); + fast.filter(0.0, 0.0); + double vSlow = 0.0, vFast = 0.0; + for (int i = 1; i <= 10; ++i) { + const double target = i * 0.5; // fast ramp + vSlow = slow.filter(target, i / 30.0); + vFast = fast.filter(target, i / 30.0); + } + EXPECT_GT(vFast, vSlow); // high beta tracks the ramp closer +} + +TEST(OneEuroFilter, NonMonotonicTimeIsIgnored) +{ + OneEuroFilter f; + f.filter(1.0, 1.0); + EXPECT_DOUBLE_EQ(f.filter(99.0, 1.0), f.filter(99.0, 0.5)); +} + +namespace { + +std::array quatAboutZ(double rad) +{ + return {0.f, 0.f, static_cast(std::sin(rad / 2)), + static_cast(std::cos(rad / 2))}; +} + +double quatAngle(const std::array& a, const std::array& b) +{ + double dot = 0; + for (int i = 0; i < 4; ++i) + dot += a[i] * b[i]; + return 2.0 * std::acos(std::fmin(1.0, std::fabs(dot))); +} + +} // namespace + +TEST(OneEuroQuatFilter, FirstSamplePassesThrough) +{ + OneEuroQuatFilter f; + const auto q = quatAboutZ(0.3); + const auto out = f.filter(q, 0.0); + // float storage: acos near 1.0 amplifies rounding, so ~1e-3 rad is exact + EXPECT_NEAR(quatAngle(out, q), 0.0, 1e-3); +} + +TEST(OneEuroQuatFilter, SmoothsTowardSample) +{ + OneEuroQuatFilter f(OneEuroFilter::Params{1.0, 0.0, 1.0}); + f.filter(quatAboutZ(0.0), 0.0); + const auto target = quatAboutZ(1.0); + const auto out = f.filter(target, 1.0 / 30.0); + const double remaining = quatAngle(out, target); + EXPECT_GT(remaining, 0.3); // did not jump straight to the target + EXPECT_LT(remaining, 1.0); // but moved toward it +} + +TEST(OneEuroQuatFilter, HemisphereConsistency) +{ + OneEuroQuatFilter f; + auto q = quatAboutZ(0.2); + const auto first = f.filter(q, 0.0); + // feed the SAME rotation with flipped sign: output must stay in the + // previous output's hemisphere (dot >= 0), no 360-degree pop + for (auto& c : q) + c = -c; + const auto out = f.filter(q, 1.0 / 30.0); + double dot = 0; + for (int i = 0; i < 4; ++i) + dot += out[i] * first[i]; + EXPECT_GE(dot, 0.0); + EXPECT_NEAR(quatAngle(out, first), 0.0, 1e-4); +} + +TEST(OneEuroQuatFilter, ContinuousUnderIncrementalRotation) +{ + OneEuroQuatFilter f(OneEuroFilter::Params{1.5, 0.5, 1.0}); + auto prev = f.filter(quatAboutZ(0.0), 0.0); + for (int i = 1; i <= 60; ++i) { + const auto out = f.filter(quatAboutZ(i * 0.05), i / 30.0); + EXPECT_LT(quatAngle(out, prev), 0.2); // no jumps between frames + double n = 0; + for (int k = 0; k < 4; ++k) + n += out[k] * out[k]; + EXPECT_NEAR(n, 1.0, 1e-4); // stays unit length + prev = out; + } +} + +#endif // ENABLE_MOCAP From 5c5bbf9d12da62cc32f457bc085e308b7522c26b Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 12 Jul 2026 22:08:35 -0300 Subject: [PATCH 04/31] feat(#873): MocapRecorder + 'qtmesh mocap --face' CLI + MCP capture_face_from_video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice D of epic #869 — the first user-visible milestone: video file -> animated mesh, end-to-end. - MocapRecorder (src/Mocap/): FaceSample stream -> morph weight keyframes on a named clip via the #519 pipeline (new MorphAnimationManager:: writeWeightKeyOn public static — the entity-explicit core of setMorphWeightKeyframe, which now delegates to it), with epsilon run-length suppression, first/last anchoring, jump pre-anchors and >0.5s face-lost gaps held at both edges. Head pose is calibrated on the take's first confident frame and keyed as rotation deltas on the Head bone (canonicalIndexForBone role 5) in '_Head', or as node-TRS deltas via NodeAnimationManager for static meshes. - RecordMocapClipCommand (src/commands/): one undo step per take — first redo snapshots the pre-existing weight/head clips keyframe-for-keyframe, undo restores them exactly (verified by test). - CLI 'qtmesh mocap' (src/Mocap/MocapCLI.cpp, the SceneLightsCLI pattern): import mesh -> mapping table (matched/unmatched printed, never silently dropped) -> FileFrameSource or --frames-dir image sequence -> FaceCapPredictor -> One-Euro -> recordFace -> optional re-export; --json emits the FaceRecordReport. Non-MOCAP builds print the standard rebuild hint. Registered in the dispatcher, subcommand list and usage text. - MCP capture_face_from_video: heavy tool, live-scene entity (selected or entity_name), single undoable clip, optional output_path export, report JSON; clean error on non-MOCAP builds. - Fixes a latent CLI bug: writeCliError text was silently lost at _exit() (unflushed static QTextStream) — every subcommand's stderr errors printed nothing; now flushed. - Sentry ai.assist.mocap_face breadcrumbs; gamification noteOperation("mocap_face", {frames, keyframes}) on CLI + MCP. Verified end-to-end on macOS with the Slice A models: a 6-frame image sequence (portrait -> rotated portrait) onto (a) a static OBJ — 5 head keys on the node path, glb exported — and (b) a minimal glTF with jawOpen/ mouthSmileLeft targets — 2 channels matched, 9 weight keys, exported glb carries the 'FaceCap' morph-weights animation (path:"weights"). Known upstream gap noted: the Assimp glTF exporter doesn't emit extras.targetNames, so reimported targets alias to Shape_N. Co-Authored-By: Claude Fable 5 --- src/AppLaunchHandler.cpp | 1 + src/CLIPipeline.cpp | 13 +- src/CMakeLists.txt | 3 + src/MCPServer.cpp | 187 +++++++++++ src/MCPServer.h | 1 + src/Mocap/MocapCLI.cpp | 428 ++++++++++++++++++++++++ src/Mocap/MocapCLI.h | 25 ++ src/Mocap/MocapRecorder.cpp | 250 ++++++++++++++ src/Mocap/MocapRecorder.h | 81 +++++ src/Mocap/MocapRecorder_test.cpp | 305 +++++++++++++++++ src/MorphAnimationManager.cpp | 44 ++- src/MorphAnimationManager.h | 12 + src/commands/RecordMocapClipCommand.cpp | 187 +++++++++++ src/commands/RecordMocapClipCommand.h | 55 +++ 14 files changed, 1576 insertions(+), 16 deletions(-) create mode 100644 src/Mocap/MocapCLI.cpp create mode 100644 src/Mocap/MocapCLI.h create mode 100644 src/Mocap/MocapRecorder.cpp create mode 100644 src/Mocap/MocapRecorder.h create mode 100644 src/Mocap/MocapRecorder_test.cpp create mode 100644 src/commands/RecordMocapClipCommand.cpp create mode 100644 src/commands/RecordMocapClipCommand.h diff --git a/src/AppLaunchHandler.cpp b/src/AppLaunchHandler.cpp index 670deeef0..2a1525f51 100644 --- a/src/AppLaunchHandler.cpp +++ b/src/AppLaunchHandler.cpp @@ -31,6 +31,7 @@ bool isCliSubcommand(const QString& arg) QStringLiteral("generate3d"), QStringLiteral("morph"), QStringLiteral("nodeanim"), QStringLiteral("ps1"), QStringLiteral("cloud"), + QStringLiteral("mocap"), }; return kSubcommands.contains(arg); } diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 550866f06..e7a2027fc 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -6,6 +6,7 @@ #include "AlembicImporter.h" #include "SceneLightsIO.h" #include "SceneLightsCLI.h" +#include "Mocap/MocapCLI.h" #include "AnimationMerger.h" #include "MotionInbetween.h" #include "MotionLibrary.h" @@ -591,7 +592,10 @@ void CLIPipeline::writeOutput(const QString& text) void CLIPipeline::writeCliError(const QString& text) { - err() << text; + // Explicit flush: the CLI exits via _exit(), which skips the static + // QTextStream's destructor — without this the error text is silently + // lost in the stream buffer. + err() << text << Qt::flush; } void CLIPipeline::printUsage() @@ -866,6 +870,12 @@ void CLIPipeline::printUsage() " nodeanim --list [--json] List node-animation clips on a scene (props, doors, machinery,\n" " animated lights — anything non-skeletal). Authoring on the CLI\n" " side needs the C5 glTF/FBX exporter round-trip first.\n" + " mocap