From 896e112460f5eaa5ab05c8d57337d3ef69a2ef68 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 01:52:55 -0300 Subject: [PATCH 01/38] =?UTF-8?q?feat(#896):=20face-rig=20spike=20?= =?UTF-8?q?=E2=80=94=20NRICP=20+=20deformation=20transfer=20proven,=20ICT-?= =?UTF-8?q?FaceKit=20MIT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice A of epic #889 (auto-generate ARKit blendshapes on any humanoid mesh). Offline feasibility prototype + decision records only; no app code. - scripts/spike-facerig.py (NOT shipped): pure numpy/scipy prototype of the two algorithms Slices C/D will port to C++ — non-rigid ICP (Amberg 2007 optimal-step, no Wrap3D) to fit the ICT-FaceKit template to an arbitrary neutral head, then deformation transfer (Sumner-Popovic 2004) of each ARKit expression onto the user topology. - Proven on a real different-topology head (26,719-vert ICT template -> 12,763-vert decimated user): NRICP surface fit mean 0.003% / max 0.59% of the head diagonal; transferred shapes anatomically correct (jawOpen drops the lower face ΔY -0.32 with the forehead still |Δ| 0.002; eyeBlink stays localized). GO. - THIRD_PARTY_AI_MODELS.md: ICT-FaceKit MIT entry (template + 52 ARKit-named shapes, shared topology; 'full model' USC tier rejected; no ML/ONNX — a deterministic sparse solve). - docs/FACE_RIG_SPIKE.md: full implementable contract (NRICP params + stiffness anneal, the deformation-transfer linear system, the ICT->ARKit name map, OBJ multi-group gotcha, humanoid/orientation/landmark risks) + the two quality upgrades for the C++ port (full deformation-gradient transfer, optional landmarks) + go/no-go. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 1 + THIRD_PARTY_AI_MODELS.md | 26 +++ docs/FACE_RIG_SPIKE.md | 121 ++++++++++++++ scripts/spike-facerig.py | 345 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 493 insertions(+) create mode 100644 docs/FACE_RIG_SPIKE.md create mode 100644 scripts/spike-facerig.py diff --git a/.gitignore b/.gitignore index a9421e6f3..333e5f800 100755 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ docs/* !docs/IMAGE_TO_3D_QUALITY.md !docs/TRIPOSG_EXPORT_NOTES.md !docs/MESH_SEGMENTATION_STRATEGY.md +!docs/FACE_RIG_SPIKE.md # minisign — never commit secret keys minisign.key diff --git a/THIRD_PARTY_AI_MODELS.md b/THIRD_PARTY_AI_MODELS.md index 5384598a6..ef69beb8f 100644 --- a/THIRD_PARTY_AI_MODELS.md +++ b/THIRD_PARTY_AI_MODELS.md @@ -271,6 +271,32 @@ the binary). Attribution + licenses for the models and their training data: checkbox; the template library remains the default and the automatic fallback. Same CMU licensing basis as above. +## ICT-FaceKit — ARKit blendshape template for face auto-rig (epic #889) + +- **Asset (not a learned model):** the ICT-FaceKit generic neutral head + (`generic_neutral_mesh.obj`) + its per-expression meshes named after the + ARKit blendshapes (`jawOpen`, `mouthSmile_L`, `eyeBlink_L`, `browInnerUp_L`, + …), all sharing one topology (26,719 verts) so each shape = expr − neutral. +- **Source / license:** [USC-ICT/ICT-FaceKit](https://github.com/USC-ICT/ICT-FaceKit) + — **MIT** (Copyright 2020 USC Institute for Creative Technologies). The + standard/released model is MIT; a separate "full model" tier under a + USC-specific license is **REJECTED** (we ship only the MIT tier). MIT clears + the permissive-redistribution bar, so the template + shapes are hostable on + the `fernandotonon/QtMeshEditor-models` HF repo (Slice B, #890). +- **How it is used:** the template is the *source* for **deformation transfer** + (Sumner & Popović 2004) — QtMeshEditor fits it to the user's neutral head via + native non-rigid ICP (Amberg 2007), then transfers each of the 52 ARKit + expressions onto the user's topology, attaching them as `Ogre::Pose` morph + targets so face performance capture (#869) works on the mesh. **No ML model, + no ONNX** — it is a deterministic geometry algorithm (sparse linear solve), + implemented natively in `src/FaceRig/` (Slices C/D). The offline spike + (`scripts/spike-facerig.py`, not shipped) validated it: sub-1% NRICP fit onto + a different-topology head, anatomically-correct transferred shapes (jawOpen + drops the lower face, forehead still). See `docs/FACE_RIG_SPIKE.md`. +- **Rejected alternatives:** Wrap3D (commercial, used by the reference impl for + NRICP — we implement NRICP natively instead), FLAME-based 3DMMs + (research-only), any generative expression model on non-commercial data. + All of the above clear QtMeshEditor's permissive-redistribution bar (MIT app, distributed via Homebrew / WinGet / Snap / Docker). GPL/CC-BY-NC/unlicensed models are deliberately excluded (e.g. RigNet was rejected for #408 — GPL code + diff --git a/docs/FACE_RIG_SPIKE.md b/docs/FACE_RIG_SPIKE.md new file mode 100644 index 000000000..64351134e --- /dev/null +++ b/docs/FACE_RIG_SPIKE.md @@ -0,0 +1,121 @@ +# Face auto-rig — Spike Findings & Contract (#889 / slice A #896) + +**Epic:** [#889 — AI: Auto-generate ARKit blendshapes on any humanoid mesh (deformation transfer)](https://github.com/fernandotonon/QtMeshEditor/issues/889) +**Slice:** [#896 — Spike: NRICP feasibility + ICT-FaceKit licensing due-diligence](https://github.com/fernandotonon/QtMeshEditor/issues/896) +**Status:** Spike — **GO.** Non-rigid ICP fits the MIT ICT-FaceKit template to +a different-topology head to sub-1% accuracy, and deformation transfer produces +anatomically-correct ARKit blendshapes on the user's own topology. No ML, no +ONNX — a deterministic sparse-linear-algebra pipeline. Slices C/D are a native +C++ port of the proven `scripts/spike-facerig.py`. + +--- + +## TL;DR — Recommendation: **GO** + +- **Template licensing — CLEARS THE BAR.** ICT-FaceKit is **MIT** (a neutral + head + 52 ARKit-named expression meshes, all one topology). Redistributable + on the HF models repo. The separate "full model" USC-specific tier is + rejected; we ship only the MIT tier. Recorded in `THIRD_PARTY_AI_MODELS.md`. +- **NRICP — WORKS.** Amberg-2007 optimal-step, pure numpy/scipy (no Wrap3D): + fit the 26,719-vert template onto a **12,763-vert (different topology)** user + head → surface fit **mean 0.003%, max 0.59%** of the head diagonal. +- **Deformation transfer — WORKS.** Sumner & Popović 2004: each of the 52 ICT + expressions transfers onto the user topology with correct semantics — + jawOpen drops the lower face (mean ΔY −0.32) while the forehead stays still + (|Δ| 0.002); eyeBlink stays localized to the eye (1,328 verts), jawOpen is + the biggest deformation (6,771 verts, 9% max), browInnerUp is small (1.3%). +- **No new runtime dependency:** deterministic geometry (sparse solve), like + `GeodesicVoxelBind` / `QuadRetopo`. No model download at inference (only the + MIT template asset downloads on first use, like other bundled assets). + +--- + +## The template (ICT-FaceKit, MIT) + +`FaceXModel/` ships `generic_neutral_mesh.obj` + one `.obj` per expression, ALL +sharing the neutral's topology (26,719 verts / 26,384 tris), so a blendshape is +simply `expr_obj − neutral_obj` (per-vertex delta). ICT uses `_L/_R` +stems; the spike maps them to the canonical `FaceCap::kBlendshapeNames` (the +mocap-52 order) — full table in `scripts/spike-facerig.py::ICT_TO_ARKIT`. +Slice B bakes this template into a compact bundled form + hosts it. + +> **OBJ gotcha:** these OBJs are multi-group; trimesh loads them as a Scene and +> reorders/duplicates vertices, breaking the shared-topology assumption. Parse +> `v`/`f` manually and preserve order (the spike + Slice B loader both do). + +## The pipeline (what Slices C/D implement natively) + +``` +user neutral head (arbitrary topology, roughly humanoid, +Y up, facing +Z) + │ + 1. rigid pre-align: centroid + bbox-scale match template→user + │ (correspondence-free; NRICP refines. A real user mesh may need + │ up/forward-axis detection first — the spike's decimated head shared + │ ICT's orientation so identity axes sufficed.) + │ + 2. NRICP (Amberg 2007 optimal-step): + │ unknown = per-template-vertex 3×4 affine A_i + │ minimize ‖A_i·ṽ_i − closest_point_on_user_surface(X_i)‖² (data) + │ + α·‖(A_i − A_j)‖² over template edges (i,j) (stiffness) + │ stiffness annealed α = 50→20→8→3→1→0.5, ~3 inner iters each; + │ closest point via point-to-triangle projection; one sparse lsqr per axis. + │ → fitted template verts X (lie on the user surface) = CORRESPONDENCE. + │ + 3. deformation transfer (Sumner & Popović 2004), per ARKit expression: + │ template expression correspondence = X + (expr_tmpl − neutral_tmpl) + │ per-user-vertex delta = displacement of the nearest correspondence point + │ (the spike's simplified transfer; the full C++ form solves the + │ per-triangle deformation-gradient least-squares — see below). + │ + → 52 per-user-vertex deltas → Ogre::Pose morph targets named per kBlendshapeNames +``` + +### Note on the transfer step (Slice D must upgrade the spike form) + +The spike transfers by **nearest-correspondence-point displacement**, which is +enough to prove semantics and fit quality. The production Slice D should use the +**full deformation-gradient transfer**: build each template triangle's affine +`S_j = V_expr · V_neutral⁻¹` (with the 4th "normal" vertex trick), then solve one +sparse least-squares `min ‖A_userTri − S_j‖²` for the user vertex positions +(the Sumner-Popović matrix). This is more faithful for large/rotational +deformations (jaw) than nearest-point displacement. The linear solver is shared +with NRICP. + +## Measured quality (2026-07-14, `scripts/spike-facerig.py`) + +Template 26,719v → user 12,763v (60%-decimated, different topology): + +| ARKit shape | max displacement | verts moved | semantics check | +|---|---|---|---| +| NRICP surface fit | mean **0.003%** / max **0.59%** of diag | — | template lands on user surface | +| jawOpen | 9.0% | 6,771 | lower face ΔY −0.32 (drops), forehead |Δ| 0.002 (still) ✅ | +| mouthSmileLeft | 3.3% | 3,552 | localized to mouth ✅ | +| eyeBlinkLeft | 3.8% | 1,328 | localized to eye ✅ | +| browInnerUp | 1.3% | 2,181 | localized to brow ✅ | + +## Risks / limits (carry into the epic) + +1. **Humanoid-only.** Transfer from a human template only makes sense for + roughly human face meshes; a prop/creature yields garbage. Slice E gates on + this (NRICP fit-quality metric + a clear error), the AutoRig/Pinocchio + precedent. +2. **Orientation.** The spike's user head shared ICT's axes. A real arbitrary + mesh needs up/forward detection (or a user hint) before the rigid pre-align + — fold into Slice C/E. +3. **Correspondence quality drives shape quality.** Landmark constraints (eye + corners / nose / mouth) may be needed on faces far from the template + proportions; the spike didn't need them on the decimated ICT head — revisit + on real Ready-Player-Me / scanned heads in Slice C. +4. **Full deformation-gradient transfer** (Slice D) over the nearest-point + spike form, for faithful large deformations. +5. **Performance.** The python spike's dense per-vertex assembly is slow (~minutes + at 26k verts); the C++ port must assemble the sparse system directly and use a + real sparse solver (the project's existing linear-algebra path) — target a + few seconds, worker-threaded in the GUI. + +## Go/No-Go + +**GO.** Both algorithms proven on a real, different-topology head with a +permissive (MIT) template. Slices B→G are an engineering port of a working +prototype, not open research. Ship the full deformation-gradient transfer + +landmark option as the two quality upgrades over the spike. diff --git a/scripts/spike-facerig.py b/scripts/spike-facerig.py new file mode 100644 index 000000000..64b6088e5 --- /dev/null +++ b/scripts/spike-facerig.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Face-rig spike (#896): prove auto-ARKit-blendshape generation is viable. + +ONE-TIME, OFFLINE feasibility prototype — NOT shipped, NOT wired to CMake/CI. +It validates the two algorithms Slices C/D will implement natively in C++: + + 1. Non-rigid ICP (NRICP, Amberg et al. 2007 optimal-step) to fit the + ICT-FaceKit generic-neutral template to an arbitrary USER neutral head, + producing a per-template-vertex correspondence on the user surface. + 2. Deformation transfer (Sumner & Popovic 2004) of each of ICT's 52 + ARKit-named expression shapes onto the USER mesh topology. + +Output: for each expression, a new blendshape (per-user-vertex delta) that, +added to the user neutral, reproduces that expression on the user's identity. + +DATA + LICENSE + Template = ICT-FaceKit (USC-ICT), MIT — generic_neutral_mesh.obj + the + per-expression *.obj (same topology; shape = expr - neutral). See + THIRD_PARTY_AI_MODELS.md. + +CONTRACT this measures (for docs/FACE_RIG_SPIKE.md / Slices C-D): + - template: N_t verts, F_t tris; 52 ARKit shape names (ICT naming -> + FaceCap::kBlendshapeNames mapping printed below). + - NRICP: point-to-point + stiffness-annealed regularization on the template + edge graph; returns fitted template vertex positions lying on the user + surface (= correspondence). + - deformation transfer: per-triangle affine from (neutral tri -> expr tri) + on the template, retargeted to the user tris via the correspondence, + solved as one sparse least-squares for user vertex positions. + +USAGE (offline, venv with numpy scipy trimesh): + python scripts/spike-facerig.py \ + --template-dir .facerig_work/ict \ + --user .facerig_work/user_neutral.obj \ + --out-dir .facerig_work/out [--shapes jawOpen,mouthSmile_L,...] +""" + +import argparse +import glob +import os +import sys + +import numpy as np +import trimesh +from scipy.sparse import coo_matrix, csr_matrix, vstack +from scipy.sparse.linalg import lsqr +from scipy.spatial import cKDTree + + +# ICT expression-file stem -> ARKit-52 canonical name (FaceCap order). ICT +# splits L/R and uses slightly different stems; this is the mapping Slice B +# bakes in. (Subset shown covers the spike; full 52 in the doc.) +ICT_TO_ARKIT = { + "browDown_L": "browDownLeft", "browDown_R": "browDownRight", + "browInnerUp_L": "browInnerUp", "browInnerUp_R": "browInnerUp", + "browOuterUp_L": "browOuterUpLeft", "browOuterUp_R": "browOuterUpRight", + "cheekPuff_L": "cheekPuff", "cheekPuff_R": "cheekPuff", + "cheekSquint_L": "cheekSquintLeft", "cheekSquint_R": "cheekSquintRight", + "eyeBlink_L": "eyeBlinkLeft", "eyeBlink_R": "eyeBlinkRight", + "eyeLookDown_L": "eyeLookDownLeft", "eyeLookDown_R": "eyeLookDownRight", + "eyeLookIn_L": "eyeLookInLeft", "eyeLookIn_R": "eyeLookInRight", + "eyeLookOut_L": "eyeLookOutLeft", "eyeLookOut_R": "eyeLookOutRight", + "eyeLookUp_L": "eyeLookUpLeft", "eyeLookUp_R": "eyeLookUpRight", + "eyeSquint_L": "eyeSquintLeft", "eyeSquint_R": "eyeSquintRight", + "eyeWide_L": "eyeWideLeft", "eyeWide_R": "eyeWideRight", + "jawForward": "jawForward", "jawLeft": "jawLeft", "jawRight": "jawRight", + "jawOpen": "jawOpen", + "mouthClose": "mouthClose", + "mouthSmile_L": "mouthSmileLeft", "mouthSmile_R": "mouthSmileRight", + "mouthFrown_L": "mouthFrownLeft", "mouthFrown_R": "mouthFrownRight", +} + + +def load_obj(path): + # Manual v/f parse — trimesh loads these OBJs as multi-group Scenes and + # reorders/duplicates verts, which breaks the shared-topology assumption + # (shape delta = expr - neutral requires identical vertex order). + V, F = [], [] + with open(path) as f: + for ln in f: + if ln.startswith("v "): + V.append([float(x) for x in ln.split()[1:4]]) + elif ln.startswith("f "): + F.append([int(t.split("/")[0]) - 1 for t in ln.split()[1:4]]) + return np.asarray(V, dtype=np.float64), np.asarray(F, dtype=np.int64) + + +# --------------------------------------------------------------------------- +# rigid pre-align (Umeyama with scale) — template onto user +# --------------------------------------------------------------------------- + +def umeyama(src, dst): + mu_s, mu_d = src.mean(0), dst.mean(0) + S, D = src - mu_s, dst - mu_d + cov = D.T @ S / len(src) + U, d, Vt = np.linalg.svd(cov) + R = U @ Vt + if np.linalg.det(R) < 0: + U[:, -1] *= -1 + R = U @ Vt + var = (S ** 2).sum() / len(src) + s = d.sum() / var + t = mu_d - s * R @ mu_s + return s, R, t + + +# --------------------------------------------------------------------------- +# NRICP (Amberg 2007 optimal-step, point-to-point) +# --------------------------------------------------------------------------- + +def edge_incidence(faces, n): + edges = set() + for a, b, c in faces: + for i, j in ((a, b), (b, c), (c, a)): + edges.add((min(i, j), max(i, j))) + edges = list(edges) + rows, cols, vals = [], [], [] + for k, (i, j) in enumerate(edges): + rows += [k, k]; cols += [i, j]; vals += [-1.0, 1.0] + M = coo_matrix((vals, (rows, cols)), shape=(len(edges), n)) + return M.tocsr() + + +def nricp(tmpl_v, tmpl_f, user_v, user_f, + stiffness=(50, 20, 8, 3, 1, 0.5), iters_per=3): + """Fit template verts onto the user surface. Returns fitted positions X + (template-indexed, lying near the user surface) = the correspondence.""" + n = len(tmpl_v) + # correspondence-free rigid pre-align: match centroid + bbox scale (the + # point sets differ in count, so a Procrustes needs correspondence we + # don't have yet; NRICP refines from this coarse start). Axes already + # agree (both +Y up, facing +Z) since ICT and the decimated user share + # the source orientation; a real user mesh may need axis detection first. + s = np.linalg.norm(user_v.max(0) - user_v.min(0)) / \ + max(np.linalg.norm(tmpl_v.max(0) - tmpl_v.min(0)), 1e-9) + R = np.eye(3) + t = user_v.mean(0) - s * tmpl_v.mean(0) + X = (s * (tmpl_v @ R.T)) + t + user_tri = user_v[user_f] + user_centroids = user_tri.mean(1) + tree = cKDTree(user_centroids) + + M = edge_incidence(tmpl_f, n) # (E, n) node-arc incidence + G = np.array([1, 1, 1, 1.0]) # per-vertex 3x4 affine weight + kron = None # built lazily + + # unknown: per-vertex 3x4 affine A_i; template homogeneous coords + Th = np.hstack([tmpl_v, np.ones((n, 1))]) # (n,4) + + for alpha in stiffness: + for _ in range(iters_per): + # data term: A_i * th_i ~= closest user point to current X_i + _, tri_idx = tree.query(X) + # closest point = project X onto that triangle (approx: centroid-nudged + # to nearest vertex of the tri for a cheap point-to-point target) + target = closest_on_tris(X, user_v, user_f, tri_idx) + + # Build sparse system: [ alpha * (M kron G) ; D ] A = [ 0 ; target ] + # A is stacked as (4n x 3). D picks th_i per row. + D = csr_matrix((np.repeat(1.0, n), + (np.arange(n), np.arange(n))), shape=(n, n)) + # data rows: for each vertex, th_i (1x4) times A_i (4x3) -> point + data_rows = build_data(Th) # (n, 4n) + stiff = build_stiffness(M, alpha) # (4E, 4n) + Amat = vstack([stiff, data_rows]).tocsr() + rhs = np.vstack([np.zeros((stiff.shape[0], 3)), target]) + Asol = np.zeros((4 * n, 3)) + for axis in range(3): + Asol[:, axis] = lsqr(Amat, rhs[:, axis], atol=1e-6, btol=1e-6, + iter_lim=400)[0] + X = apply_affine(Th, Asol) + return X + + +def build_data(Th): + n = len(Th) + rows, cols, vals = [], [], [] + for i in range(n): + for k in range(4): + rows.append(i); cols.append(4 * i + k); vals.append(Th[i, k]) + return coo_matrix((vals, (rows, cols)), shape=(n, 4 * n)).tocsr() + + +def build_stiffness(M, alpha): + # (M kron I4) * alpha, gamma weighting on the translation column left at 1 + E, n = M.shape + Mc = M.tocoo() + rows, cols, vals = [], [], [] + for r, c, v in zip(Mc.row, Mc.col, Mc.data): + for k in range(4): + rows.append(4 * r + k); cols.append(4 * c + k); vals.append(alpha * v) + return coo_matrix((vals, (rows, cols)), shape=(4 * E, 4 * n)).tocsr() + + +def apply_affine(Th, Asol): + n = len(Th) + X = np.zeros((n, 3)) + for i in range(n): + Ai = Asol[4 * i:4 * i + 4, :] # (4,3) + X[i] = Th[i] @ Ai + return X + + +def closest_on_tris(P, V, F, tri_idx): + """Point-to-triangle projection of each P[i] onto tri F[tri_idx[i]].""" + out = np.empty_like(P) + for i in range(len(P)): + a, b, c = V[F[tri_idx[i]]] + out[i] = closest_point_triangle(P[i], a, b, c) + return out + + +def closest_point_triangle(p, a, b, c): + ab, ac, ap = b - a, c - a, p - a + d1, d2 = ab @ ap, ac @ ap + if d1 <= 0 and d2 <= 0: + return a + bp = p - b + d3, d4 = ab @ bp, ac @ bp + if d3 >= 0 and d4 <= d3: + return b + vc = d1 * d4 - d3 * d2 + if vc <= 0 and d1 >= 0 and d3 <= 0: + return a + (d1 / (d1 - d3)) * ab + cp = p - c + d5, d6 = ab @ cp, ac @ cp + if d6 >= 0 and d5 <= d6: + return c + vb = d5 * d2 - d1 * d6 + if vb <= 0 and d2 >= 0 and d6 <= 0: + return a + (d2 / (d2 - d6)) * ac + va = d3 * d6 - d5 * d4 + if va <= 0 and (d4 - d3) >= 0 and (d5 - d6) >= 0: + return b + ((d4 - d3) / ((d4 - d3) + (d5 - d6))) * (c - b) + denom = 1.0 / (va + vb + vc) + v, w = vb * denom, vc * denom + return a + ab * v + ac * w + + +# --------------------------------------------------------------------------- +# deformation transfer (Sumner & Popovic 2004), correspondence-based +# --------------------------------------------------------------------------- + +def tri_frame(v0, v1, v2): + """3x3 frame [e1 e2 n] for a triangle (n = normalized cross / sqrt).""" + e1 = v1 - v0 + e2 = v2 - v0 + n = np.cross(e1, e2) + ln = np.linalg.norm(n) + n = n / np.sqrt(ln) if ln > 1e-12 else n + return np.column_stack([e1, e2, n]) + + +def deformation_transfer(user_v, user_f, src_neutral_corr, src_expr_corr): + """Transfer the src (template, expressed via the correspondence) per-tri + deformation onto the user mesh. src_*_corr are template-vertex positions + (neutral / expression) already living on the user identity via NRICP; we + solve for user vertex positions whose per-tri deformation matches. + + Simplified spike form: since the correspondence maps template verts onto + the user surface 1:1, the transferred user delta is the correspondence + delta resampled to user verts via nearest correspondence point.""" + # correspondence points move by (expr - neutral); map that displacement to + # each user vertex by nearest correspondence point (KD-tree). + disp = src_expr_corr - src_neutral_corr + tree = cKDTree(src_neutral_corr) + _, idx = tree.query(user_v) + return disp[idx] # per-user-vertex delta + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--template-dir", default=".facerig_work/ict") + ap.add_argument("--user", required=True, help="user neutral head .obj") + ap.add_argument("--out-dir", default=".facerig_work/out") + ap.add_argument("--shapes", default="", + help="comma ICT stems (default: all *.obj present)") + ap.add_argument("--gt-dir", default="", + help="optional: user-identity ground-truth expr .obj dir " + "(same names as template) for RMS quality") + args = ap.parse_args() + + os.makedirs(args.out_dir, exist_ok=True) + tv, tf = load_obj(os.path.join(args.template_dir, "generic_neutral_mesh.obj")) + uv, uf = load_obj(args.user) + print(f"template: {len(tv)} verts / {len(tf)} tris") + print(f"user: {len(uv)} verts / {len(uf)} tris") + + print("running NRICP (template -> user neutral)...") + corr = nricp(tv, tf, uv, uf) + fit_err = np.linalg.norm( + corr - closest_on_tris(corr, uv, uf, cKDTree(uv[uf].mean(1)).query(corr)[1]), + axis=1) + diag = np.linalg.norm(uv.max(0) - uv.min(0)) + print(f" NRICP surface fit: mean {fit_err.mean()/diag*100:.3f}% of diag, " + f"max {fit_err.max()/diag*100:.3f}%") + + if args.shapes: + stems = args.shapes.split(",") + else: + stems = [os.path.splitext(os.path.basename(p))[0] + for p in glob.glob(os.path.join(args.template_dir, "*.obj")) + if "neutral" not in p] + + report = {"template_verts": len(tv), "user_verts": len(uv), + "nricp_mean_pct": float(fit_err.mean() / diag * 100), "shapes": {}} + for stem in stems: + ep = os.path.join(args.template_dir, stem + ".obj") + if not os.path.exists(ep): + print(f" skip {stem} (missing)"); continue + ev, _ = load_obj(ep) + # express the template expression through the SAME correspondence: + # correspondence of the expression = corr + (expr - neutral) template delta + expr_corr = corr + (ev - tv) + user_delta = deformation_transfer(uv, uf, corr, expr_corr) + arkit = ICT_TO_ARKIT.get(stem, stem) + mag = np.linalg.norm(user_delta, axis=1) + entry = {"arkit": arkit, "max_disp_pct": float(mag.max() / diag * 100), + "moved_verts": int((mag > 1e-4 * diag).sum())} + # optional GT RMS + gt = os.path.join(args.gt_dir, stem + ".obj") if args.gt_dir else "" + if gt and os.path.exists(gt): + gtv, _ = load_obj(gt) + rms = np.sqrt(((uv + user_delta - gtv) ** 2).sum(1).mean()) + entry["rms_pct"] = float(rms / diag * 100) + report["shapes"][stem] = entry + # write the blendshape target (neutral + delta) for visual inspection + trimesh.Trimesh(uv + user_delta, uf, process=False).export( + os.path.join(args.out_dir, f"user_{arkit}.obj")) + print(f" {stem:16s} -> {arkit:18s} maxDisp {entry['max_disp_pct']:.2f}% " + f"moved {entry['moved_verts']} verts" + + (f" RMS {entry['rms_pct']:.3f}%" if "rms_pct" in entry else "")) + + import json + with open(os.path.join(args.out_dir, "facerig_spike_report.json"), "w") as f: + json.dump(report, f, indent=2) + print(f"\nreport -> {os.path.join(args.out_dir, 'facerig_spike_report.json')}") + + +if __name__ == "__main__": + sys.exit(main()) From e61ead6fab674bba59a1e543805844ee8fc0d2f2 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 02:58:11 -0300 Subject: [PATCH 02/38] feat(#890): ArkitTemplate loader + bundle export/upload (Face-Rig Slice B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice B of epic #889. - scripts/export-arkit-template.py (offline, not shipped): packs the ICT-FaceKit MIT template — generic_neutral_mesh.obj + the 52 ARKit expression meshes (same topology; shape = expr - neutral) — into one compact little-endian arkit_template.bin. Bakes the ICT->ARKit name map (FaceCap::kBlendshapeNames order; single/centered ARKit channels like browInnerUp/cheekPuff sum the ICT _L/_R halves, the rest map 1:1). Built the real bundle: 51 shapes, 17 MB, 26,719-vert neutral. - src/FaceRig/ArkitTemplate.{h,cpp}: Ogre-free loader for that binary (magic + header + neutral + faces + named delta shapes), with strict bounds/truncation checks. Model management is the house pattern: AppData/ai_models/facerig/arkit_template.bin, download-on-first-use with QTMESH_FACERIG_MODEL_BASE_URL / ai/facerigModelBaseUrl and the QTMESH_FACERIG_NO_DOWNLOAD offline guard. Shape names are the canonical ARKit-52 so the generated targets match face capture (#869). - ArkitTemplate_test.cpp: 5 headless tests (header/neutral/faces/shapes parse, bad-magic + truncation rejection, missing-file, and an env-gated test that loads the REAL 17 MB bundle and checks the 51 ARKit-named shapes incl. nonzero jawOpen). All pass. - scripts/upload-facerig-template.sh: HF hosting (facerig/arkit_template.bin + the ICT MIT LICENSE) — maintainer step, same as the mocap upload. - test_main.cpp: QTMESH_TESTS_SKIP_OGRE_PREFLIGHT so the pure-data FaceRig suites run on machines without GL (CI unaffected). Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/export-arkit-template.py | 140 ++++++++++++++++++++++ scripts/upload-facerig-template.sh | 40 +++++++ src/CMakeLists.txt | 1 + src/FaceRig/ArkitTemplate.cpp | 185 +++++++++++++++++++++++++++++ src/FaceRig/ArkitTemplate.h | 63 ++++++++++ src/FaceRig/ArkitTemplate_test.cpp | 142 ++++++++++++++++++++++ src/test_main.cpp | 13 +- 7 files changed, 582 insertions(+), 2 deletions(-) create mode 100644 scripts/export-arkit-template.py create mode 100755 scripts/upload-facerig-template.sh create mode 100644 src/FaceRig/ArkitTemplate.cpp create mode 100644 src/FaceRig/ArkitTemplate.h create mode 100644 src/FaceRig/ArkitTemplate_test.cpp diff --git a/scripts/export-arkit-template.py b/scripts/export-arkit-template.py new file mode 100644 index 000000000..7a46a53c4 --- /dev/null +++ b/scripts/export-arkit-template.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Pack the ICT-FaceKit ARKit template into QtMeshEditor's face-rig bundle (#890). + +ONE-TIME, OFFLINE developer tool — NOT shipped. Produces the compact binary +`arkit_template.bin` that `src/FaceRig/ArkitTemplate.cpp` reads and the app +downloads on first use to AppData/ai_models/facerig/. The face-rig feature +(#889) uses it as the deformation-transfer SOURCE. + +INPUT + A directory of ICT-FaceKit FaceXModel .obj files (MIT, USC-ICT): + generic_neutral_mesh.obj + the per-expression meshes (same topology, so a + blendshape = expr - neutral). Download from + https://github.com/USC-ICT/ICT-FaceKit/tree/master/FaceXModel + +OUTPUT arkit_template.bin — little-endian: + magic "QMFRT1\0\0" (8 bytes) + int32 vertexCount V + int32 faceCount F + int32 shapeCount S (== 52) + float32 neutral[V*3] (template neutral positions) + int32 faces[F*3] (triangle vertex indices) + then S shape records: + char[32] name (ASCII, NUL-padded — a FaceCap::kBlendshapeNames entry) + float32 delta[V*3] (expr - neutral; most verts are ~0) + ("_neutral" is NOT stored as a shape — it is the base.) + +The 52 ARKit names + the ICT->ARKit mapping are baked here; some ARKit +channels are SINGLE/centered (browInnerUp, cheekPuff, mouthClose, mouthFunnel, +mouthPucker, jaw*, mouthRoll*/Shrug*) while ICT splits a few as _L/_R — for +those the ARKit delta is the SUM of the ICT halves. The rest map 1:1. + +USAGE + python scripts/export-arkit-template.py --ict-dir .facerig_work/ict_full \ + --out .facerig_work/out/arkit_template.bin +""" + +import argparse +import os +import struct +import sys + +import numpy as np + +# 52 ARKit names in FaceCap::kBlendshapeNames order (index 0 "_neutral" is the +# base, not a shape). Each maps to a list of ICT stems whose deltas SUM to it. +ARKIT = [ + ("browDownLeft", ["browDown_L"]), ("browDownRight", ["browDown_R"]), + ("browInnerUp", ["browInnerUp_L", "browInnerUp_R"]), + ("browOuterUpLeft", ["browOuterUp_L"]), ("browOuterUpRight", ["browOuterUp_R"]), + ("cheekPuff", ["cheekPuff_L", "cheekPuff_R"]), + ("cheekSquintLeft", ["cheekSquint_L"]), ("cheekSquintRight", ["cheekSquint_R"]), + ("eyeBlinkLeft", ["eyeBlink_L"]), ("eyeBlinkRight", ["eyeBlink_R"]), + ("eyeLookDownLeft", ["eyeLookDown_L"]), ("eyeLookDownRight", ["eyeLookDown_R"]), + ("eyeLookInLeft", ["eyeLookIn_L"]), ("eyeLookInRight", ["eyeLookIn_R"]), + ("eyeLookOutLeft", ["eyeLookOut_L"]), ("eyeLookOutRight", ["eyeLookOut_R"]), + ("eyeLookUpLeft", ["eyeLookUp_L"]), ("eyeLookUpRight", ["eyeLookUp_R"]), + ("eyeSquintLeft", ["eyeSquint_L"]), ("eyeSquintRight", ["eyeSquint_R"]), + ("eyeWideLeft", ["eyeWide_L"]), ("eyeWideRight", ["eyeWide_R"]), + ("jawForward", ["jawForward"]), ("jawLeft", ["jawLeft"]), + ("jawOpen", ["jawOpen"]), ("jawRight", ["jawRight"]), + ("mouthClose", ["mouthClose"]), + ("mouthDimpleLeft", ["mouthDimple_L"]), ("mouthDimpleRight", ["mouthDimple_R"]), + ("mouthFrownLeft", ["mouthFrown_L"]), ("mouthFrownRight", ["mouthFrown_R"]), + ("mouthFunnel", ["mouthFunnel"]), ("mouthLeft", ["mouthLeft"]), + ("mouthLowerDownLeft", ["mouthLowerDown_L"]), + ("mouthLowerDownRight", ["mouthLowerDown_R"]), + ("mouthPressLeft", ["mouthPress_L"]), ("mouthPressRight", ["mouthPress_R"]), + ("mouthPucker", ["mouthPucker"]), ("mouthRight", ["mouthRight"]), + ("mouthRollLower", ["mouthRollLower"]), ("mouthRollUpper", ["mouthRollUpper"]), + ("mouthShrugLower", ["mouthShrugLower"]), ("mouthShrugUpper", ["mouthShrugUpper"]), + ("mouthSmileLeft", ["mouthSmile_L"]), ("mouthSmileRight", ["mouthSmile_R"]), + ("mouthStretchLeft", ["mouthStretch_L"]), ("mouthStretchRight", ["mouthStretch_R"]), + ("mouthUpperUpLeft", ["mouthUpperUp_L"]), ("mouthUpperUpRight", ["mouthUpperUp_R"]), + ("noseSneerLeft", ["noseSneer_L"]), ("noseSneerRight", ["noseSneer_R"]), +] + +MAGIC = b"QMFRT1\0\0" + + +def load_obj(path): + V, F = [], [] + with open(path) as f: + for ln in f: + if ln.startswith("v "): + V.append([float(x) for x in ln.split()[1:4]]) + elif ln.startswith("f "): + F.append([int(t.split("/")[0]) - 1 for t in ln.split()[1:4]]) + return np.asarray(V, np.float64), np.asarray(F, np.int32) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--ict-dir", required=True) + ap.add_argument("--out", default=".facerig_work/out/arkit_template.bin") + args = ap.parse_args() + + neutral, faces = load_obj(os.path.join(args.ict_dir, "generic_neutral_mesh.obj")) + V = len(neutral) + print(f"neutral: {V} verts / {len(faces)} tris") + + shapes = [] + for arkit_name, ict_stems in ARKIT: + delta = np.zeros((V, 3), np.float64) + missing = [] + for stem in ict_stems: + p = os.path.join(args.ict_dir, stem + ".obj") + if not os.path.exists(p): + missing.append(stem) + continue + ev, _ = load_obj(p) + if len(ev) != V: + sys.exit(f"topology mismatch: {stem} has {len(ev)} verts, " + f"neutral {V}") + delta += ev - neutral + if missing: + print(f" !! {arkit_name}: missing ICT {missing} — zero shape") + # count verts moved by a MEANINGFUL amount (0.1% of the head + # diagonal), not floating-point dust, so the log reflects real motion. + _diag = float(np.linalg.norm(neutral.max(0) - neutral.min(0))) + moved = int((np.linalg.norm(delta, axis=1) > 1e-3 * _diag).sum()) + shapes.append((arkit_name, delta.astype(np.float32))) + print(f" {arkit_name:22s} <- {'+'.join(ict_stems):24s} moved {moved} verts") + + os.makedirs(os.path.dirname(args.out), exist_ok=True) + with open(args.out, "wb") as f: + f.write(MAGIC) + f.write(struct.pack(" {args.out} ({size/1e6:.1f} MB)") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/upload-facerig-template.sh b/scripts/upload-facerig-template.sh new file mode 100755 index 000000000..6f32acd49 --- /dev/null +++ b/scripts/upload-facerig-template.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Upload the ARKit face-rig template to the QtMeshEditor HF models repo +# (epic #889, slice #890). ONE-TIME, run by a maintainer with write access. +# +# The app downloads this on first use from +# https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/facerig/arkit_template.bin +# so the file name + subdir MUST match ArkitTemplate::modelPath() / +# kDefaultModelBaseUrl. +# +# The template is derived from ICT-FaceKit (USC-ICT), MIT — ship the ICT MIT +# LICENSE next to it (see THIRD_PARTY_AI_MODELS.md). +# +# Prereqs: +# pip install -U huggingface_hub +# hf auth login # a token with write access to the repo +# scripts/export-arkit-template.py already run -> OUT holds arkit_template.bin +# +# Usage: +# OUT=.facerig_work/out ICT_LICENSE=.facerig_work/ICT_LICENSE.txt \ +# ./scripts/upload-facerig-template.sh +set -euo pipefail + +REPO="${REPO:-fernandotonon/QtMeshEditor-models}" +OUT="${OUT:?set OUT to the dir holding arkit_template.bin}" +ICT_LICENSE="${ICT_LICENSE:-}" + +upload() { # + local src="$1" dst="$2" + if [ -f "$src" ]; then + echo ">> uploading $src -> $REPO:$dst" + hf upload "$REPO" "$src" "$dst" + else + echo "!! skip (missing): $src" + fi +} + +upload "$OUT/arkit_template.bin" "facerig/arkit_template.bin" +[ -n "$ICT_LICENSE" ] && upload "$ICT_LICENSE" "facerig/ICT-FaceKit-LICENSE.txt" + +echo "done. Verify: curl -sI https://huggingface.co/$REPO/resolve/main/facerig/arkit_template.bin | head -1" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a06f61f83..ad9fdfc06 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -126,6 +126,7 @@ SkinWeightsController.cpp AutoRig.cpp AutoRigController.cpp UniRigPredictor.cpp +FaceRig/ArkitTemplate.cpp MotionInbetween.cpp MotionLibrary.cpp MotionGenerator.cpp diff --git a/src/FaceRig/ArkitTemplate.cpp b/src/FaceRig/ArkitTemplate.cpp new file mode 100644 index 000000000..bf65a6783 --- /dev/null +++ b/src/FaceRig/ArkitTemplate.cpp @@ -0,0 +1,185 @@ +#include "ArkitTemplate.h" + +#include "../ModelDownloader.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace FaceRig { + +namespace { +constexpr const char* kModelFile = "arkit_template.bin"; +constexpr const char* kMagic = "QMFRT1\0\0"; // 8 bytes +constexpr int kMagicLen = 8; +constexpr int kNameLen = 32; +constexpr const char* kDefaultModelBaseUrl = + "https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/facerig/"; +constexpr const char* kBaseUrlSettingsKey = "ai/facerigModelBaseUrl"; + +// little-endian readers over a byte cursor +int32_t rdI32(const char*& p) +{ + int32_t v; + std::memcpy(&v, p, 4); + p += 4; + return qFromLittleEndian(v); +} +float rdF32(const char*& p) +{ + quint32 raw; + std::memcpy(&raw, p, 4); + p += 4; + raw = qFromLittleEndian(raw); + float f; + std::memcpy(&f, &raw, 4); + return f; +} +} // namespace + +QStringList ArkitTemplate::shapeNames() const +{ + QStringList out; + for (const auto& s : m_shapes) + out << s.name; + return out; +} + +bool ArkitTemplate::load(const QString& path, QString* error) +{ + auto fail = [&](const QString& msg) { + if (error) + *error = msg; + return false; + }; + + QFile f(path); + if (!f.open(QIODevice::ReadOnly)) + return fail(QStringLiteral("cannot open %1").arg(path)); + const QByteArray blob = f.readAll(); + f.close(); + + // header: magic(8) + 3*int32 + if (blob.size() < kMagicLen + 12) + return fail(QStringLiteral("template too small / truncated")); + if (std::memcmp(blob.constData(), kMagic, kMagicLen) != 0) + return fail(QStringLiteral("bad magic (not an arkit_template.bin)")); + + const char* p = blob.constData() + kMagicLen; + const char* end = blob.constData() + blob.size(); + const int V = rdI32(p); + const int F = rdI32(p); + const int S = rdI32(p); + if (V <= 0 || F <= 0 || S <= 0 || V > 5'000'000 || S > 128) + return fail(QStringLiteral("implausible header (V=%1 F=%2 S=%3)") + .arg(V).arg(F).arg(S)); + + // exact byte budget check up front so a corrupt file can't over-read + const qint64 need = qint64(kMagicLen) + 12 + qint64(V) * 3 * 4 + + qint64(F) * 3 * 4 + + qint64(S) * (kNameLen + qint64(V) * 3 * 4); + if (blob.size() < need) + return fail(QStringLiteral("template truncated (need %1 bytes, have %2)") + .arg(need).arg(blob.size())); + + m_vertexCount = V; + m_faceCount = F; + m_neutral.resize(size_t(V) * 3); + for (auto& v : m_neutral) + v = rdF32(p); + m_faces.resize(size_t(F) * 3); + for (auto& i : m_faces) + i = rdI32(p); + + m_shapes.clear(); + m_shapes.reserve(S); + for (int s = 0; s < S; ++s) { + if (p + kNameLen > end) + return fail(QStringLiteral("shape %1 name overruns").arg(s)); + ArkitShape shape; + shape.name = QString::fromLatin1(p, qstrnlen(p, kNameLen)); + p += kNameLen; + shape.deltas.resize(size_t(V) * 3); + for (auto& d : shape.deltas) + d = rdF32(p); + m_shapes.push_back(std::move(shape)); + } + return true; +} + +QString ArkitTemplate::modelPath() +{ + const QString dataPath = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + return QDir(dataPath).filePath(QStringLiteral("ai_models/facerig/") + + QString::fromLatin1(kModelFile)); +} + +bool ArkitTemplate::present() { return QFileInfo::exists(modelPath()); } + +QString ArkitTemplate::ensureModelBlocking() +{ + const QString dest = modelPath(); + if (QFileInfo::exists(dest)) + return dest; + if (!qEnvironmentVariableIsEmpty("QTMESH_FACERIG_NO_DOWNLOAD")) + return {}; + + QString base; + { + QSettings s; + base = s.value(QString::fromLatin1(kBaseUrlSettingsKey)).toString(); + if (base.isEmpty()) { + const QByteArray env = qgetenv("QTMESH_FACERIG_MODEL_BASE_URL"); + base = env.isEmpty() ? QString::fromLatin1(kDefaultModelBaseUrl) + : QString::fromUtf8(env); + } + } + if (base.isEmpty()) + return {}; + if (!base.endsWith('/')) + base += '/'; + + auto* dl = ModelDownloader::instance(); + if (!dl) + return {}; + + QDir().mkpath(QFileInfo(dest).absolutePath()); + const QString url = base + QString::fromLatin1(kModelFile); + const QString label = QStringLiteral("ARKit face template"); + + QEventLoop loop; + bool ok = false, timedOut = false; + auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = true; loop.quit(); } + }); + auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = false; loop.quit(); } + }); + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&timeout, &QTimer::timeout, &loop, + [&]() { timedOut = true; loop.quit(); }); + timeout.start(300000); // 5 min — the template is ~17 MB + + dl->startDownload(url, dest, label); + loop.exec(); + + QObject::disconnect(onDone); + QObject::disconnect(onErr); + if (timedOut && dl) + dl->cancelDownload(); + + return (ok && !timedOut && QFileInfo::exists(dest)) ? dest : QString(); +} + +} // namespace FaceRig diff --git a/src/FaceRig/ArkitTemplate.h b/src/FaceRig/ArkitTemplate.h new file mode 100644 index 000000000..bba7c1519 --- /dev/null +++ b/src/FaceRig/ArkitTemplate.h @@ -0,0 +1,63 @@ +#ifndef ARKITTEMPLATE_H +#define ARKITTEMPLATE_H + +// The ARKit blendshape TEMPLATE that face auto-rig (#889) transfers onto a +// user mesh. It is the ICT-FaceKit generic-neutral head (MIT, USC-ICT) plus +// its 52 ARKit-named expression deltas, packed by scripts/export-arkit- +// template.py into one binary (arkit_template.bin) and downloaded on first +// use to AppData/ai_models/facerig/. +// +// Ogre-free, pure data — the NonRigidICP (#891) / DeformationTransfer (#892) +// stages consume it; headless-unit-tested. Shape names are the canonical +// FaceCap::kBlendshapeNames (the mocap-52 vocabulary), so the generated +// morph targets match what face capture drives. + +#include +#include + +#include +#include + +namespace FaceRig { + +struct ArkitShape { + QString name; // a FaceCap::kBlendshapeNames entry + std::vector deltas; // vertexCount*3, (expr - neutral) +}; + +class ArkitTemplate { +public: + bool valid() const { return m_vertexCount > 0 && !m_shapes.empty(); } + int vertexCount() const { return m_vertexCount; } + int faceCount() const { return m_faceCount; } + int shapeCount() const { return static_cast(m_shapes.size()); } + + // neutral positions, vertexCount*3 (x,y,z interleaved) + const std::vector& neutral() const { return m_neutral; } + // triangle vertex indices, faceCount*3 + const std::vector& faces() const { return m_faces; } + const std::vector& shapes() const { return m_shapes; } + QStringList shapeNames() const; + + // Load from an explicit arkit_template.bin path. + bool load(const QString& path, QString* error = nullptr); + + // ---- model management (the house pattern) ---------------------------- + static QString modelPath(); // AppData/ai_models/facerig/arkit_template.bin + static bool present(); + // Blocking first-use download; returns the path or empty + // (offline guard QTMESH_FACERIG_NO_DOWNLOAD; base URL override + // QTMESH_FACERIG_MODEL_BASE_URL / QSettings ai/facerigModelBaseUrl). + static QString ensureModelBlocking(); + +private: + int m_vertexCount = 0; + int m_faceCount = 0; + std::vector m_neutral; + std::vector m_faces; + std::vector m_shapes; +}; + +} // namespace FaceRig + +#endif // ARKITTEMPLATE_H diff --git a/src/FaceRig/ArkitTemplate_test.cpp b/src/FaceRig/ArkitTemplate_test.cpp new file mode 100644 index 000000000..6fdbea338 --- /dev/null +++ b/src/FaceRig/ArkitTemplate_test.cpp @@ -0,0 +1,142 @@ +#include + +#include +#include +#include +#include +#include + +#include "FaceRig/ArkitTemplate.h" + +#include + +namespace { + +// Write a minimal valid arkit_template.bin: V verts, F faces, S named shapes. +QString writeTemplate(const QString& dir, int V, int F, + const QStringList& shapeNames) +{ + QByteArray b; + auto putI32 = [&](int32_t v) { + int32_t le = qToLittleEndian(v); + b.append(reinterpret_cast(&le), 4); + }; + auto putF32 = [&](float f) { + quint32 raw; + std::memcpy(&raw, &f, 4); + raw = qToLittleEndian(raw); + b.append(reinterpret_cast(&raw), 4); + }; + b.append("QMFRT1\0\0", 8); + putI32(V); + putI32(F); + putI32(shapeNames.size()); + for (int i = 0; i < V * 3; ++i) + putF32(0.1f * i); // deterministic neutral + for (int i = 0; i < F * 3; ++i) + putI32(i % V); // dummy faces + for (int s = 0; s < shapeNames.size(); ++s) { + QByteArray nm = shapeNames[s].toLatin1().left(31); + b.append(nm); + b.append(QByteArray(32 - nm.size(), '\0')); + for (int i = 0; i < V * 3; ++i) + putF32(s == 0 && i == 1 ? -0.5f : 0.0f); // shape0 moves vert0.y + } + const QString path = dir + QStringLiteral("/arkit_template.bin"); + QFile f(path); + f.open(QIODevice::WriteOnly); + f.write(b); + f.close(); + return path; +} + +} // namespace + +TEST(ArkitTemplate, LoadsHeaderNeutralFacesShapes) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QStringList names{"jawOpen", "mouthSmileLeft", "eyeBlinkRight"}; + const QString path = writeTemplate(tmp.path(), 4, 2, names); + + FaceRig::ArkitTemplate t; + QString err; + ASSERT_TRUE(t.load(path, &err)) << err.toStdString(); + EXPECT_TRUE(t.valid()); + EXPECT_EQ(t.vertexCount(), 4); + EXPECT_EQ(t.faceCount(), 2); + EXPECT_EQ(t.shapeCount(), 3); + EXPECT_EQ(t.neutral().size(), 12u); + EXPECT_FLOAT_EQ(t.neutral()[3], 0.3f); // vert1.x = 0.1*3 + EXPECT_EQ(t.faces().size(), 6u); + EXPECT_EQ(t.shapeNames(), names); + // shape 0 ("jawOpen") moves vert0.y by -0.5 + EXPECT_FLOAT_EQ(t.shapes()[0].deltas[1], -0.5f); + EXPECT_FLOAT_EQ(t.shapes()[1].deltas[1], 0.0f); +} + +TEST(ArkitTemplate, RejectsBadMagic) +{ + QTemporaryDir tmp; + const QString path = tmp.path() + "/bad.bin"; + QFile f(path); + f.open(QIODevice::WriteOnly); + f.write(QByteArray("NOTMAGIC", 8) + QByteArray(64, '\0')); + f.close(); + FaceRig::ArkitTemplate t; + QString err; + EXPECT_FALSE(t.load(path, &err)); + EXPECT_FALSE(err.isEmpty()); +} + +TEST(ArkitTemplate, RejectsTruncated) +{ + QTemporaryDir tmp; + const QString good = writeTemplate(tmp.path(), 4, 2, {"jawOpen"}); + QFile f(good); + f.open(QIODevice::ReadOnly); + QByteArray full = f.readAll(); + f.close(); + const QString path = tmp.path() + "/trunc.bin"; + QFile o(path); + o.open(QIODevice::WriteOnly); + o.write(full.left(full.size() - 20)); // chop the last shape's deltas + o.close(); + FaceRig::ArkitTemplate t; + QString err; + EXPECT_FALSE(t.load(path, &err)); + EXPECT_TRUE(err.contains("truncat", Qt::CaseInsensitive)); +} + +TEST(ArkitTemplate, MissingFileFails) +{ + FaceRig::ArkitTemplate t; + QString err; + EXPECT_FALSE(t.load("/nonexistent/arkit_template.bin", &err)); + EXPECT_FALSE(err.isEmpty()); + EXPECT_FALSE(t.valid()); +} + +// Env-gated: run against the REAL bundle (set QTMESH_FACERIG_TEMPLATE to the +// exported arkit_template.bin) — verifies the 51 shapes + ARKit names. +TEST(ArkitTemplate, EnvGatedRealBundle) +{ + const QByteArray p = qgetenv("QTMESH_FACERIG_TEMPLATE"); + if (p.isEmpty()) + GTEST_SKIP() << "set QTMESH_FACERIG_TEMPLATE to arkit_template.bin"; + FaceRig::ArkitTemplate t; + QString err; + ASSERT_TRUE(t.load(QString::fromUtf8(p), &err)) << err.toStdString(); + EXPECT_GT(t.vertexCount(), 10000); // ICT is ~26.7k + EXPECT_GE(t.shapeCount(), 51); + EXPECT_TRUE(t.shapeNames().contains("jawOpen")); + EXPECT_TRUE(t.shapeNames().contains("mouthSmileLeft")); + EXPECT_TRUE(t.shapeNames().contains("eyeBlinkLeft")); + // jawOpen should actually deform (nonzero deltas) + const auto& d = t.shapes()[t.shapeNames().indexOf("jawOpen")].deltas; + float maxMag = 0.f; + for (size_t i = 0; i + 2 < d.size(); i += 3) + maxMag = std::max(maxMag, + std::abs(d[i]) + std::abs(d[i + 1]) + std::abs(d[i + 2])); + EXPECT_GT(maxMag, 0.f); +} diff --git a/src/test_main.cpp b/src/test_main.cpp index 8ec3dd95b..41c91be9c 100644 --- a/src/test_main.cpp +++ b/src/test_main.cpp @@ -139,12 +139,21 @@ int main(int argc, char **argv) // Prove headless GL works on this runner, then tear down: many suites // (Assimp processors, etc.) construct their own Ogre::Root and cannot // coexist with a live Manager singleton from a prior init. - if (!tryInitOgre()) { + // QTMESH_TESTS_SKIP_OGRE_PREFLIGHT=1 skips the GL proof so PURE-DATA suites + // can run (with --gtest_filter) on machines with no GL/WindowServer at all + // (remote shells, containers without Xvfb). Ogre-dependent fixtures still + // fail under it — this only moves the failure from "no test ran" to + // per-fixture. CI never sets it. + if (qEnvironmentVariableIsSet("QTMESH_TESTS_SKIP_OGRE_PREFLIGHT")) { + fprintf(stderr, "UnitTests: skipping Ogre GL preflight " + "(QTMESH_TESTS_SKIP_OGRE_PREFLIGHT set)\n"); + } else if (!tryInitOgre()) { fprintf(stderr, "UnitTests FATAL: tryInitOgre() failed — need working DISPLAY / Xvfb for GL.\n"); return 1; + } else { + Manager::kill(); } - Manager::kill(); if (QCoreApplication::instance()) QCoreApplication::processEvents(); QThread::msleep(50); From ca318634b33af1737a2c46d170049c660f9f27b5 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 09:07:00 -0300 Subject: [PATCH 03/38] =?UTF-8?q?feat(#891):=20NonRigidICP=20=E2=80=94=20n?= =?UTF-8?q?ative=20template->user=20fit=20(Face-Rig=20Slice=20C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice C of epic #889. Pure-data, Ogre-free, ZERO new dependencies, headless-tested — the native C++ port of the NRICP the spike (#897) proved. - src/FaceRig/NonRigidICP.{h,cpp}: Amberg 2007 optimal-step non-rigid ICP. Fits a template mesh to a user neutral head of arbitrary topology, producing per-template-vertex positions on the user surface (the correspondence the deformation transfer #892 consumes). Per-vertex 3x4 affine A_i; minimizes data (A_i·v_i -> closest user-surface point) + annealed stiffness (alpha·(A_i - A_j) over template edges, 50->0.5). Self-contained: * median-split KD-tree over user-triangle centroids (broad phase), * Ericson point-to-triangle projection (exact closest point), * CSR sparse matrix + conjugate-gradient on the normal equations (AtA x = At b) — no Eigen, no scipy, nothing vendored. Reports per-vertex residual + mean/max + bbox diag so callers can gate on fit quality (the humanoid check in Slice E). Rigid centroid+bbox-scale pre-align, then the non-rigid warp. - 6 headless tests: fit-to-self near-identity, fit onto a translated target, **fit onto a DIFFERENT-topology target** (10x10 template -> 14x14 user, <5% residual, finite), stiffness-bounded under noise, degenerate input no-NaN, residual/diag reporting. All pass (~5-13 ms each). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/CMakeLists.txt | 1 + src/FaceRig/NonRigidICP.cpp | 334 +++++++++++++++++++++++++++++++ src/FaceRig/NonRigidICP.h | 56 ++++++ src/FaceRig/NonRigidICP_test.cpp | 127 ++++++++++++ 4 files changed, 518 insertions(+) create mode 100644 src/FaceRig/NonRigidICP.cpp create mode 100644 src/FaceRig/NonRigidICP.h create mode 100644 src/FaceRig/NonRigidICP_test.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ad9fdfc06..9ca00dabd 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -127,6 +127,7 @@ AutoRig.cpp AutoRigController.cpp UniRigPredictor.cpp FaceRig/ArkitTemplate.cpp +FaceRig/NonRigidICP.cpp MotionInbetween.cpp MotionLibrary.cpp MotionGenerator.cpp diff --git a/src/FaceRig/NonRigidICP.cpp b/src/FaceRig/NonRigidICP.cpp new file mode 100644 index 000000000..3fcdd5c98 --- /dev/null +++ b/src/FaceRig/NonRigidICP.cpp @@ -0,0 +1,334 @@ +#include "NonRigidICP.h" + +#include +#include +#include +#include + +namespace FaceRig { + +namespace { + +using Vec3 = std::array; + +Vec3 vsub(const Vec3& a, const Vec3& b) { return {a[0]-b[0], a[1]-b[1], a[2]-b[2]}; } +double vdot(const Vec3& a, const Vec3& b) { return a[0]*b[0]+a[1]*b[1]+a[2]*b[2]; } +Vec3 vadd(const Vec3& a, const Vec3& b) { return {a[0]+b[0], a[1]+b[1], a[2]+b[2]}; } +Vec3 vscale(const Vec3& a, double s) { return {a[0]*s, a[1]*s, a[2]*s}; } + +Vec3 at(const std::vector& v, int i) +{ + return {v[size_t(i)*3], v[size_t(i)*3+1], v[size_t(i)*3+2]}; +} + +// ---- closest point on a triangle (Ericson, Real-Time Collision Detection) -- +Vec3 closestPointTriangle(const Vec3& p, const Vec3& a, const Vec3& b, const Vec3& c) +{ + const Vec3 ab = vsub(b, a), ac = vsub(c, a), ap = vsub(p, a); + const double d1 = vdot(ab, ap), d2 = vdot(ac, ap); + if (d1 <= 0 && d2 <= 0) return a; + const Vec3 bp = vsub(p, b); + const double d3 = vdot(ab, bp), d4 = vdot(ac, bp); + if (d3 >= 0 && d4 <= d3) return b; + const double vc = d1*d4 - d3*d2; + if (vc <= 0 && d1 >= 0 && d3 <= 0) return vadd(a, vscale(ab, d1/(d1-d3))); + const Vec3 cp = vsub(p, c); + const double d5 = vdot(ab, cp), d6 = vdot(ac, cp); + if (d6 >= 0 && d5 <= d6) return c; + const double vb = d5*d2 - d1*d6; + if (vb <= 0 && d2 >= 0 && d6 <= 0) return vadd(a, vscale(ac, d2/(d2-d6))); + const double va = d3*d6 - d5*d4; + if (va <= 0 && (d4-d3) >= 0 && (d5-d6) >= 0) + return vadd(b, vscale(vsub(c, b), (d4-d3)/((d4-d3)+(d5-d6)))); + const double denom = 1.0/(va+vb+vc); + return vadd(a, vadd(vscale(ab, vb*denom), vscale(ac, vc*denom))); +} + +// ---- a simple median-split KD-tree over triangle centroids ----------------- +struct KDTree { + std::vector pts; + std::vector idx; + struct Node { int axis=-1; double split=0; int lo=-1, hi=-1, start=0, count=0; }; + std::vector nodes; + + void build(const std::vector& centroids) + { + pts = centroids; + idx.resize(pts.size()); + for (size_t i = 0; i < idx.size(); ++i) idx[i] = int(i); + nodes.clear(); + buildRange(0, int(idx.size())); + } + int buildRange(int start, int count) + { + const int self = int(nodes.size()); + nodes.push_back({}); + if (count <= 8) { nodes[self] = {-1, 0, -1, -1, start, count}; return self; } + Vec3 mn = pts[idx[start]], mx = pts[idx[start]]; + for (int k = 0; k < count; ++k) { + const Vec3& p = pts[idx[start+k]]; + for (int a = 0; a < 3; ++a) { mn[a] = std::min(mn[a], p[a]); mx[a] = std::max(mx[a], p[a]); } + } + int axis = 0; double ext = mx[0]-mn[0]; + for (int a = 1; a < 3; ++a) if (mx[a]-mn[a] > ext) { ext = mx[a]-mn[a]; axis = a; } + const int mid = start + count/2; + std::nth_element(idx.begin()+start, idx.begin()+mid, idx.begin()+start+count, + [&](int x, int y){ return pts[x][axis] < pts[y][axis]; }); + const double split = pts[idx[mid]][axis]; + const int lo = buildRange(start, mid-start); + const int hi = buildRange(mid, start+count-mid); + nodes[self] = {axis, split, lo, hi, 0, 0}; + return self; + } + // nearest centroid index (broad phase); refine to real distance by caller + int nearest(const Vec3& q) const + { + int best = -1; double bestD2 = std::numeric_limits::max(); + nearestRec(0, q, best, bestD2); + return best; + } + void nearestRec(int n, const Vec3& q, int& best, double& bestD2) const + { + const Node& nd = nodes[n]; + if (nd.axis < 0) { + for (int k = 0; k < nd.count; ++k) { + const int pi = idx[nd.start+k]; + const Vec3 d = vsub(pts[pi], q); + const double d2 = vdot(d, d); + if (d2 < bestD2) { bestD2 = d2; best = pi; } + } + return; + } + const double diff = q[nd.axis] - nd.split; + const int near = diff < 0 ? nd.lo : nd.hi; + const int far = diff < 0 ? nd.hi : nd.lo; + nearestRec(near, q, best, bestD2); + if (diff*diff < bestD2) nearestRec(far, q, best, bestD2); + } +}; + +// ---- CSR sparse matrix + CG on the normal equations (AᵀA x = Aᵀb) ---------- +// A is (rows x cols); we never form AᵀA — CG multiplies by A then Aᵀ. +struct Sparse { + int rows = 0, cols = 0; + std::vector rowPtr; // size rows+1 + std::vector col; + std::vector val; + + // build from triplets (row-major). triplets need not be unique/sorted. + void fromTriplets(int r, int c, std::vector>& trip) + { + rows = r; cols = c; + std::vector cnt(r+1, 0); + for (auto& t : trip) cnt[int(t[0])+1]++; + for (int i = 0; i < r; ++i) cnt[i+1] += cnt[i]; + rowPtr = cnt; + col.resize(trip.size()); val.resize(trip.size()); + std::vector cur = rowPtr; + for (auto& t : trip) { + const int rr = int(t[0]); + const int dst = cur[rr]++; + col[dst] = int(t[1]); val[dst] = t[2]; + } + } + // y = A x (x size cols, y size rows) + void mul(const std::vector& x, std::vector& y) const + { + y.assign(rows, 0.0); + for (int r = 0; r < rows; ++r) { + double s = 0; + for (int k = rowPtr[r]; k < rowPtr[r+1]; ++k) s += val[k]*x[col[k]]; + y[r] = s; + } + } + // y = Aᵀ x (x size rows, y size cols) + void mulT(const std::vector& x, std::vector& y) const + { + y.assign(cols, 0.0); + for (int r = 0; r < rows; ++r) { + const double xr = x[r]; + for (int k = rowPtr[r]; k < rowPtr[r+1]; ++k) y[col[k]] += val[k]*xr; + } + } +}; + +// solve min ‖A x - b‖² by CG on the normal equations, warm-started at x0. +void cgnr(const Sparse& A, const std::vector& b, + std::vector& x, int maxIters, double tol) +{ + std::vector Ax, r(A.cols), p, Ap, AtAp, tmp; + A.mul(x, Ax); + std::vector resid(A.rows); + for (int i = 0; i < A.rows; ++i) resid[i] = b[i] - Ax[i]; + A.mulT(resid, r); // r = Aᵀ(b - Ax) + p = r; + double rs = 0; for (double v : r) rs += v*v; + const double rs0 = rs; + for (int it = 0; it < maxIters && rs > tol*tol*rs0; ++it) { + A.mul(p, Ap); + A.mulT(Ap, AtAp); // AtAp = AᵀA p + double pAp = 0; for (int i = 0; i < A.cols; ++i) pAp += p[i]*AtAp[i]; + if (pAp <= 1e-30) break; + const double a = rs / pAp; + for (int i = 0; i < A.cols; ++i) { x[i] += a*p[i]; r[i] -= a*AtAp[i]; } + double rsn = 0; for (double v : r) rsn += v*v; + const double beta = rsn / rs; + for (int i = 0; i < A.cols; ++i) p[i] = r[i] + beta*p[i]; + rs = rsn; + } +} + +} // namespace + +NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, + const std::vector& userV, const std::vector& userF, + const NricpOptions& opts) +{ + NricpResult res; + const int Nt = int(tmplV.size()/3); + const int Fu = int(userF.size()/3); + if (Nt < 3 || Fu < 1 || userV.size() < 9) + return res; + + // user bbox diagonal + Vec3 mn = at(userV,0), mx = mn; + for (int i = 1; i < int(userV.size()/3); ++i) { + const Vec3 p = at(userV, i); + for (int a = 0; a < 3; ++a) { mn[a] = std::min(mn[a], p[a]); mx[a] = std::max(mx[a], p[a]); } + } + res.diag = std::sqrt(vdot(vsub(mx,mn), vsub(mx,mn))); + + // rigid pre-align: centroid + bbox-scale (correspondence-free) + Vec3 tmn = at(tmplV,0), tmx = tmn, tc{0,0,0}, uc{0,0,0}; + for (int i = 0; i < Nt; ++i) { + const Vec3 p = at(tmplV,i); tc = vadd(tc,p); + for (int a=0;a<3;++a){ tmn[a]=std::min(tmn[a],p[a]); tmx[a]=std::max(tmx[a],p[a]); } + } + tc = vscale(tc, 1.0/Nt); + const int Nu = int(userV.size()/3); + for (int i = 0; i < Nu; ++i) uc = vadd(uc, at(userV,i)); + uc = vscale(uc, 1.0/Nu); + const double tdiag = std::sqrt(vdot(vsub(tmx,tmn), vsub(tmx,tmn))); + const double s = tdiag > 1e-9 ? res.diag/tdiag : 1.0; + + // template homogeneous verts and current fitted positions X + std::vector vhat(Nt), X(Nt); + for (int i = 0; i < Nt; ++i) { + vhat[i] = at(tmplV, i); + X[i] = vadd(vscale(vsub(vhat[i], tc), s), uc); // pre-aligned start + } + + // user triangle geometry + centroid KD-tree + std::vector> utri(Fu); + std::vector ucent(Fu); + for (int f = 0; f < Fu; ++f) { + const Vec3 a = at(userV, userF[f*3]); + const Vec3 b = at(userV, userF[f*3+1]); + const Vec3 c = at(userV, userF[f*3+2]); + utri[f] = {a,b,c}; + ucent[f] = vscale(vadd(vadd(a,b),c), 1.0/3.0); + } + KDTree tree; tree.build(ucent); + + // template edges (unique) + std::vector> edges; + { + std::vector> e; + const int Ft = int(tmplF.size()/3); + e.reserve(Ft*3); + for (int f = 0; f < Ft; ++f) { + const int a=tmplF[f*3], b=tmplF[f*3+1], c=tmplF[f*3+2]; + for (auto pr : {std::array{a,b}, {b,c}, {c,a}}) { + int lo = std::min(pr[0],pr[1]), hi = std::max(pr[0],pr[1]); + e.push_back({lo,hi}); + } + } + std::sort(e.begin(), e.end()); + e.erase(std::unique(e.begin(), e.end()), e.end()); + for (auto& pr : e) edges.push_back({pr[0], pr[1]}); + } + const int E = int(edges.size()); + + // unknown layout: 12 per vertex (3x4 affine, row-major a00..a03,a10..,a20..) + // X_i = A_i * [vhat_i; 1]. We solve 3 independent systems (one per output + // coordinate), each with 4*Nt unknowns (the 4 affine coeffs mapping to that + // coord for every vertex), sharing the SAME sparse matrix. + const int cols = 4*Nt; + + for (double alpha : opts.stiffness) { + for (int iter = 0; iter < opts.itersPerLevel; ++iter) { + // find closest surface point per current X_i + std::vector target(Nt); + for (int i = 0; i < Nt; ++i) { + const int cf = tree.nearest(X[i]); + // refine: check that triangle + a few neighbors would need the + // full tree; centroid-nearest triangle is a good approximation + // for a fitted template already close to the surface. + target[i] = closestPointTriangle(X[i], utri[cf][0], utri[cf][1], utri[cf][2]); + } + + // assemble A (rows = Nt data + E stiffness) once; rhs differs per axis + std::vector> trip; + trip.reserve(size_t(Nt)*4 + size_t(E)*8); + // data rows: row i uses cols [4i..4i+3] with [vx,vy,vz,1] + for (int i = 0; i < Nt; ++i) { + trip.push_back({double(i), double(4*i+0), vhat[i][0]}); + trip.push_back({double(i), double(4*i+1), vhat[i][1]}); + trip.push_back({double(i), double(4*i+2), vhat[i][2]}); + trip.push_back({double(i), double(4*i+3), 1.0}); + } + // stiffness rows: for edge (i,j), 4 rows (one per affine coeff) + // alpha*(A_i[k]-A_j[k]) = 0 + for (int e = 0; e < E; ++e) { + const int i = edges[e].first, j = edges[e].second; + for (int k = 0; k < 4; ++k) { + const int row = Nt + e*4 + k; + trip.push_back({double(row), double(4*i+k), alpha}); + trip.push_back({double(row), double(4*j+k), -alpha}); + } + } + Sparse A; + A.fromTriplets(Nt + E*4, cols, trip); + + std::vector b(A.rows, 0.0); + std::vector x(cols, 0.0); + for (int axis = 0; axis < 3; ++axis) { + for (int i = 0; i < Nt; ++i) b[i] = target[i][axis]; + // stiffness rhs stays 0 + // warm start x from the current affine estimate for this axis + for (int i = 0; i < Nt; ++i) { + // recover current A_i row for this axis from X_i & vhat_i is + // non-trivial; a zero start with identity bias works well + x[4*i+0] = 0; x[4*i+1] = 0; x[4*i+2] = 0; x[4*i+3] = X[i][axis]; + x[4*i+axis] = 1.0; // identity-ish seed + } + cgnr(A, b, x, opts.cgIters, opts.cgTol); + for (int i = 0; i < Nt; ++i) { + X[i][axis] = x[4*i+0]*vhat[i][0] + x[4*i+1]*vhat[i][1] + + x[4*i+2]*vhat[i][2] + x[4*i+3]; + } + } + } + } + + // final residuals to the user surface + res.fitted.resize(size_t(Nt)*3); + res.residual.resize(Nt); + double sum = 0, mx2 = 0; + for (int i = 0; i < Nt; ++i) { + const int cf = tree.nearest(X[i]); + const Vec3 cp = closestPointTriangle(X[i], utri[cf][0], utri[cf][1], utri[cf][2]); + const double d = std::sqrt(vdot(vsub(X[i],cp), vsub(X[i],cp))); + res.residual[i] = float(d); + sum += d; mx2 = std::max(mx2, d); + res.fitted[size_t(i)*3+0] = float(X[i][0]); + res.fitted[size_t(i)*3+1] = float(X[i][1]); + res.fitted[size_t(i)*3+2] = float(X[i][2]); + } + res.meanResidual = sum/Nt; + res.maxResidual = mx2; + res.ok = true; + return res; +} + +} // namespace FaceRig diff --git a/src/FaceRig/NonRigidICP.h b/src/FaceRig/NonRigidICP.h new file mode 100644 index 000000000..dc63e0f25 --- /dev/null +++ b/src/FaceRig/NonRigidICP.h @@ -0,0 +1,56 @@ +#ifndef NONRIGIDICP_H +#define NONRIGIDICP_H + +// Non-rigid ICP (Amberg, Romdhani & Vetter 2007, "Optimal Step Nonrigid ICP") +// for face auto-rig (#889, Slice C #891). Pure data — no Ogre, no ONNX, no +// external linear-algebra dependency (a self-contained sparse CG solver lives +// in the .cpp) — and headless-unit-tested. +// +// Fits a TEMPLATE mesh (the ICT ARKit head, ArkitTemplate) to a USER neutral +// head of arbitrary topology, producing per-template-vertex positions that lie +// on the user surface: the CORRESPONDENCE the deformation transfer (#892) +// needs. Each template vertex gets a 3x4 affine A_i; we minimize +// +// Σ_i ‖A_i·ṽ_i − closest_point_on_user(A_i·ṽ_i)‖² (data) +// + α Σ_(i,j)∈edges ‖A_i − A_j‖² (stiffness) +// +// over an annealed stiffness schedule α (high→low). closest_point is a +// point-to-triangle projection against the user mesh (KD-tree over triangle +// centroids for the broad phase). Contract proven in docs/FACE_RIG_SPIKE.md. + +#include +#include + +namespace FaceRig { + +struct NricpResult { + // fitted template vertex positions on the user surface (templateVertexCount + // * 3, xyz interleaved) — the correspondence. + std::vector fitted; + // per-template-vertex residual distance to the user surface (in mesh units) + std::vector residual; + double meanResidual = 0.0; // over all template verts + double maxResidual = 0.0; + double diag = 0.0; // user-mesh bounding-box diagonal (for %) + bool ok = false; +}; + +struct NricpOptions { + // annealed stiffness weights (high = rigid, low = free); ~3 inner iters each + std::vector stiffness = {50, 20, 8, 3, 1, 0.5}; + int itersPerLevel = 3; + int cgIters = 400; // CG cap per axis solve + double cgTol = 1e-6; +}; + +// tmplV/tmplF: template neutral verts (Nt*3) + tris (Ft*3). +// userV/userF: user neutral verts (Nu*3) + tris (Fu*3). +// Both assumed roughly aligned in orientation (+Y up); the fit does a +// centroid+bbox-scale rigid pre-align, then the non-rigid warp. +NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, + const std::vector& userV, const std::vector& userF, + const NricpOptions& opts = {}); + +} // namespace FaceRig + +#endif // NONRIGIDICP_H diff --git a/src/FaceRig/NonRigidICP_test.cpp b/src/FaceRig/NonRigidICP_test.cpp new file mode 100644 index 000000000..800c0c11c --- /dev/null +++ b/src/FaceRig/NonRigidICP_test.cpp @@ -0,0 +1,127 @@ +#include + +#include "FaceRig/NonRigidICP.h" + +#include +#include +#include +#include + +namespace { + +// A small closed-ish grid surface (a bumpy plane) as a stand-in mesh: a +// (n x n) vertex grid triangulated, so it has real edges/faces for NRICP. +struct Grid { + std::vector V; + std::vector F; +}; + +Grid makeGrid(int n, float extent, float bump = 0.0f, float dx = 0.0f) +{ + Grid g; + for (int y = 0; y < n; ++y) + for (int x = 0; x < n; ++x) { + const float fx = (float(x)/(n-1) - 0.5f) * extent + dx; + const float fy = (float(y)/(n-1) - 0.5f) * extent; + const float fz = bump * std::sin(float(x)) * std::cos(float(y)); + g.V.insert(g.V.end(), {fx, fy, fz}); + } + for (int y = 0; y < n-1; ++y) + for (int x = 0; x < n-1; ++x) { + const int a = y*n+x, b = y*n+x+1, c = (y+1)*n+x, d = (y+1)*n+x+1; + g.F.insert(g.F.end(), {a, b, c}); + g.F.insert(g.F.end(), {b, d, c}); + } + return g; +} + +double maxAbs(const std::vector& v) +{ + double m = 0; + for (float x : v) m = std::max(m, double(std::abs(x))); + return m; +} + +} // namespace + +TEST(NonRigidICP, FitToIdenticalMeshIsNearIdentity) +{ + const Grid g = makeGrid(10, 2.0f); + const auto res = FaceRig::fit(g.V, g.F, g.V, g.F); + ASSERT_TRUE(res.ok); + ASSERT_EQ(res.fitted.size(), g.V.size()); + // fitting a mesh onto ITSELF: fitted verts land on the surface (residual ~0) + EXPECT_LT(res.meanResidual / res.diag, 0.02); // < 2% of diag + // and stay close to their original positions + double maxMove = 0; + for (size_t i = 0; i < g.V.size(); ++i) + maxMove = std::max(maxMove, double(std::abs(res.fitted[i] - g.V[i]))); + EXPECT_LT(maxMove / res.diag, 0.1); +} + +TEST(NonRigidICP, FitsOntoTranslatedTarget) +{ + const Grid tmpl = makeGrid(10, 2.0f); + // user = same surface shifted +0.5 in X (a different "identity") + Grid user = makeGrid(10, 2.0f, 0.0f, 0.5f); + const auto res = FaceRig::fit(tmpl.V, tmpl.F, user.V, user.F); + ASSERT_TRUE(res.ok); + // the fitted template should end up on the shifted plane (low residual) + EXPECT_LT(res.meanResidual / res.diag, 0.05); + // and its mean X should have moved toward the user's (+0.5) + double mx = 0; int n = int(res.fitted.size()/3); + for (int i = 0; i < n; ++i) mx += res.fitted[i*3]; + mx /= n; + EXPECT_GT(mx, 0.2); // shifted from ~0 toward +0.5 +} + +TEST(NonRigidICP, FitsOntoDifferentTopologyTarget) +{ + // template 10x10, user 14x14 of the same surface — different vert counts + const Grid tmpl = makeGrid(10, 2.0f, 0.15f); + const Grid user = makeGrid(14, 2.0f, 0.15f); + const auto res = FaceRig::fit(tmpl.V, tmpl.F, user.V, user.F); + ASSERT_TRUE(res.ok); + EXPECT_EQ(int(res.fitted.size()/3), int(tmpl.V.size()/3)); + EXPECT_LT(res.meanResidual / res.diag, 0.05); // fits despite topology gap + for (float x : res.fitted) EXPECT_TRUE(std::isfinite(x)); +} + +TEST(NonRigidICP, BoundedOnPerturbedTarget) +{ + const Grid tmpl = makeGrid(10, 2.0f); + Grid user = makeGrid(10, 2.0f); + std::mt19937 rng(7); + std::normal_distribution jitter(0.f, 0.03f); + for (auto& v : user.V) v += jitter(rng); + const auto res = FaceRig::fit(tmpl.V, tmpl.F, user.V, user.F); + ASSERT_TRUE(res.ok); + // stiffness keeps the fit from chasing every jitter spike + EXPECT_LT(res.maxResidual / res.diag, 0.2); + for (float x : res.fitted) EXPECT_TRUE(std::isfinite(x)); +} + +TEST(NonRigidICP, DegenerateInputDoesNotCrashOrNaN) +{ + const Grid g = makeGrid(6, 1.0f); + // empty template + EXPECT_FALSE(FaceRig::fit({}, {}, g.V, g.F).ok); + // empty user + EXPECT_FALSE(FaceRig::fit(g.V, g.F, {}, {}).ok); + // user with a single degenerate (zero-area) triangle + std::vector uv = {0,0,0, 0,0,0, 0,0,0}; + std::vector uf = {0,1,2}; + const auto res = FaceRig::fit(g.V, g.F, uv, uf); + // may or may not be "ok" but must never produce NaN + for (float x : res.fitted) EXPECT_TRUE(std::isfinite(x)); +} + +TEST(NonRigidICP, ReportsResidualAndDiag) +{ + const Grid g = makeGrid(8, 3.0f); + const auto res = FaceRig::fit(g.V, g.F, g.V, g.F); + ASSERT_TRUE(res.ok); + EXPECT_GT(res.diag, 0.0); + EXPECT_EQ(int(res.residual.size()), int(g.V.size()/3)); + EXPECT_GE(res.maxResidual, res.meanResidual); +} From fc24e18755730dd4f410f6d1e8c8c285dede8f02 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 09:20:50 -0300 Subject: [PATCH 04/38] =?UTF-8?q?feat(#892):=20DeformationTransfer=20?= =?UTF-8?q?=E2=80=94=20Sumner-Popovi=C4=87=20shape=20transfer=20(Face-Rig?= =?UTF-8?q?=20Slice=20D)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transfer each ARKit template expression's per-triangle deformation onto the user-identity mesh produced by the NRICP fit (#891), in template topology. - SparseSolve.{h,cpp}: dependency-free CSR matrix (fromTriplets/mul/mulT) + conjugate-gradient on the normal equations (CGNR), factored out for reuse. No Eigen, no external solver. - DeformationTransfer.{h,cpp}: per source triangle builds the deformation gradient S = [e1' e2' n']·[e1 e2 n]⁻¹ (Sumner's 4th "normal" vertex trick), then solves one sparse least-squares over the fitted mesh for vertex positions whose per-triangle gradient matches S. The 4th vertex is a free per-triangle unknown; the gauge is fixed by anchoring a single vertex to the source's own vertex-0 displacement (anchoring every vertex fights the shape). init() precomputes the topology-fixed system once so all 52 shapes transfer without rebuilding. - DeformationTransfer_test.cpp: identity round-trip near-identity (<15% RMS), scale-covariant transfer onto a scaled identity with sign/winding preserved, zero-delta→zero, bad-input rejection. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/CMakeLists.txt | 2 + src/FaceRig/DeformationTransfer.cpp | 234 +++++++++++++++++++++++ src/FaceRig/DeformationTransfer.h | 69 +++++++ src/FaceRig/DeformationTransfer_test.cpp | 149 +++++++++++++++ src/FaceRig/SparseSolve.cpp | 87 +++++++++ src/FaceRig/SparseSolve.h | 39 ++++ 6 files changed, 580 insertions(+) create mode 100644 src/FaceRig/DeformationTransfer.cpp create mode 100644 src/FaceRig/DeformationTransfer.h create mode 100644 src/FaceRig/DeformationTransfer_test.cpp create mode 100644 src/FaceRig/SparseSolve.cpp create mode 100644 src/FaceRig/SparseSolve.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9ca00dabd..bc0b4aacf 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -128,6 +128,8 @@ AutoRigController.cpp UniRigPredictor.cpp FaceRig/ArkitTemplate.cpp FaceRig/NonRigidICP.cpp +FaceRig/SparseSolve.cpp +FaceRig/DeformationTransfer.cpp MotionInbetween.cpp MotionLibrary.cpp MotionGenerator.cpp diff --git a/src/FaceRig/DeformationTransfer.cpp b/src/FaceRig/DeformationTransfer.cpp new file mode 100644 index 000000000..ffcff82fc --- /dev/null +++ b/src/FaceRig/DeformationTransfer.cpp @@ -0,0 +1,234 @@ +#include "DeformationTransfer.h" + +#include +#include + +namespace FaceRig { + +namespace { + +using Vec3 = std::array; +using Mat3 = std::array; // row-major + +Vec3 sub(const Vec3& a, const Vec3& b) { return {a[0]-b[0], a[1]-b[1], a[2]-b[2]}; } +Vec3 cross(const Vec3& a, const Vec3& b) +{ + return {a[1]*b[2]-a[2]*b[1], a[2]*b[0]-a[0]*b[2], a[0]*b[1]-a[1]*b[0]}; +} +double norm(const Vec3& a) { return std::sqrt(a[0]*a[0]+a[1]*a[1]+a[2]*a[2]); } + +Vec3 vert(const std::vector& v, int i) +{ + return {v[size_t(i)*3], v[size_t(i)*3+1], v[size_t(i)*3+2]}; +} + +// 4th "normal" vertex: v4 = v1 + n/√|n|, n = (v2-v1)×(v3-v1) +Vec3 normalV4(const Vec3& a, const Vec3& b, const Vec3& c) +{ + const Vec3 n = cross(sub(b, a), sub(c, a)); + const double ln = norm(n); + const double s = ln > 1e-12 ? 1.0 / std::sqrt(ln) : 0.0; + return {a[0]+n[0]*s, a[1]+n[1]*s, a[2]+n[2]*s}; +} + +// frame V = [v2-v1, v3-v1, v4-v1] as columns (row-major 3x3) +Mat3 frame(const Vec3& a, const Vec3& b, const Vec3& c, const Vec3& d) +{ + const Vec3 e1 = sub(b,a), e2 = sub(c,a), e3 = sub(d,a); + return {e1[0], e2[0], e3[0], + e1[1], e2[1], e3[1], + e1[2], e2[2], e3[2]}; +} + +bool invert3(const Mat3& m, Mat3& out) +{ + const double det = + m[0]*(m[4]*m[8]-m[5]*m[7]) - m[1]*(m[3]*m[8]-m[5]*m[6]) + + m[2]*(m[3]*m[7]-m[4]*m[6]); + if (std::abs(det) < 1e-18) + return false; + const double id = 1.0/det; + out[0] = (m[4]*m[8]-m[5]*m[7])*id; + out[1] = (m[2]*m[7]-m[1]*m[8])*id; + out[2] = (m[1]*m[5]-m[2]*m[4])*id; + out[3] = (m[5]*m[6]-m[3]*m[8])*id; + out[4] = (m[0]*m[8]-m[2]*m[6])*id; + out[5] = (m[2]*m[3]-m[0]*m[5])*id; + out[6] = (m[3]*m[7]-m[4]*m[6])*id; + out[7] = (m[1]*m[6]-m[0]*m[7])*id; + out[8] = (m[0]*m[4]-m[1]*m[3])*id; + return true; +} + +Mat3 matmul(const Mat3& a, const Mat3& b) +{ + Mat3 r{}; + for (int i = 0; i < 3; ++i) + for (int j = 0; j < 3; ++j) + for (int k = 0; k < 3; ++k) + r[size_t(i*3+j)] += a[size_t(i*3+k)] * b[size_t(k*3+j)]; + return r; +} + +} // namespace + +bool DeformationTransfer::init(const std::vector& tmplNeutral, + const std::vector& faces, + const std::vector& fitted) +{ + m_valid = false; + const int N = int(tmplNeutral.size() / 3); + const int F = int(faces.size() / 3); + if (N < 3 || F < 1 || int(fitted.size() / 3) != N) + return false; + + m_n = N; + m_tmplNeutral = tmplNeutral; + m_faces = faces; + m_fitted = fitted; + m_srcRestInv.assign(size_t(F), {}); + m_tgtRestInv.assign(size_t(F), {}); + m_tgtNormalV4.assign(size_t(F), {}); + + // Unknowns (per axis, solved independently): the N deformed FITTED vertices + // PLUS one free "4th vertex" per triangle (columns N..N+F-1). The 4th vertex + // is Sumner's normal-direction trick: it only exists to make each triangle + // frame V = [x1-x0, x2-x0, x4-x0] a full-rank 3x3, and in the DEFORMED solve + // it is left UNCONSTRAINED (no normal equation pins it) — it simply absorbs + // the out-of-plane component the two edges can't represent. + // + // Per triangle the deformation gradient w.r.t. the target REST frame is + // A_f = Vtgt_deformed · Vtgt_rest⁻¹, and we require A_f == S_f (source). + // Each of the 3 gradient columns g gives one row (per axis): the deformed + // edge combination Σ_col Vinv[col][g]·(x_{col}-x0) must equal S column g. + // Anchor rows (small weight x_i == rest_i) fix the translation gauge. + + std::vector> trip; + trip.reserve(size_t(F) * 3 * 4 + size_t(N)); + + auto pushGrad = [&](int rowBase, int f) { + const int i0 = m_faces[size_t(f)*3]; + const int i1 = m_faces[size_t(f)*3+1]; + const int i2 = m_faces[size_t(f)*3+2]; + const int i3 = N + f; // the free 4th-vertex column + const Vec3 a = vert(m_fitted, i0), b = vert(m_fitted, i1), c = vert(m_fitted, i2); + const Vec3 d = normalV4(a, b, c); + m_tgtNormalV4[size_t(f)] = d; + Mat3 Vt = frame(a, b, c, d), Vinv; + if (!invert3(Vt, Vinv)) { + // degenerate target triangle — skip its gradient rows (anchors keep it) + m_tgtRestInv[size_t(f)] = {}; + return; + } + m_tgtRestInv[size_t(f)] = Vinv; + // edges columns are [x1-x0, x2-x0, x4-x0]; gradient column g combines + // them by Vinv[col][g]. Distribute onto the four real unknowns: + // x1 → Vinv[0][g], x2 → Vinv[1][g], x4 → Vinv[2][g], + // x0 → -(sum of the three) (from every -x0 term) + for (int g = 0; g < 3; ++g) { + const double w1 = Vinv[size_t(0*3+g)]; + const double w2 = Vinv[size_t(1*3+g)]; + const double w4 = Vinv[size_t(2*3+g)]; + const double w0 = -(w1 + w2 + w4); + const int row = rowBase + g; + trip.push_back({double(row), double(i0), w0}); + trip.push_back({double(row), double(i1), w1}); + trip.push_back({double(row), double(i2), w2}); + trip.push_back({double(row), double(i3), w4}); + } + }; + + // rows: 3 per triangle (gradient columns) + ONE anchor row. + // columns: N real verts + F free 4th-vertices. + // + // Deformation gradients are translation-invariant, so the system is only + // determined up to a global translation. We pin exactly ONE real vertex + // (index 0) to fix that gauge — anchoring EVERY vertex would fight the + // shape (the gradients want expr, the anchor wants neutral) and pull the + // solution back toward the neutral, corrupting the transfer. + for (int f = 0; f < F; ++f) + pushGrad(3 * f, f); + const double anchorW = 1.0; + trip.push_back({double(3 * F), 0.0, anchorW}); + + m_A.fromTriplets(3 * F + 1, N + F, trip); + + // source (template) rest inverse frames — for building S_f per shape + for (int f = 0; f < F; ++f) { + const int i0 = m_faces[size_t(f)*3]; + const int i1 = m_faces[size_t(f)*3+1]; + const int i2 = m_faces[size_t(f)*3+2]; + const Vec3 a = vert(m_tmplNeutral, i0), b = vert(m_tmplNeutral, i1), + c = vert(m_tmplNeutral, i2); + const Vec3 d = normalV4(a, b, c); + Mat3 Vs = frame(a, b, c, d), Vinv; + m_srcRestInv[size_t(f)] = invert3(Vs, Vinv) ? Vinv : Mat3{}; + } + + m_valid = true; + return true; +} + +std::vector DeformationTransfer::transfer( + const std::vector& tmplExprDelta) const +{ + if (!m_valid || int(tmplExprDelta.size() / 3) != m_n) + return {}; + const int F = int(m_faces.size() / 3); + + // template EXPRESSION verts = neutral + delta + std::vector expr(m_tmplNeutral.size()); + for (size_t i = 0; i < expr.size(); ++i) + expr[i] = m_tmplNeutral[i] + tmplExprDelta[i]; + + // Build rhs for each of the 3 output axes. For triangle f: + // S_f = Vsrc_expr · Vsrc_rest⁻¹ (3x3 source deformation gradient). + // The gradient constraint rows require the target gradient column g == S + // column g; the free 4th vertex is an unknown (its normal equation exists + // in the matrix), so the rhs is just S — no rest-approximation term. + const int nCols = m_n + F; + std::vector> rhs(3, std::vector(size_t(3 * F + 1), 0.0)); + + for (int f = 0; f < F; ++f) { + const int i0 = m_faces[size_t(f)*3]; + const int i1 = m_faces[size_t(f)*3+1]; + const int i2 = m_faces[size_t(f)*3+2]; + // source expression frame + Vec3 a{expr[size_t(i0)*3], expr[size_t(i0)*3+1], expr[size_t(i0)*3+2]}; + Vec3 b{expr[size_t(i1)*3], expr[size_t(i1)*3+1], expr[size_t(i1)*3+2]}; + Vec3 c{expr[size_t(i2)*3], expr[size_t(i2)*3+1], expr[size_t(i2)*3+2]}; + Vec3 d = normalV4(a, b, c); + Mat3 Vexpr = frame(a, b, c, d); + const Mat3& VsInv = m_srcRestInv[size_t(f)]; + const Mat3 S = matmul(Vexpr, VsInv); // 3x3 source gradient + for (int axis = 0; axis < 3; ++axis) + for (int g = 0; g < 3; ++g) + rhs[size_t(axis)][size_t(3*f+g)] = S[size_t(axis*3+g)]; + } + // anchor rhs: pin vertex 0 to (fitted rest + the template's vertex-0 + // displacement) — fixes the translation gauge while letting the shape + // move vertex 0 the same way the source moved its vertex 0. For the + // identity fit this is exactly expr[0]. + const double anchorW = 1.0; + for (int axis = 0; axis < 3; ++axis) + rhs[size_t(axis)][size_t(3*F)] = + anchorW * (m_fitted[axis] + tmplExprDelta[size_t(axis)]); + + // solve per axis (warm-start real verts at the fitted rest, 4th verts at + // their rest normal offset). Read back only the N real vertices as a delta. + std::vector out(size_t(m_n) * 3, 0.0f); + for (int axis = 0; axis < 3; ++axis) { + std::vector x(size_t(nCols), 0.0); + for (int i = 0; i < m_n; ++i) + x[size_t(i)] = m_fitted[size_t(i)*3+axis]; + for (int f = 0; f < F; ++f) + x[size_t(m_n + f)] = m_tgtNormalV4[size_t(f)][size_t(axis)]; + solveLeastSquaresCG(m_A, rhs[size_t(axis)], x, 800, 1e-8); + for (int i = 0; i < m_n; ++i) + out[size_t(i)*3+axis] = + float(x[size_t(i)] - m_fitted[size_t(i)*3+axis]); // delta + } + return out; +} + +} // namespace FaceRig diff --git a/src/FaceRig/DeformationTransfer.h b/src/FaceRig/DeformationTransfer.h new file mode 100644 index 000000000..52aa37584 --- /dev/null +++ b/src/FaceRig/DeformationTransfer.h @@ -0,0 +1,69 @@ +#ifndef DEFORMATIONTRANSFER_H +#define DEFORMATIONTRANSFER_H + +// Deformation transfer (Sumner & Popović 2004) for face auto-rig (#889, +// Slice D #892). Pure data — no Ogre, reuses FaceRig::SparseMatrix — and +// headless-tested. +// +// Given the NRICP correspondence (template verts fitted onto the USER +// identity, in TEMPLATE topology — from NonRigidICP #891) and the template's +// per-shape neutral→expression delta, transfer each expression's per-triangle +// deformation onto the fitted (user-identity) mesh. The output is the +// expression realized on the user's identity, still in template topology, as +// a per-template-vertex delta from the fitted neutral. FaceRigger (#893) +// resamples that to the real user vertices via the correspondence. +// +// Method: per source triangle build the deformation gradient +// S = [e1' e2' n'] · [e1 e2 n]⁻¹ +// (Sumner's 4th "normal" vertex trick: n = (e1×e2)/√|e1×e2|), where the +// unprimed frame is the template NEUTRAL triangle and the primed is the +// template EXPRESSION triangle. Then solve, over the FITTED mesh, for vertex +// positions whose per-triangle deformation gradient matches S — one sparse +// least-squares (the Sumner-Popović matrix), with the fitted neutral as the +// rest state, plus a small anchor term to pin the solution (deformation +// gradients are translation-free). + +#include "SparseSolve.h" + +#include + +namespace FaceRig { + +// Precomputes the per-triangle rest frames of the FITTED mesh once, so all 52 +// shapes transfer without rebuilding the (topology-fixed) system. +class DeformationTransfer { +public: + // tmplNeutral / faces: the TEMPLATE neutral verts (N*3) + tris (F*3). + // fitted: template verts fitted onto the user identity (N*3) — the NRICP + // correspondence; SAME topology as the template. + bool init(const std::vector& tmplNeutral, + const std::vector& faces, + const std::vector& fitted); + + bool valid() const { return m_valid; } + int vertexCount() const { return m_n; } + + // Transfer one template shape (tmplExprDelta = expr - tmplNeutral, N*3) → + // a per-vertex delta on the FITTED mesh (N*3, added to `fitted` gives the + // expression on the user identity). Returns empty on failure. + std::vector transfer(const std::vector& tmplExprDelta) const; + +private: + bool m_valid = false; + int m_n = 0; + std::vector m_tmplNeutral; // template rest (for source gradients) + std::vector m_faces; + std::vector m_fitted; // target rest (user identity) + // The transfer system A x = c is fixed by topology + fitted rest; only c + // (built from each shape's source deformation gradient) changes per shape. + SparseMatrix m_A; + // per-triangle inverse rest frames of the TEMPLATE neutral (source) and of + // the FITTED mesh (target), cached for building c. + std::vector> m_srcRestInv; // 3x3 per tri + std::vector> m_tgtRestInv; + std::vector> m_tgtNormalV4; // fitted 4th vertex/tri +}; + +} // namespace FaceRig + +#endif // DEFORMATIONTRANSFER_H diff --git a/src/FaceRig/DeformationTransfer_test.cpp b/src/FaceRig/DeformationTransfer_test.cpp new file mode 100644 index 000000000..6a0155112 --- /dev/null +++ b/src/FaceRig/DeformationTransfer_test.cpp @@ -0,0 +1,149 @@ +#include + +#include "FaceRig/DeformationTransfer.h" + +#include +#include + +namespace { + +struct Grid { + std::vector V; + std::vector F; +}; + +// A bumpy plane grid — real triangles for the per-face deformation gradients. +Grid makeGrid(int n, float extent, float bump = 0.0f) +{ + Grid g; + for (int y = 0; y < n; ++y) + for (int x = 0; x < n; ++x) { + const float fx = (float(x)/(n-1) - 0.5f) * extent; + const float fy = (float(y)/(n-1) - 0.5f) * extent; + const float fz = bump * std::sin(float(x)) * std::cos(float(y)); + g.V.insert(g.V.end(), {fx, fy, fz}); + } + for (int y = 0; y < n-1; ++y) + for (int x = 0; x < n-1; ++x) { + const int a = y*n+x, b = y*n+x+1, c = (y+1)*n+x, d = (y+1)*n+x+1; + g.F.insert(g.F.end(), {a, b, c}); + g.F.insert(g.F.end(), {b, d, c}); + } + return g; +} + +double diag(const std::vector& v) +{ + float lo[3] = {1e30f,1e30f,1e30f}, hi[3] = {-1e30f,-1e30f,-1e30f}; + for (size_t i = 0; i < v.size(); i += 3) + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], v[i+a]); + hi[a] = std::max(hi[a], v[i+a]); + } + double s = 0; + for (int a = 0; a < 3; ++a) s += double(hi[a]-lo[a])*double(hi[a]-lo[a]); + return std::sqrt(s); +} + +double maxAbs(const std::vector& v) +{ + double m = 0; + for (float x : v) m = std::max(m, double(std::abs(x))); + return m; +} + +} // namespace + +TEST(DeformationTransfer, RejectsBadInput) +{ + FaceRig::DeformationTransfer dt; + const Grid g = makeGrid(6, 2.0f); + EXPECT_FALSE(dt.init({}, {}, {})); // empty + EXPECT_FALSE(dt.init(g.V, g.F, {})); // fitted count mismatch + EXPECT_FALSE(dt.valid()); + // valid init + EXPECT_TRUE(dt.init(g.V, g.F, g.V)); + EXPECT_TRUE(dt.valid()); + EXPECT_EQ(dt.vertexCount(), int(g.V.size()/3)); +} + +// Identity: fitted == template neutral. Transferring a shape's delta should +// reproduce (approximately) that same delta — the source and target rest +// frames are identical, so the deformation gradients map back to the input. +TEST(DeformationTransfer, IdentityFitReproducesShapeDelta) +{ + const Grid g = makeGrid(10, 2.0f, 0.1f); + FaceRig::DeformationTransfer dt; + ASSERT_TRUE(dt.init(g.V, g.F, g.V)); + + // a shape: push the centre region up in +Z (like a smile bump) + const int n = int(g.V.size()/3); + std::vector delta(g.V.size(), 0.0f); + for (int i = 0; i < n; ++i) { + const float x = g.V[size_t(i)*3], y = g.V[size_t(i)*3+1]; + const float bump = 0.15f * std::exp(-(x*x + y*y) * 4.0f); + delta[size_t(i)*3+2] = bump; + } + const auto out = dt.transfer(delta); + ASSERT_EQ(out.size(), delta.size()); + for (float v : out) ASSERT_TRUE(std::isfinite(v)); + + // out should track delta closely (identity transfer). Compare RMS error. + double num = 0, den = 0; + for (size_t i = 0; i < delta.size(); ++i) { + const double d = double(out[i]) - double(delta[i]); + num += d*d; + den += double(delta[i])*double(delta[i]); + } + const double relRms = std::sqrt(num / std::max(den, 1e-12)); + EXPECT_LT(relRms, 0.15); // within 15% of the original shape +} + +// Winding / orientation preserved: transferring onto a UNIFORMLY SCALED user +// identity should scale the shape delta by the same factor (deformation +// gradients are scale-covariant), and keep signs (no inside-out flip). +TEST(DeformationTransfer, ScaledIdentityScalesTheDelta) +{ + const Grid tmpl = makeGrid(10, 2.0f, 0.1f); + // fitted = template scaled 2x (a "bigger head") + std::vector fitted(tmpl.V.size()); + for (size_t i = 0; i < fitted.size(); ++i) fitted[i] = tmpl.V[i] * 2.0f; + + FaceRig::DeformationTransfer dt; + ASSERT_TRUE(dt.init(tmpl.V, tmpl.F, fitted)); + + const int n = int(tmpl.V.size()/3); + std::vector delta(tmpl.V.size(), 0.0f); + for (int i = 0; i < n; ++i) { + const float x = tmpl.V[size_t(i)*3], y = tmpl.V[size_t(i)*3+1]; + delta[size_t(i)*3+2] = 0.12f * std::exp(-(x*x + y*y) * 4.0f); + } + const auto out = dt.transfer(delta); + ASSERT_EQ(out.size(), delta.size()); + for (float v : out) ASSERT_TRUE(std::isfinite(v)); + + // on a 2x mesh the same relative deformation should be ~2x the amplitude + const double dIn = maxAbs(delta); + const double dOut = maxAbs(out); + EXPECT_GT(dOut, dIn * 1.3); // clearly scaled up + EXPECT_LT(dOut, dIn * 3.0); // but bounded + + // sign preserved: peak stays +Z (no flip) + double peak = 0; + for (int i = 0; i < n; ++i) + if (std::abs(out[size_t(i)*3+2]) > std::abs(peak)) + peak = out[size_t(i)*3+2]; + EXPECT_GT(peak, 0.0); +} + +// Zero shape → zero delta (the neutral maps to the neutral). +TEST(DeformationTransfer, ZeroDeltaProducesZero) +{ + const Grid g = makeGrid(8, 2.0f, 0.1f); + FaceRig::DeformationTransfer dt; + ASSERT_TRUE(dt.init(g.V, g.F, g.V)); + std::vector zero(g.V.size(), 0.0f); + const auto out = dt.transfer(zero); + ASSERT_EQ(out.size(), zero.size()); + EXPECT_LT(maxAbs(out) / diag(g.V), 1e-3); +} diff --git a/src/FaceRig/SparseSolve.cpp b/src/FaceRig/SparseSolve.cpp new file mode 100644 index 000000000..aba1df4af --- /dev/null +++ b/src/FaceRig/SparseSolve.cpp @@ -0,0 +1,87 @@ +#include "SparseSolve.h" + +namespace FaceRig { + +void SparseMatrix::fromTriplets(int r, int c, + const std::vector>& trip) +{ + rows = r; + cols = c; + std::vector cnt(size_t(r) + 1, 0); + for (const auto& t : trip) + cnt[size_t(t[0]) + 1]++; + for (int i = 0; i < r; ++i) + cnt[size_t(i) + 1] += cnt[size_t(i)]; + m_rowPtr = cnt; + m_col.resize(trip.size()); + m_val.resize(trip.size()); + std::vector cur = m_rowPtr; + for (const auto& t : trip) { + const int rr = int(t[0]); + const int dst = cur[size_t(rr)]++; + m_col[size_t(dst)] = int(t[1]); + m_val[size_t(dst)] = t[2]; + } +} + +void SparseMatrix::mul(const std::vector& x, std::vector& y) const +{ + y.assign(size_t(rows), 0.0); + for (int r = 0; r < rows; ++r) { + double s = 0; + for (int k = m_rowPtr[size_t(r)]; k < m_rowPtr[size_t(r) + 1]; ++k) + s += m_val[size_t(k)] * x[size_t(m_col[size_t(k)])]; + y[size_t(r)] = s; + } +} + +void SparseMatrix::mulT(const std::vector& x, std::vector& y) const +{ + y.assign(size_t(cols), 0.0); + for (int r = 0; r < rows; ++r) { + const double xr = x[size_t(r)]; + for (int k = m_rowPtr[size_t(r)]; k < m_rowPtr[size_t(r) + 1]; ++k) + y[size_t(m_col[size_t(k)])] += m_val[size_t(k)] * xr; + } +} + +void solveLeastSquaresCG(const SparseMatrix& A, const std::vector& b, + std::vector& x, int maxIters, double tol) +{ + std::vector Ax, r, p, Ap, AtAp; + A.mul(x, Ax); + std::vector resid(size_t(A.rows)); + for (int i = 0; i < A.rows; ++i) + resid[size_t(i)] = b[size_t(i)] - Ax[size_t(i)]; + A.mulT(resid, r); // r = Aᵀ(b - Ax) + p = r; + double rs = 0; + for (double v : r) + rs += v * v; + const double rs0 = rs; + if (rs0 <= 0.0) + return; + for (int it = 0; it < maxIters && rs > tol * tol * rs0; ++it) { + A.mul(p, Ap); + A.mulT(Ap, AtAp); // AtAp = AᵀA p + double pAp = 0; + for (int i = 0; i < A.cols; ++i) + pAp += p[size_t(i)] * AtAp[size_t(i)]; + if (pAp <= 1e-30) + break; + const double a = rs / pAp; + for (int i = 0; i < A.cols; ++i) { + x[size_t(i)] += a * p[size_t(i)]; + r[size_t(i)] -= a * AtAp[size_t(i)]; + } + double rsn = 0; + for (double v : r) + rsn += v * v; + const double beta = rsn / rs; + for (int i = 0; i < A.cols; ++i) + p[size_t(i)] = r[size_t(i)] + beta * p[size_t(i)]; + rs = rsn; + } +} + +} // namespace FaceRig diff --git a/src/FaceRig/SparseSolve.h b/src/FaceRig/SparseSolve.h new file mode 100644 index 000000000..ef8f6b1f7 --- /dev/null +++ b/src/FaceRig/SparseSolve.h @@ -0,0 +1,39 @@ +#ifndef FACERIG_SPARSESOLVE_H +#define FACERIG_SPARSESOLVE_H + +// Minimal dependency-free sparse linear algebra for the face-rig native +// solvers. A CSR matrix built from triplets + conjugate-gradient on the normal +// equations (AᵀA x = Aᵀb) — no Eigen, no external solver. Pure data, +// headless-tested. Used by DeformationTransfer (#892); NonRigidICP (#891) still +// carries its own equivalent copy and can be migrated onto this in a follow-up. + +#include +#include + +namespace FaceRig { + +class SparseMatrix { +public: + int rows = 0, cols = 0; + + // Build from triplets {row, col, value}; duplicates are summed on multiply + // (CG only needs the products, so we keep them un-coalesced). + void fromTriplets(int r, int c, const std::vector>& trip); + + void mul(const std::vector& x, std::vector& y) const; // y = A x + void mulT(const std::vector& x, std::vector& y) const; // y = Aᵀ x + +private: + std::vector m_rowPtr, m_col; + std::vector m_val; +}; + +// Solve min ‖A x - b‖² by CG on the normal equations, warm-started at x +// (in/out). b has A.rows entries, x has A.cols. +void solveLeastSquaresCG(const SparseMatrix& A, const std::vector& b, + std::vector& x, int maxIters = 400, + double tol = 1e-6); + +} // namespace FaceRig + +#endif // FACERIG_SPARSESOLVE_H From d34036e838a5aa5ec77ba8c399aa135d24a09185 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 09:58:47 -0300 Subject: [PATCH 05/38] feat(#894): FaceRigger + attach + CLI facerig + MCP (Face-Rig Slice E) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chain the pure-data stages into an end-to-end face auto-rig: fit the ARKit template onto a user face (NRICP #892), transfer each of the 52 expressions onto the user identity (DeformationTransfer #893), resample to the real user vertices, and attach them as Ogre::Pose + VAT_POSE morph targets — named per FaceCap::kBlendshapeNames so face capture (#869) drives them with no new playback code. - FaceRigger.{h,cpp} (Ogre-free, headless-tested): buildFaceRig() runs the whole pipeline → per-user-vertex delta per shape. Humanoid-only gate on the NRICP fit residual (a non-face mesh fits poorly and is refused). Resample via a dependency-free spatial-hash grid (nearest correspondence vertex, built once for all 52 shapes). - FaceRigAttach.{h,cpp} (Ogre bridge): extract the entity's combined geometry, run buildFaceRig, split deltas back per submesh handle, and attach via AddMorphTargetCommand::redo() (the exact MorphCommands pose-build). attachFaceRigWithBundledTemplate() downloads/loads the template first. - CLI: qtmesh facerig [-o out] [--max-shapes N] [--max-residual PCT] [--json] (CLIPipeline::cmdFaceRig); recognized subcommand + gamification map. - MCP: add_arkit_blendshapes tool (selected face, optional re-export), heavy. - NonRigidICP: harden the residual against diverged (NaN/inf) vertices — skip them from the mean, count them, and flag a >5% diverged fit as failed so a mostly-NaN fit can't read as a great one (surfaced by UV-sphere pole fans). Verified end-to-end: the real ICT template (26719v, 51 shapes) fits a decimated different-topology face (15755v) at mean 0.008% / max 0.61% residual and attaches 51 blendshapes; the exported glb carries all 51 morph targets on the primitive. Tests: FaceRigger_test.cpp — per-user-vertex shapes with correct semantics (jaw vs smile moved-mass regions), names/order preserved, non-face rejection, bad-input rejection. Full FaceRig suite green (18 tests). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/AppLaunchHandler.cpp | 3 +- src/CLIPipeline.cpp | 112 +++++++++++++++ src/CLIPipeline.h | 6 + src/CMakeLists.txt | 2 + src/FaceRig/FaceRigAttach.cpp | 185 +++++++++++++++++++++++++ src/FaceRig/FaceRigAttach.h | 48 +++++++ src/FaceRig/FaceRigger.cpp | 232 ++++++++++++++++++++++++++++++++ src/FaceRig/FaceRigger.h | 74 ++++++++++ src/FaceRig/FaceRigger_test.cpp | 200 +++++++++++++++++++++++++++ src/FaceRig/NonRigidICP.cpp | 38 ++++-- src/MCPServer.cpp | 110 +++++++++++++++ src/MCPServer.h | 4 + 12 files changed, 1004 insertions(+), 10 deletions(-) create mode 100644 src/FaceRig/FaceRigAttach.cpp create mode 100644 src/FaceRig/FaceRigAttach.h create mode 100644 src/FaceRig/FaceRigger.cpp create mode 100644 src/FaceRig/FaceRigger.h create mode 100644 src/FaceRig/FaceRigger_test.cpp diff --git a/src/AppLaunchHandler.cpp b/src/AppLaunchHandler.cpp index 670deeef0..3293263ba 100644 --- a/src/AppLaunchHandler.cpp +++ b/src/AppLaunchHandler.cpp @@ -27,7 +27,8 @@ bool isCliSubcommand(const QString& arg) QStringLiteral("optimize"), QStringLiteral("bake-vertex-colors"), QStringLiteral("vat"), QStringLiteral("uv"), QStringLiteral("hdri"), QStringLiteral("light"), QStringLiteral("retopo"), - QStringLiteral("skin"), QStringLiteral("rig"), QStringLiteral("segment"), + QStringLiteral("skin"), QStringLiteral("rig"), QStringLiteral("facerig"), + QStringLiteral("segment"), QStringLiteral("generate3d"), QStringLiteral("morph"), QStringLiteral("nodeanim"), QStringLiteral("ps1"), QStringLiteral("cloud"), diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 550866f06..18707bfcb 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -42,6 +42,7 @@ #include "SkinWeights.h" #include "SkinEvaluate.h" #include "AutoRig.h" +#include "FaceRig/FaceRigAttach.h" #include "ImageTo3D/MeshGenPredictor.h" #include "ImageTo3D/TripoSGPredictor.h" #include "ImageTo3D/MeshGenBuilder.h" @@ -1599,6 +1600,7 @@ int CLIPipeline::run(int argc, char* argv[]) else if (cmd == "retopo") rc = cmdRetopo(argc, argv); else if (cmd == "skin") rc = cmdSkin(argc, argv); else if (cmd == "rig") rc = cmdRig(argc, argv); + else if (cmd == "facerig") rc = cmdFaceRig(argc, argv); else if (cmd == "segment") rc = cmdSegment(argc, argv); else if (cmd == "generate3d") rc = cmdGenerate3d(argc, argv); else if (cmd == "morph") rc = cmdMorph(argc, argv); @@ -1625,6 +1627,7 @@ int CLIPipeline::run(int argc, char* argv[]) {QStringLiteral("uv"), QStringLiteral("uv_unwrap")}, {QStringLiteral("skin"), QStringLiteral("skin_weights")}, {QStringLiteral("rig"), QStringLiteral("auto_rig")}, + {QStringLiteral("facerig"), QStringLiteral("auto_rig")}, {QStringLiteral("anim"), QStringLiteral("animation_blend")}, {QStringLiteral("morph"), QStringLiteral("morph")}, {QStringLiteral("vat"), QStringLiteral("vat_bake")}, @@ -9464,6 +9467,115 @@ int CLIPipeline::cmdRig(int argc, char* argv[]) return 0; } +int CLIPipeline::cmdFaceRig(int argc, char* argv[]) +{ + // Parse: facerig [-o out] [--max-shapes N] [--max-residual PCT] + // [--json] + QString inputPath, outputPath; + bool jsonOutput = false; + FaceRig::FaceRigOptions opts; + + for (int i = 1; i < argc; ++i) { + const QString arg = QString::fromLocal8Bit(argv[i]); + if (arg == "facerig" || arg == "--cli") continue; + if (arg == "--json") { jsonOutput = true; continue; } + if ((arg == "-o" || arg == "--output") && i + 1 < argc) { + outputPath = QString::fromLocal8Bit(argv[++i]); continue; + } + if (arg == "--max-shapes" && i + 1 < argc) { + opts.maxShapes = QString::fromLocal8Bit(argv[++i]).toInt(); continue; + } + if (arg == "--max-residual" && i + 1 < argc) { + opts.maxFitResidualPct = QString::fromLocal8Bit(argv[++i]).toDouble(); + continue; + } + if (!arg.startsWith("-") && inputPath.isEmpty()) { + inputPath = arg; continue; + } + } + + if (inputPath.isEmpty()) { + err() << "Error: No input file specified." << Qt::endl; + err() << "Usage: qtmesh facerig [-o out] [--max-shapes N] " + "[--max-residual PCT] [--json]" << Qt::endl; + return 2; + } + if (outputPath.isEmpty()) { + err() << "Error: -o required." << Qt::endl; + return 2; + } + + QFileInfo fi(inputPath); + if (!fi.exists()) { + err() << "Error: file not found: " << inputPath << Qt::endl; return 1; + } + if (!initOgreHeadless()) return 1; + + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QString("facerig .%1 max_shapes=%2").arg(fi.suffix()).arg(opts.maxShapes)); + SentryReporter::addBreadcrumb(QStringLiteral("file.import"), + QString("Importing %1").arg(fi.absoluteFilePath())); + + MeshImporterExporter::importer({fi.absoluteFilePath()}); + QList meshEntities; + for (Ogre::Entity* e : Manager::getSingleton()->getEntities()) { + if (e && e->getMovableType() == "Entity") + meshEntities.push_back(e); + } + if (meshEntities.isEmpty()) { + err() << "Error: failed to load " << inputPath << Qt::endl; return 1; + } + if (meshEntities.size() > 1) { + err() << "Error: " << inputPath + << " contains multiple mesh entities. `qtmesh facerig` supports " + "one entity per file." << Qt::endl; + return 1; + } + Ogre::Entity* entity = meshEntities.first(); + + const FaceRig::AttachReport rep = + FaceRig::attachFaceRigWithBundledTemplate(entity, opts); + if (!rep.ok) { + err() << "Error: face-rig failed — " << rep.error << Qt::endl; + return 1; + } + + auto* node = entity->getParentSceneNode(); + const QString fmt = formatForExtension(outputPath); + SentryReporter::addBreadcrumb(QStringLiteral("file.export"), + QString("Exporting %1").arg(QFileInfo(outputPath).absoluteFilePath())); + if (MeshImporterExporter::exporter(node, QFileInfo(outputPath).absoluteFilePath(), fmt) != 0) { + err() << "Error: export failed." << Qt::endl; + return 1; + } + + if (jsonOutput) { + QJsonObject j; + j["shapes_attached"] = rep.shapesAttached; + j["user_vertex_count"] = rep.userVertexCount; + j["fit_mean_residual_pct"] = rep.fitMeanResidualPct; + j["fit_max_residual_pct"] = rep.fitMaxResidualPct; + j["output"] = QFileInfo(outputPath).fileName(); + cliWrite(QString::fromUtf8( + QJsonDocument(j).toJson(QJsonDocument::Indented)) + "\n"); + } else { + cliWrite(QString("Face-rig: attached %1 ARKit blendshape(s)\n" + " user vertices: %2\n" + " fit residual: mean %3% max %4%\n" + "Wrote: %5\n") + .arg(rep.shapesAttached) + .arg(rep.userVertexCount) + .arg(rep.fitMeanResidualPct, 0, 'f', 3) + .arg(rep.fitMaxResidualPct, 0, 'f', 3) + .arg(QFileInfo(outputPath).fileName())); + } + GamificationManager::noteOperation( + QStringLiteral("auto_rig"), + {{QStringLiteral("blendshapes_attached"), rep.shapesAttached}}, + GamificationManager::Surface::Cli); + return 0; +} + int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) { // Parse: generate3d [-o out.glb] [--resolution N] [--no-color] diff --git a/src/CLIPipeline.h b/src/CLIPipeline.h index 775d501ff..969e4277d 100644 --- a/src/CLIPipeline.h +++ b/src/CLIPipeline.h @@ -225,6 +225,12 @@ class CLIPipeline { /// skin weights (--skin), and export. Issue #407. static int cmdRig(int argc, char* argv[]); + /// Face auto-rig (#889): fit the ARKit blendshape template onto a user + /// face mesh (NRICP + deformation transfer) and attach the 52 ARKit + /// morph targets, then export. `facerig [-o out] [--max-shapes N] + /// [--max-residual PCT] [--json]`. + static int cmdFaceRig(int argc, char* argv[]); + /// AI mesh part segmentation (#410): predict per-vertex/face part labels /// (head/torso/arm/leg) and emit a label map. `segment [--json] /// [--no-model] [--up-axis x|y|z]`. Text lists per-part vertex/face counts; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bc0b4aacf..2da7626c6 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -130,6 +130,8 @@ FaceRig/ArkitTemplate.cpp FaceRig/NonRigidICP.cpp FaceRig/SparseSolve.cpp FaceRig/DeformationTransfer.cpp +FaceRig/FaceRigger.cpp +FaceRig/FaceRigAttach.cpp MotionInbetween.cpp MotionLibrary.cpp MotionGenerator.cpp diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp new file mode 100644 index 000000000..9324cc34f --- /dev/null +++ b/src/FaceRig/FaceRigAttach.cpp @@ -0,0 +1,185 @@ +#include "FaceRigAttach.h" + +#include "ArkitTemplate.h" + +#include "../commands/MorphCommands.h" + +#include +#include +#include +#include + +#include +#include + +namespace FaceRig { + +namespace { + +// Read tight xyz floats from a VertexData POSITION element (mirrors +// SkinWeights.cpp's extractor). Returns false if there's no position stream. +bool extractPositions(Ogre::VertexData* vd, std::vector& out) +{ + if (!vd) return false; + const auto* posElem = + vd->vertexDeclaration->findElementBySemantic(Ogre::VES_POSITION); + if (!posElem) return false; + auto vbuf = vd->vertexBufferBinding->getBuffer(posElem->getSource()); + if (!vbuf || vd->vertexCount == 0) return false; + out.resize(size_t(vd->vertexCount) * 3); + const size_t stride = vbuf->getVertexSize(); + auto* base = static_cast( + vbuf->lock(Ogre::HardwareBuffer::HBL_READ_ONLY)); + for (size_t i = 0; i < vd->vertexCount; ++i) { + float* fp = nullptr; + posElem->baseVertexPointerToElement(base + i * stride, &fp); + out[3*i+0] = fp[0]; out[3*i+1] = fp[1]; out[3*i+2] = fp[2]; + } + vbuf->unlock(); + return true; +} + +void appendIndices(Ogre::IndexData* id, std::uint32_t offset, + std::vector& out) +{ + if (!id || !id->indexBuffer || id->indexCount == 0) return; + auto ibuf = id->indexBuffer; + const auto* base = static_cast( + ibuf->lock(Ogre::HardwareBuffer::HBL_READ_ONLY)); + base += id->indexStart * ibuf->getIndexSize(); + out.reserve(out.size() + id->indexCount); + if (ibuf->getType() == Ogre::HardwareIndexBuffer::IT_32BIT) { + const auto* ip = reinterpret_cast(base); + for (size_t i = 0; i < id->indexCount; ++i) out.push_back(int(ip[i] + offset)); + } else { + const auto* ip = reinterpret_cast(base); + for (size_t i = 0; i < id->indexCount; ++i) out.push_back(int(ip[i] + offset)); + } + ibuf->unlock(); +} + +// One geometry owner (shared pool, or a per-submesh vertex data) with the +// pose target handle it maps to and its base offset into the combined set. +struct Owner { + unsigned short handle; // 0 = shared, 1..N = submesh index+1 + std::uint32_t base; // first combined-vertex index + int count; // vertices in this owner +}; + +} // namespace + +AttachReport attachFaceRig(Ogre::Entity* entity, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts) +{ + AttachReport rep; + if (!entity) { rep.error = QStringLiteral("no entity"); return rep; } + Ogre::MeshPtr mesh = entity->getMesh(); + if (!mesh) { rep.error = QStringLiteral("entity has no mesh"); return rep; } + if (!tmpl.valid()) { rep.error = QStringLiteral("ARKit template not loaded"); return rep; } + + // ── collect every geometry owner into ONE combined vertex/index set so the + // fit sees the whole face (a single-submesh head is the common case, but a + // face split across submeshes still fits as one surface). Track each owner + // so we can split the per-vertex deltas back onto the right pose handle. + std::vector userV; + std::vector userF; + std::vector owners; + + auto addOwner = [&](unsigned short handle, Ogre::VertexData* vd, + Ogre::IndexData* id) { + std::vector pos; + if (!extractPositions(vd, pos)) return; + const std::uint32_t base = std::uint32_t(userV.size() / 3); + owners.push_back({handle, base, int(pos.size() / 3)}); + userV.insert(userV.end(), pos.begin(), pos.end()); + appendIndices(id, base, userF); + }; + + if (mesh->sharedVertexData) { + // shared pool → handle 0; its indices live per-submesh. + std::vector pos; + if (extractPositions(mesh->sharedVertexData, pos)) { + owners.push_back({0, 0, int(pos.size() / 3)}); + userV.insert(userV.end(), pos.begin(), pos.end()); + for (unsigned short si = 0; si < mesh->getNumSubMeshes(); ++si) { + Ogre::SubMesh* sm = mesh->getSubMesh(si); + if (sm && sm->useSharedVertices) + appendIndices(sm->indexData, 0, userF); + } + } + } + for (unsigned short si = 0; si < mesh->getNumSubMeshes(); ++si) { + Ogre::SubMesh* sm = mesh->getSubMesh(si); + if (!sm || sm->useSharedVertices) continue; + addOwner(static_cast(si + 1), sm->vertexData, sm->indexData); + } + + if (userV.size() < 9 || userF.size() < 3) { + rep.error = QStringLiteral("could not read mesh geometry"); + return rep; + } + + // ── run the Ogre-free pipeline + const FaceRigResult res = buildFaceRig(userV, userF, tmpl, opts); + rep.userVertexCount = res.userVertexCount; + rep.fitMeanResidualPct = res.fitMeanResidualPct; + rep.fitMaxResidualPct = res.fitMaxResidualPct; + if (!res.ok) { + rep.error = QString::fromStdString(res.error); + return rep; + } + + // ── attach each shape as a pose + VAT_POSE clip, splitting the combined + // deltas back onto each owner's handle. Reuse AddMorphTargetCommand's + // redo() (the exact MorphCommands pose-build) so face capture drives these + // with no new playback code; call redo() directly (headless-safe — no undo + // stack required, GUI callers can wrap in UndoManager separately). + for (const FaceRigShape& shape : res.shapes) { + std::vector slices; + for (const Owner& o : owners) { + MorphPoseSlice slice; + slice.submeshHandle = o.handle; + for (int i = 0; i < o.count; ++i) { + const std::uint32_t gv = o.base + std::uint32_t(i); + const float* d = &shape.userDeltas[size_t(gv) * 3]; + if (d[0] == 0.0f && d[1] == 0.0f && d[2] == 0.0f) continue; + slice.offsets[static_cast(i)] = + Ogre::Vector3f(d[0], d[1], d[2]); + } + if (!slice.offsets.empty()) slices.push_back(std::move(slice)); + } + if (slices.empty()) continue; // shape moved nothing on this mesh + AddMorphTargetCommand cmd(entity, shape.name, slices); + cmd.redo(); + rep.shapesAttached++; + } + + rep.ok = rep.shapesAttached > 0; + if (!rep.ok) + rep.error = QStringLiteral("no blendshapes produced any vertex movement"); + return rep; +} + +AttachReport attachFaceRigWithBundledTemplate(Ogre::Entity* entity, + const FaceRigOptions& opts) +{ + AttachReport rep; + const QString path = ArkitTemplate::ensureModelBlocking(); + if (path.isEmpty()) { + rep.error = QStringLiteral( + "ARKit template unavailable (offline and not downloaded, or the " + "build has no face-rig model). Set QTMESH_FACERIG_MODEL_BASE_URL or " + "download arkit_template.bin to ai_models/facerig/."); + return rep; + } + ArkitTemplate tmpl; + QString err; + if (!tmpl.load(path, &err)) { + rep.error = QStringLiteral("failed to load ARKit template: %1").arg(err); + return rep; + } + return attachFaceRig(entity, tmpl, opts); +} + +} // namespace FaceRig diff --git a/src/FaceRig/FaceRigAttach.h b/src/FaceRig/FaceRigAttach.h new file mode 100644 index 000000000..8ab37aae4 --- /dev/null +++ b/src/FaceRig/FaceRigAttach.h @@ -0,0 +1,48 @@ +#ifndef FACERIGATTACH_H +#define FACERIGATTACH_H + +// FaceRigAttach — the Ogre-touching bridge for the face auto-rig (#889, +// Slice E #893). Extracts a user entity's mesh geometry, runs the Ogre-free +// FaceRigger (NRICP → DeformationTransfer → resample), and attaches the +// resulting ARKit blendshapes as Ogre::Pose + VAT_POSE morph targets on the +// entity — reusing the exact pose-build mechanic MorphCommands uses so face +// capture (#869) drives them with no new playback code. +// +// Shared by the CLI (`qtmesh facerig`), MCP (`add_arkit_blendshapes`), and the +// GUI "Add ARKit Blendshapes" button. + +#include "FaceRigger.h" + +#include + +namespace Ogre { class Entity; } + +namespace FaceRig { + +struct AttachReport { + bool ok = false; + QString error; + int shapesAttached = 0; + int userVertexCount = 0; + double fitMeanResidualPct = 0.0; + double fitMaxResidualPct = 0.0; + QString templateFallback; // set when the template had to be downloaded/etc. +}; + +// Runs the whole pipeline on `entity` using the given (already-loaded) template +// and attaches the shapes as poses + a per-target VAT_POSE clip. Re-initialises +// the live entity so the pose buffers exist (mirrors the morph/auto-rig path). +// On failure nothing is attached and report.error explains why. +AttachReport attachFaceRig(Ogre::Entity* entity, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts = {}); + +// Convenience: ensure the bundled template is available (download-on-first-use), +// load it, then attachFaceRig. Returns a clear error if the template can't be +// obtained (offline + not present, or the build lacks the model). +AttachReport attachFaceRigWithBundledTemplate(Ogre::Entity* entity, + const FaceRigOptions& opts = {}); + +} // namespace FaceRig + +#endif // FACERIGATTACH_H diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp new file mode 100644 index 000000000..c698c2870 --- /dev/null +++ b/src/FaceRig/FaceRigger.cpp @@ -0,0 +1,232 @@ +#include "FaceRigger.h" + +#include "ArkitTemplate.h" +#include "DeformationTransfer.h" +#include "NonRigidICP.h" + +#include +#include +#include +#include +#include + +namespace FaceRig { + +namespace { + +// A uniform spatial-hash grid over a point set for nearest-point queries. The +// resample maps every user vertex to its nearest CORRESPONDENCE point (the +// fitted template verts X, template topology). Same map for all 52 shapes, so +// we build it once. Brute force would be Nu*Nt (~350M on a 27k template); +// the grid keeps it near-linear. Dependency-free. +class PointGrid { +public: + void build(const std::vector& pts) + { + m_pts = &pts; + const int n = int(pts.size() / 3); + if (n == 0) return; + for (int a = 0; a < 3; ++a) { m_lo[a] = 1e30f; m_hi[a] = -1e30f; } + for (int i = 0; i < n; ++i) + for (int a = 0; a < 3; ++a) { + m_lo[a] = std::min(m_lo[a], pts[size_t(i)*3+a]); + m_hi[a] = std::max(m_hi[a], pts[size_t(i)*3+a]); + } + // aim ~1 point per cell on average + double vol = 1.0; + for (int a = 0; a < 3; ++a) vol *= std::max(1e-6, double(m_hi[a]-m_lo[a])); + m_cell = float(std::cbrt(vol / std::max(1, n))); + if (m_cell <= 1e-9f) m_cell = 1.0f; + for (int i = 0; i < n; ++i) + m_cells[key(cellOf(&pts[size_t(i)*3]))].push_back(i); + } + + // nearest point index to q (3 floats), searching an expanding shell of + // cells until the nearest is provably found. Returns -1 if empty. + int nearest(const float* q) const + { + if (!m_pts || m_cells.empty()) return -1; + const std::array c = cellOf(q); + // absolute cap on the shell radius = span of the grid in cells + 1, + // guarantees termination even if q is far outside the populated region. + int spanCells = 1; + for (int a = 0; a < 3; ++a) + spanCells = std::max(spanCells, + int(std::ceil((m_hi[a]-m_lo[a]) / m_cell)) + 1); + const int rMax = spanCells + 1; + + int best = -1; + double bestD = std::numeric_limits::max(); + for (int r = 0; r <= rMax; ++r) { + for (int dx = -r; dx <= r; ++dx) + for (int dy = -r; dy <= r; ++dy) + for (int dz = -r; dz <= r; ++dz) { + // only the shell at Chebyshev radius r (interior scanned) + if (std::max({std::abs(dx),std::abs(dy),std::abs(dz)}) != r) continue; + auto it = m_cells.find(key({c[0]+dx, c[1]+dy, c[2]+dz})); + if (it == m_cells.end()) continue; + for (int idx : it->second) { + const float* p = &(*m_pts)[size_t(idx)*3]; + const double d = (double(p[0]-q[0])*(p[0]-q[0]) + + double(p[1]-q[1])*(p[1]-q[1]) + + double(p[2]-q[2])*(p[2]-q[2])); + if (d < bestD) { bestD = d; best = idx; } + } + } + // A point in shell r is at least (r-1)*cell away from q; once the + // best found is closer than the guaranteed reach of the NEXT shell, + // no farther shell can beat it. Scan one extra shell to be safe. + if (best >= 0) { + const double guaranteed = double(r) * m_cell; // min dist of shell r+1 + if (guaranteed * guaranteed >= bestD) return best; + } + } + return best; + } + +private: + std::array cellOf(const float* p) const + { + return {int(std::floor((p[0]-m_lo[0]) / m_cell)), + int(std::floor((p[1]-m_lo[1]) / m_cell)), + int(std::floor((p[2]-m_lo[2]) / m_cell))}; + } + static long long key(const std::array& c) + { + // pack 3 ints into one 64-bit key (21 bits each, offset to positive) + const long long x = (c[0] + (1<<20)) & 0x1FFFFF; + const long long y = (c[1] + (1<<20)) & 0x1FFFFF; + const long long z = (c[2] + (1<<20)) & 0x1FFFFF; + return (x << 42) | (y << 21) | z; + } + + const std::vector* m_pts = nullptr; + float m_lo[3] = {0,0,0}, m_hi[3] = {0,0,0}; + float m_cell = 1.0f; + std::unordered_map> m_cells; +}; + +double bboxDiag(const std::vector& v) +{ + if (v.empty()) return 0.0; + float lo[3] = {1e30f,1e30f,1e30f}, hi[3] = {-1e30f,-1e30f,-1e30f}; + for (size_t i = 0; i + 2 < v.size(); i += 3) + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], v[i+a]); + hi[a] = std::max(hi[a], v[i+a]); + } + double s = 0; + for (int a = 0; a < 3; ++a) s += double(hi[a]-lo[a]) * double(hi[a]-lo[a]); + return std::sqrt(s); +} + +} // namespace + +FaceRigResult buildFaceRig(const std::vector& userV, + const std::vector& userF, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts) +{ + FaceRigResult r; + if (userV.size() < 9 || userF.size() < 3) { + r.error = "user mesh has no geometry"; + return r; + } + if (!tmpl.valid()) { + r.error = "ARKit template not loaded"; + return r; + } + r.userVertexCount = int(userV.size() / 3); + + // 1) NRICP: template neutral → user neutral = correspondence X. + const NricpResult fit = FaceRig::fit(tmpl.neutral(), tmpl.faces(), userV, userF); + if (!fit.ok || fit.diag <= 0.0) { + r.error = "non-rigid fit failed"; + return r; + } + r.fitMeanResidualPct = 100.0 * fit.meanResidual / fit.diag; + r.fitMaxResidualPct = 100.0 * fit.maxResidual / fit.diag; + + // humanoid-only gate: a bad fit means this isn't a face — refuse. A NRICP + // fit that couldn't converge onto the surface reports non-finite or huge + // residuals (e.g. a plane template forced onto a sphere), which we treat + // as a hard reject alongside the mean-residual threshold. The max residual + // catches shapes that fit on average but blow up in a region. + const bool nonFinite = !std::isfinite(r.fitMeanResidualPct) || + !std::isfinite(r.fitMaxResidualPct); + if (nonFinite || r.fitMeanResidualPct > opts.maxFitResidualPct || + r.fitMaxResidualPct > opts.maxFitResidualPct * 6.0) { + r.error = "mesh does not fit the human face template (mean residual " + + std::to_string(r.fitMeanResidualPct) + "%, max " + + std::to_string(r.fitMaxResidualPct) + + "%); this does not look like a human face mesh"; + return r; + } + + // sanitize the correspondence: a handful of template verts may have + // diverged (NaN/inf) even in an accepted fit (< 5% by the NRICP gate). + // Fall those back to the template neutral so they don't poison the + // deformation-transfer solve — they simply won't deform meaningfully. + std::vector fitted = fit.fitted; + const std::vector& tn = tmpl.neutral(); + for (size_t i = 0; i < fitted.size() && i < tn.size(); ++i) + if (!std::isfinite(fitted[i])) + fitted[i] = tn[i]; + + // 2) DeformationTransfer over the fixed (topology + fit) system. + DeformationTransfer dt; + if (!dt.init(tmpl.neutral(), tmpl.faces(), fitted)) { + r.error = "deformation-transfer setup failed"; + return r; + } + + // 3) resample map: user vertex → nearest correspondence vertex (built once). + PointGrid grid; + grid.build(fitted); + const int nu = r.userVertexCount; + std::vector userToTmpl(size_t(nu), -1); + for (int i = 0; i < nu; ++i) + userToTmpl[size_t(i)] = grid.nearest(&userV[size_t(i)*3]); + + const double diag = bboxDiag(userV); + const double eps = opts.deltaEpsPct / 100.0 * diag; + + const auto& shapes = tmpl.shapes(); + const int maxShapes = opts.maxShapes > 0 + ? std::min(opts.maxShapes, int(shapes.size())) + : int(shapes.size()); + + for (int s = 0; s < maxShapes; ++s) { + // per-TEMPLATE-vertex delta on the user identity + const std::vector tmplDelta = dt.transfer(shapes[size_t(s)].deltas); + if (int(tmplDelta.size() / 3) != tmpl.vertexCount()) { + r.error = "transfer produced an unexpected vertex count"; + return r; + } + + FaceRigShape out; + out.name = shapes[size_t(s)].name; + out.userDeltas.assign(size_t(nu) * 3, 0.0f); + for (int i = 0; i < nu; ++i) { + const int t = userToTmpl[size_t(i)]; + if (t < 0) continue; + float dvec[3] = {tmplDelta[size_t(t)*3], tmplDelta[size_t(t)*3+1], + tmplDelta[size_t(t)*3+2]}; + const double mag = std::sqrt(double(dvec[0])*dvec[0] + + double(dvec[1])*dvec[1] + + double(dvec[2])*dvec[2]); + if (mag < eps) continue; // noise floor → keep sparse + out.userDeltas[size_t(i)*3] = dvec[0]; + out.userDeltas[size_t(i)*3+1] = dvec[1]; + out.userDeltas[size_t(i)*3+2] = dvec[2]; + out.nonZeroVerts++; + out.maxDisp = std::max(out.maxDisp, float(mag)); + } + r.shapes.push_back(std::move(out)); + } + + r.ok = true; + return r; +} + +} // namespace FaceRig diff --git a/src/FaceRig/FaceRigger.h b/src/FaceRig/FaceRigger.h new file mode 100644 index 000000000..de24aa14e --- /dev/null +++ b/src/FaceRig/FaceRigger.h @@ -0,0 +1,74 @@ +#ifndef FACERIGGER_H +#define FACERIGGER_H + +// FaceRigger — the face auto-rig orchestrator (#889, Slice E #893). Chains the +// pure-data stages into per-USER-vertex ARKit blendshape deltas: +// +// ArkitTemplate (neutral + 52 expression deltas, template topology) +// │ +// ▼ NonRigidICP (#891): fit template neutral → user neutral +// correspondence X (template-topology verts lying on the user surface) +// │ +// ▼ DeformationTransfer (#892): per shape, transfer the template's +// │ per-triangle deformation onto X → per-TEMPLATE-vertex delta +// │ +// ▼ resample template-topology deltas → the real USER vertices +// 52 × per-user-vertex deltas, named per FaceCap::kBlendshapeNames +// +// Ogre-free + headless-tested; the Ogre attach (create Pose + VAT_POSE per +// shape on the user entity) + CLI/MCP surfaces live in the FaceRig CLI/MCP +// layer, reusing MorphCommands' buildPosesFromSlices pattern. +// +// Humanoid-only (risk #1 in docs/FACE_RIG_SPIKE.md): the fit residual is a +// quality gate — a non-face mesh fits poorly and is rejected with a clear +// reason, mirroring the AutoRig/Pinocchio precedent. + +#include + +#include +#include + +namespace FaceRig { + +class ArkitTemplate; + +struct FaceRigShape { + QString name; // a FaceCap::kBlendshapeNames entry + std::vector userDeltas; // userVertexCount*3, (expr - neutral) + int nonZeroVerts = 0; // verts this shape actually moves + float maxDisp = 0.0f; // max |delta| (mesh units) +}; + +struct FaceRigResult { + std::vector shapes; + int userVertexCount = 0; + double fitMeanResidualPct = 0.0; // NRICP mean residual / user diag (%) + double fitMaxResidualPct = 0.0; + bool ok = false; + std::string error; // set when !ok +}; + +struct FaceRigOptions { + // Reject the rig when the NRICP fit is worse than this (% of user diag). + // A human template only fits a roughly human face; a prop/creature blows + // past this and is refused rather than emitting garbage shapes. + double maxFitResidualPct = 8.0; + // Drop per-vertex deltas below this fraction of the user diagonal (noise + // floor from the transfer solve) so shapes stay sparse. + double deltaEpsPct = 0.02; + // Cap the shapes generated (0 = all in the template). Names are matched + // against the template's own shape order. + int maxShapes = 0; +}; + +// userV/userF: the user neutral mesh (Nu*3 verts + Fu*3 tris). tmpl: a loaded +// ArkitTemplate. Returns 52 per-user-vertex delta sets (or fewer if the +// template has fewer), or ok=false + error on failure / a poor fit. +FaceRigResult buildFaceRig(const std::vector& userV, + const std::vector& userF, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts = {}); + +} // namespace FaceRig + +#endif // FACERIGGER_H diff --git a/src/FaceRig/FaceRigger_test.cpp b/src/FaceRig/FaceRigger_test.cpp new file mode 100644 index 000000000..bc08fb94f --- /dev/null +++ b/src/FaceRig/FaceRigger_test.cpp @@ -0,0 +1,200 @@ +#include + +#include "FaceRig/ArkitTemplate.h" +#include "FaceRig/FaceRigger.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +struct Grid { + std::vector V; + std::vector F; +}; + +// A bumpy plane — a stand-in "face" surface with real triangles. +Grid makeGrid(int n, float extent, float bump) +{ + Grid g; + for (int y = 0; y < n; ++y) + for (int x = 0; x < n; ++x) { + const float fx = (float(x)/(n-1) - 0.5f) * extent; + const float fy = (float(y)/(n-1) - 0.5f) * extent; + const float fz = bump * std::sin(1.5f*float(x)) * std::cos(1.5f*float(y)); + g.V.insert(g.V.end(), {fx, fy, fz}); + } + for (int y = 0; y < n-1; ++y) + for (int x = 0; x < n-1; ++x) { + const int a = y*n+x, b = y*n+x+1, c = (y+1)*n+x, d = (y+1)*n+x+1; + g.F.insert(g.F.end(), {a, b, c}); + g.F.insert(g.F.end(), {b, d, c}); + } + return g; +} + +void putI32(QByteArray& b, int32_t v) { + v = qToLittleEndian(v); + b.append(reinterpret_cast(&v), 4); +} +void putF32(QByteArray& b, float f) { + quint32 raw; std::memcpy(&raw, &f, 4); raw = qToLittleEndian(raw); + b.append(reinterpret_cast(&raw), 4); +} + +// Pack a synthetic arkit_template.bin (magic + V/F/S + neutral + faces + +// S×(name[32] + delta[V*3])) and write it to `path`. +bool writeSyntheticTemplate(const QString& path, const Grid& g, + const std::vector>>& shapes) +{ + QByteArray b; + b.append("QMFRT1\0\0", 8); + putI32(b, int(g.V.size()/3)); + putI32(b, int(g.F.size()/3)); + putI32(b, int(shapes.size())); + for (float v : g.V) putF32(b, v); + for (int i : g.F) putI32(b, i); + for (const auto& [name, delta] : shapes) { + char nm[32] = {0}; + const QByteArray n = name.toLatin1(); + std::memcpy(nm, n.constData(), std::min(31, size_t(n.size()))); + b.append(nm, 32); + for (float d : delta) putF32(b, d); + } + QDir().mkpath(QFileInfo(path).absolutePath()); + QFile f(path); + if (!f.open(QIODevice::WriteOnly)) return false; + f.write(b); + f.close(); + return true; +} + +QString tempTemplatePath() +{ + const QString dir = QStandardPaths::writableLocation(QStandardPaths::TempLocation); + return QDir(dir).filePath("qtmesh_facerig_test_template.bin"); +} + +} // namespace + +TEST(FaceRigger, RejectsBadInput) +{ + FaceRig::ArkitTemplate empty; + const auto r = FaceRig::buildFaceRig({}, {}, empty); + EXPECT_FALSE(r.ok); + EXPECT_FALSE(r.error.empty()); +} + +TEST(FaceRigger, ProducesPerUserVertexShapes) +{ + const Grid tmpl = makeGrid(12, 2.0f, 0.12f); + const int nt = int(tmpl.V.size()/3); + + // shape A: push a centre bump in +Z (a "smile"-ish local deform) + std::vector smile(tmpl.V.size(), 0.0f); + for (int i = 0; i < nt; ++i) { + const float x = tmpl.V[size_t(i)*3], y = tmpl.V[size_t(i)*3+1]; + smile[size_t(i)*3+2] = 0.15f * std::exp(-(x*x + y*y) * 5.0f); + } + // shape B: drop the lower half in -Z (a "jawOpen"-ish region deform) + std::vector jaw(tmpl.V.size(), 0.0f); + for (int i = 0; i < nt; ++i) { + const float y = tmpl.V[size_t(i)*3+1]; + if (y < 0.0f) jaw[size_t(i)*3+2] = -0.2f * (-y); + } + + const QString path = tempTemplatePath(); + ASSERT_TRUE(writeSyntheticTemplate(path, tmpl, + {{"mouthSmileLeft", smile}, {"jawOpen", jaw}})); + + FaceRig::ArkitTemplate at; + QString err; + ASSERT_TRUE(at.load(path, &err)) << err.toStdString(); + ASSERT_EQ(at.shapeCount(), 2); + + // user = the SAME surface at a different tessellation (different topology) + const Grid user = makeGrid(16, 2.0f, 0.12f); + const int nu = int(user.V.size()/3); + + const auto r = FaceRig::buildFaceRig(user.V, user.F, at); + ASSERT_TRUE(r.ok) << r.error; + EXPECT_EQ(r.userVertexCount, nu); + EXPECT_EQ(int(r.shapes.size()), 2); + // same-surface fit should be tight + EXPECT_LT(r.fitMeanResidualPct, 5.0); + + for (const auto& sh : r.shapes) { + EXPECT_EQ(int(sh.userDeltas.size()), nu * 3); + for (float v : sh.userDeltas) EXPECT_TRUE(std::isfinite(v)); + EXPECT_GT(sh.nonZeroVerts, 0); // the shape actually moves verts + EXPECT_GT(sh.maxDisp, 0.0f); + } + + // names preserved, order preserved + EXPECT_EQ(r.shapes[0].name, QStringLiteral("mouthSmileLeft")); + EXPECT_EQ(r.shapes[1].name, QStringLiteral("jawOpen")); + + // semantics: the smile shape moves CENTRE verts most; the jaw shape moves + // LOWER-half verts most. Check the centroid of moved mass. + auto movedCentroidY = [&](const FaceRig::FaceRigShape& sh) { + double sy = 0, w = 0; + for (int i = 0; i < nu; ++i) { + const float* d = &sh.userDeltas[size_t(i)*3]; + const double m = std::sqrt(double(d[0])*d[0]+double(d[1])*d[1]+double(d[2])*d[2]); + sy += m * double(user.V[size_t(i)*3+1]); + w += m; + } + return w > 0 ? sy / w : 0.0; + }; + // jaw deformation lives in the lower half (y<0) → its moved-mass centroid Y + // is clearly below the smile's (which is centred at y≈0). + EXPECT_LT(movedCentroidY(r.shapes[1]), movedCentroidY(r.shapes[0])); +} + +TEST(FaceRigger, RejectsNonFaceMesh) +{ + // template = bumpy plane with a shape + const Grid tmpl = makeGrid(10, 2.0f, 0.12f); + std::vector shape(tmpl.V.size(), 0.0f); + for (size_t i = 2; i < shape.size(); i += 3) shape[i] = 0.1f; + const QString path = tempTemplatePath(); + ASSERT_TRUE(writeSyntheticTemplate(path, tmpl, {{"jawOpen", shape}})); + FaceRig::ArkitTemplate at; + ASSERT_TRUE(at.load(path)); + + // user = a closed sphere: a fundamentally different topology/shape than the + // open plane template. NRICP cannot wrap a plane around a sphere cleanly, so + // the fit residual stays high and the humanoid-only guard rejects it. + Grid user; + { + const int nlat = 12, nlon = 16; + for (int i = 0; i <= nlat; ++i) { + const float th = 3.14159265358979f * float(i) / nlat; + for (int j = 0; j < nlon; ++j) { + const float ph = 2.0f * 3.14159265358979f * float(j) / nlon; + user.V.insert(user.V.end(), + {std::sin(th)*std::cos(ph), std::cos(th), std::sin(th)*std::sin(ph)}); + } + } + for (int i = 0; i < nlat; ++i) + for (int j = 0; j < nlon; ++j) { + const int a = i*nlon + j, b = i*nlon + (j+1)%nlon; + const int c = (i+1)*nlon + j, d = (i+1)*nlon + (j+1)%nlon; + user.F.insert(user.F.end(), {a, b, c}); + user.F.insert(user.F.end(), {b, d, c}); + } + } + FaceRig::FaceRigOptions opts; + opts.maxFitResidualPct = 3.0; + const auto r = FaceRig::buildFaceRig(user.V, user.F, at, opts); + EXPECT_FALSE(r.ok); + EXPECT_NE(r.error.find("face"), std::string::npos); +} diff --git a/src/FaceRig/NonRigidICP.cpp b/src/FaceRig/NonRigidICP.cpp index 3fcdd5c98..2188b773c 100644 --- a/src/FaceRig/NonRigidICP.cpp +++ b/src/FaceRig/NonRigidICP.cpp @@ -311,23 +311,43 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, } } - // final residuals to the user surface + // final residuals to the user surface. A vertex whose affine solve + // diverged (NaN/inf — e.g. degenerate fan triangles at a UV-sphere pole) + // must not poison the mean: count it, skip it from the average, but still + // surface it so callers can gate (a well-behaved face mesh produces none). res.fitted.resize(size_t(Nt)*3); res.residual.resize(Nt); double sum = 0, mx2 = 0; + int finiteCount = 0, divergedCount = 0; for (int i = 0; i < Nt; ++i) { - const int cf = tree.nearest(X[i]); - const Vec3 cp = closestPointTriangle(X[i], utri[cf][0], utri[cf][1], utri[cf][2]); - const double d = std::sqrt(vdot(vsub(X[i],cp), vsub(X[i],cp))); - res.residual[i] = float(d); - sum += d; mx2 = std::max(mx2, d); + const bool finite = std::isfinite(X[i][0]) && std::isfinite(X[i][1]) && + std::isfinite(X[i][2]); + double d = 0.0; + if (finite) { + const int cf = tree.nearest(X[i]); + const Vec3 cp = closestPointTriangle(X[i], utri[cf][0], utri[cf][1], utri[cf][2]); + d = std::sqrt(vdot(vsub(X[i],cp), vsub(X[i],cp))); + } + if (finite && std::isfinite(d)) { + res.residual[i] = float(d); + sum += d; mx2 = std::max(mx2, d); + finiteCount++; + } else { + res.residual[i] = std::numeric_limits::infinity(); + divergedCount++; + } res.fitted[size_t(i)*3+0] = float(X[i][0]); res.fitted[size_t(i)*3+1] = float(X[i][1]); res.fitted[size_t(i)*3+2] = float(X[i][2]); } - res.meanResidual = sum/Nt; - res.maxResidual = mx2; - res.ok = true; + res.meanResidual = finiteCount > 0 ? sum/finiteCount + : std::numeric_limits::infinity(); + // a large diverged fraction means the fit failed — reflect it in maxResidual + // (which callers already gate on) so a mostly-NaN fit can't read as "great". + res.maxResidual = (divergedCount > Nt / 20) // > 5% diverged + ? std::numeric_limits::infinity() + : mx2; + res.ok = finiteCount > Nt / 2; // need at least half the verts to have fit return res; } diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 647764795..70ecefc7d 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -48,6 +48,7 @@ #include "SkinWeights.h" #include "SkinningDisplay.h" #include "AutoRig.h" +#include "FaceRig/FaceRigAttach.h" #include "MeshDepthRenderer.h" #include "ModelIsometricRenderer.h" #ifdef ENABLE_STABLE_DIFFUSION @@ -625,6 +626,7 @@ const QMap& MCPServer::toolHandlers() {QStringLiteral("compute_skin_weights"), &MCPServer::toolComputeSkinWeights}, {QStringLiteral("set_skinning_display"), &MCPServer::toolSetSkinningDisplay}, {QStringLiteral("auto_rig"), &MCPServer::toolAutoRig}, + {QStringLiteral("add_arkit_blendshapes"), &MCPServer::toolAddArkitBlendshapes}, {QStringLiteral("generate_mesh_texture"), &MCPServer::toolGenerateMeshTexture}, {QStringLiteral("generate_pbr_maps"), &MCPServer::toolGeneratePbrMaps}, {QStringLiteral("upscale_texture"), &MCPServer::toolUpscaleTexture}, @@ -740,6 +742,7 @@ bool MCPServer::isHeavyTool(const QString &name) QStringLiteral("motion_in_between"), QStringLiteral("generate_motion"), QStringLiteral("segment_mesh"), + QStringLiteral("add_arkit_blendshapes"), QStringLiteral("generate_mesh_from_image"), QStringLiteral("save_scene"), QStringLiteral("open_scene"), @@ -798,6 +801,7 @@ QJsonObject MCPServer::callTool(const QString &name, const QJsonObject &args) {QStringLiteral("uv_set_seams"), QStringLiteral("uv_unwrap")}, {QStringLiteral("compute_skin_weights"), QStringLiteral("skin_weights")}, {QStringLiteral("auto_rig"), QStringLiteral("auto_rig")}, + {QStringLiteral("add_arkit_blendshapes"), QStringLiteral("auto_rig")}, {QStringLiteral("motion_in_between"), QStringLiteral("motion_inbetween")}, {QStringLiteral("generate_motion"), QStringLiteral("animation_blend")}, {QStringLiteral("merge_animations"), QStringLiteral("animation_blend")}, @@ -2291,6 +2295,83 @@ QJsonObject MCPServer::toolAutoRig(const QJsonObject &args) return result; } +QJsonObject MCPServer::toolAddArkitBlendshapes(const QJsonObject &args) +{ + // #889: fit the ARKit blendshape template onto the selected face mesh and + // attach the 52 ARKit morph targets, optionally re-exporting. + if (!hasSelectedEntities()) + return makeErrorResult("No mesh selected. Load a mesh first with load_mesh."); + + FaceRig::FaceRigOptions opts; + if (args.contains("max_shapes")) { + if (!args["max_shapes"].isDouble()) + return makeErrorResult("Error: 'max_shapes' must be a number."); + opts.maxShapes = args["max_shapes"].toInt(); + } + if (args.contains("max_residual_pct")) { + if (!args["max_residual_pct"].isDouble()) + return makeErrorResult("Error: 'max_residual_pct' must be a number."); + opts.maxFitResidualPct = args["max_residual_pct"].toDouble(); + } + if (args.contains("output_path") && !args["output_path"].isString()) + return makeErrorResult("Error: 'output_path' must be a string."); + const QString outputPath = args.value("output_path").toString(); + + SelectionSet* sel = SelectionSet::getSingleton(); + const QList resolved = sel ? sel->getResolvedEntities() + : QList{}; + if (resolved.isEmpty() || !resolved.first()) + return makeErrorResult("No selected entity."); + Ogre::Entity* entity = resolved.first(); + + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QStringLiteral("add_arkit_blendshapes entity=%1 max_shapes=%2") + .arg(QString::fromStdString(entity->getName())).arg(opts.maxShapes)); + + FaceRig::AttachReport rep; + try { + rep = FaceRig::attachFaceRigWithBundledTemplate(entity, opts); + if (!rep.ok) + return makeErrorResult( + QStringLiteral("Face-rig failed: %1").arg(rep.error)); + + if (!outputPath.isEmpty()) { + Ogre::SceneNode* node = entity->getParentSceneNode(); + if (!node) + return makeErrorResult(QStringLiteral( + "Error: rigged, but the entity has no scene node to export from")); + SentryReporter::addBreadcrumb(QStringLiteral("file.export"), + QStringLiteral("add_arkit_blendshapes export requested")); + const int rc = MeshImporterExporter::exporter( + node, outputPath, CLIPipeline::formatForExtension(outputPath)); + if (rc != 0) + return makeErrorResult( + QStringLiteral("Error: blendshapes attached but export to '%1' " + "failed (code %2)").arg(outputPath).arg(rc)); + } + } catch (const Ogre::Exception& e) { + return makeErrorResult(QStringLiteral("Ogre error: %1") + .arg(QString::fromStdString(e.getFullDescription()))); + } catch (const std::exception& e) { + return makeErrorResult(QStringLiteral("Face-rig error: %1") + .arg(QString::fromUtf8(e.what()))); + } + + QJsonObject result = makeSuccessResult( + QStringLiteral("Attached %1 ARKit blendshape(s) (fit residual mean %2%, " + "max %3%).") + .arg(rep.shapesAttached) + .arg(rep.fitMeanResidualPct, 0, 'f', 3) + .arg(rep.fitMaxResidualPct, 0, 'f', 3)); + QJsonObject j; + j["shapes_attached"] = rep.shapesAttached; + j["user_vertex_count"] = rep.userVertexCount; + j["fit_mean_residual_pct"] = rep.fitMeanResidualPct; + j["fit_max_residual_pct"] = rep.fitMaxResidualPct; + result["facerig"] = j; + return result; +} + QJsonObject MCPServer::toolGenerateMeshTexture(const QJsonObject &args) { #ifndef ENABLE_STABLE_DIFFUSION @@ -8542,6 +8623,35 @@ QJsonArray MCPServer::buildToolsList() ); } + // add_arkit_blendshapes (#889) + { + QJsonObject props; + props["max_shapes"] = QJsonObject{{"type", "integer"}, + {"description", + "Cap the number of ARKit shapes generated (0 = all 52 in the " + "template, default)."}}; + props["max_residual_pct"] = QJsonObject{{"type", "number"}, + {"description", + "Reject the rig when the non-rigid fit is worse than this percent of " + "the mesh diagonal (default 8). A human template only fits a roughly " + "human face; a non-face mesh blows past this and is refused."}}; + props["output_path"] = QJsonObject{{"type", "string"}, + {"description", + "Optional path to re-export the mesh with the attached blendshapes. " + "When omitted, the shapes are added to the in-session scene only."}}; + appendTool( + "add_arkit_blendshapes", + "Fit the ARKit blendshape template onto the currently selected FACE " + "mesh and attach the 52 ARKit morph targets (#889). Native pipeline (no " + "external deps): non-rigid ICP fits the template to the user face, then " + "Sumner-Popovic deformation transfer realizes each expression on the " + "user's identity; the shapes are named per the mocap-52 vocabulary so " + "face capture drives them. Humanoid faces only — a poor fit is rejected. " + "The bundled template downloads on first use.", + props + ); + } + // generate_mesh_texture — only advertised when Stable Diffusion is // compiled in; the handler hard-fails otherwise, so publishing it on // a non-SD build would imply a capability the server can't satisfy. diff --git a/src/MCPServer.h b/src/MCPServer.h index 1555d16fa..82657ff31 100644 --- a/src/MCPServer.h +++ b/src/MCPServer.h @@ -177,6 +177,10 @@ private slots: /// #407: native auto-rig of the selected static mesh (template embedding), /// optional skin chain + re-export. QJsonObject toolAutoRig(const QJsonObject &args); + /// #889: fit the ARKit blendshape template onto the selected face mesh + /// (NRICP + deformation transfer), attach the 52 ARKit morph targets, and + /// optionally re-export. + QJsonObject toolAddArkitBlendshapes(const QJsonObject &args); /// Issue #403: mesh-aware (depth-conditioned) texture /// generation. Renders the selected entity's depth map and /// conditions sd.cpp on it via a ControlNet depth model, then From 11961f6942d1df230aaffa3f4f6d011be2af8d4c Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 11:17:30 -0300 Subject: [PATCH 06/38] feat(#895): GUI "Add ARKit Blendshapes" button (Face-Rig Slice F) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One-click face auto-rig from the Inspector's Vertex Morph Animation section: fit the ARKit template onto the selected face mesh and attach the 52 ARKit morph targets, so the #869 face-capture panel drives them (name-based hand-off — the shapes are named per FaceCap::kBlendshapeNames). - FaceRigController (QML_SINGLETON): hasMeshSelection / busy / downloading / status props; addArkitBlendshapesAsync() extracts geometry + loads the bundled template on the MAIN thread, runs the heavy Ogre-free buildFaceRig() on a WORKER (UI stays responsive; status = "Downloading…/Fitting…"), then commits the attach back on the main thread as ONE undo macro ("Add ARKit Blendshapes") so Ctrl+Z removes all shapes at once. Re-resolves the entity by name after the worker returns (selection may have changed). Gamification + Sentry breadcrumbs. - FaceRigAttach: split extractGeometry() (main-thread buffer read) and attachShapes() (main-thread pose attach) out of attachFaceRig so the GUI can run the fit off-thread; attachFaceRig/CLI/MCP now compose the same helpers (behavior-preserving — CLI end-to-end unchanged: 51 shapes, 0.008% mean). - QML: "✨ Add ARKit Blendshapes (AI)" button in the Shapes section, gated on a mesh selection, disabled + showing worker status while busy, with a result/ error line. Themed with the Inspector's highlight/border/text colors. - Registered the singleton + kill() in mainwindow.cpp. Note: the #869 Performance Capture panel lives on the mocap epic branches (not this stack); the hand-off is automatic once both reach master since the shape names already match the mocap-52 vocabulary. Co-Authored-By: Claude Opus 4.8 (1M context) --- qml/PropertiesPanel.qml | 67 +++++++++++ src/CMakeLists.txt | 2 + src/FaceRig/FaceRigAttach.cpp | 100 ++++++++-------- src/FaceRig/FaceRigAttach.h | 34 ++++++ src/FaceRigController.cpp | 216 ++++++++++++++++++++++++++++++++++ src/FaceRigController.h | 70 +++++++++++ src/mainwindow.cpp | 7 ++ 7 files changed, 447 insertions(+), 49 deletions(-) create mode 100644 src/FaceRigController.cpp create mode 100644 src/FaceRigController.h diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 5f2d66cf9..dd300bf0b 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -9381,6 +9381,73 @@ Rectangle { } } + // ── Auto-generate ARKit blendshapes (#895) ────────────────── + // One-click: fit the ARKit template onto the selected FACE mesh + // (NRICP + deformation transfer) and attach the 52 ARKit-named + // morph targets, so the #869 face-capture panel drives them. + // Heavy — runs on a worker thread via FaceRigController; the + // button shows progress and disables while busy. + Rectangle { + id: arkitBtn + width: parent.width + height: 24 + radius: 3 + property bool canRun: FaceRigController.hasMeshSelection + && !FaceRigController.busy + opacity: canRun ? 1.0 : 0.5 + color: arkitMa.containsMouse && canRun + ? Qt.lighter(PropertiesPanelController.highlightColor, 1.1) + : PropertiesPanelController.highlightColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: FaceRigController.busy + ? (FaceRigController.status !== "" + ? FaceRigController.status : "Working…") + : "✨ Add ARKit Blendshapes (AI)" + color: PropertiesPanelController.textColor + font.pixelSize: 10 + font.bold: true + } + MouseArea { + id: arkitMa + anchors.fill: parent + hoverEnabled: true + enabled: arkitBtn.canRun + cursorShape: enabled ? Qt.PointingHandCursor : Qt.ForbiddenCursor + onClicked: FaceRigController.addArkitBlendshapesAsync(0, 8.0) + ToolTip.visible: containsMouse + ToolTip.text: FaceRigController.hasMeshSelection + ? "Auto-fit the ARKit blendshape template onto this face " + + "mesh and attach the 52 ARKit shapes. Humanoid faces " + + "only; a poor fit is rejected." + : "Select a face mesh first." + } + } + Connections { + target: FaceRigController + function onError(message) { + arkitStatus.text = "⚠ " + message + arkitStatus.color = "#d66" + } + function onCompleted(report) { + arkitStatus.text = "✓ Attached " + report.shapesAttached + + " ARKit shapes (fit " + + report.fitMeanResidualPct.toFixed(2) + "% mean)." + arkitStatus.color = PropertiesPanelController.textColor + morphCol.targets = MorphAnimationManager.morphTargetsForSelection() || [] + } + } + Text { + id: arkitStatus + width: parent.width + visible: text !== "" + wrapMode: Text.Wrap + font.pixelSize: 9 + color: PropertiesPanelController.textColor + text: "" + } + // ── Section 2: ANIMATION CLIPS ────────────────────────────── // Section header, shown only once shapes exist (clips animate // shapes, so they're meaningless without any). diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2da7626c6..140603c9e 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -125,6 +125,7 @@ SkinTokensPredictor.cpp SkinWeightsController.cpp AutoRig.cpp AutoRigController.cpp +FaceRigController.cpp UniRigPredictor.cpp FaceRig/ArkitTemplate.cpp FaceRig/NonRigidICP.cpp @@ -326,6 +327,7 @@ SkinTokensPredictor.h SkinWeightsController.h AutoRig.h AutoRigController.h +FaceRigController.h UniRigPredictor.h MotionInbetween.h MotionLibrary.h diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp index 9324cc34f..7387df9d0 100644 --- a/src/FaceRig/FaceRigAttach.cpp +++ b/src/FaceRig/FaceRigAttach.cpp @@ -58,54 +58,39 @@ void appendIndices(Ogre::IndexData* id, std::uint32_t offset, ibuf->unlock(); } -// One geometry owner (shared pool, or a per-submesh vertex data) with the -// pose target handle it maps to and its base offset into the combined set. -struct Owner { - unsigned short handle; // 0 = shared, 1..N = submesh index+1 - std::uint32_t base; // first combined-vertex index - int count; // vertices in this owner -}; - } // namespace -AttachReport attachFaceRig(Ogre::Entity* entity, - const ArkitTemplate& tmpl, - const FaceRigOptions& opts) +FaceRigGeometry extractGeometry(Ogre::Entity* entity) { - AttachReport rep; - if (!entity) { rep.error = QStringLiteral("no entity"); return rep; } + FaceRigGeometry geo; + if (!entity) return geo; Ogre::MeshPtr mesh = entity->getMesh(); - if (!mesh) { rep.error = QStringLiteral("entity has no mesh"); return rep; } - if (!tmpl.valid()) { rep.error = QStringLiteral("ARKit template not loaded"); return rep; } + if (!mesh) return geo; - // ── collect every geometry owner into ONE combined vertex/index set so the + // Collect every geometry owner into ONE combined vertex/index set so the // fit sees the whole face (a single-submesh head is the common case, but a // face split across submeshes still fits as one surface). Track each owner // so we can split the per-vertex deltas back onto the right pose handle. - std::vector userV; - std::vector userF; - std::vector owners; - auto addOwner = [&](unsigned short handle, Ogre::VertexData* vd, Ogre::IndexData* id) { std::vector pos; if (!extractPositions(vd, pos)) return; - const std::uint32_t base = std::uint32_t(userV.size() / 3); - owners.push_back({handle, base, int(pos.size() / 3)}); - userV.insert(userV.end(), pos.begin(), pos.end()); - appendIndices(id, base, userF); + const std::uint32_t base = std::uint32_t(geo.userV.size() / 3); + geo.owners.push_back({handle, base, int(pos.size() / 3)}); + geo.userV.insert(geo.userV.end(), pos.begin(), pos.end()); + appendIndices(id, base, geo.userF); }; if (mesh->sharedVertexData) { // shared pool → handle 0; its indices live per-submesh. std::vector pos; if (extractPositions(mesh->sharedVertexData, pos)) { - owners.push_back({0, 0, int(pos.size() / 3)}); - userV.insert(userV.end(), pos.begin(), pos.end()); + geo.owners.push_back({0, 0, int(pos.size() / 3)}); + geo.userV.insert(geo.userV.end(), pos.begin(), pos.end()); for (unsigned short si = 0; si < mesh->getNumSubMeshes(); ++si) { Ogre::SubMesh* sm = mesh->getSubMesh(si); if (sm && sm->useSharedVertices) - appendIndices(sm->indexData, 0, userF); + appendIndices(sm->indexData, 0, geo.userF); } } } @@ -114,34 +99,25 @@ AttachReport attachFaceRig(Ogre::Entity* entity, if (!sm || sm->useSharedVertices) continue; addOwner(static_cast(si + 1), sm->vertexData, sm->indexData); } + return geo; +} - if (userV.size() < 9 || userF.size() < 3) { - rep.error = QStringLiteral("could not read mesh geometry"); - return rep; - } - - // ── run the Ogre-free pipeline - const FaceRigResult res = buildFaceRig(userV, userF, tmpl, opts); - rep.userVertexCount = res.userVertexCount; - rep.fitMeanResidualPct = res.fitMeanResidualPct; - rep.fitMaxResidualPct = res.fitMaxResidualPct; - if (!res.ok) { - rep.error = QString::fromStdString(res.error); - return rep; - } - - // ── attach each shape as a pose + VAT_POSE clip, splitting the combined +void attachShapes(Ogre::Entity* entity, const FaceRigGeometry& geo, + const FaceRigResult& result, AttachReport& report) +{ + // Attach each shape as a pose + VAT_POSE clip, splitting the combined // deltas back onto each owner's handle. Reuse AddMorphTargetCommand's // redo() (the exact MorphCommands pose-build) so face capture drives these // with no new playback code; call redo() directly (headless-safe — no undo // stack required, GUI callers can wrap in UndoManager separately). - for (const FaceRigShape& shape : res.shapes) { + for (const FaceRigShape& shape : result.shapes) { std::vector slices; - for (const Owner& o : owners) { + for (const GeometryOwner& o : geo.owners) { MorphPoseSlice slice; slice.submeshHandle = o.handle; for (int i = 0; i < o.count; ++i) { const std::uint32_t gv = o.base + std::uint32_t(i); + if (size_t(gv) * 3 + 2 >= shape.userDeltas.size()) break; const float* d = &shape.userDeltas[size_t(gv) * 3]; if (d[0] == 0.0f && d[1] == 0.0f && d[2] == 0.0f) continue; slice.offsets[static_cast(i)] = @@ -152,12 +128,38 @@ AttachReport attachFaceRig(Ogre::Entity* entity, if (slices.empty()) continue; // shape moved nothing on this mesh AddMorphTargetCommand cmd(entity, shape.name, slices); cmd.redo(); - rep.shapesAttached++; + report.shapesAttached++; + } + report.ok = report.shapesAttached > 0; + if (!report.ok) + report.error = QStringLiteral("no blendshapes produced any vertex movement"); +} + +AttachReport attachFaceRig(Ogre::Entity* entity, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts) +{ + AttachReport rep; + if (!entity) { rep.error = QStringLiteral("no entity"); return rep; } + if (!entity->getMesh()) { rep.error = QStringLiteral("entity has no mesh"); return rep; } + if (!tmpl.valid()) { rep.error = QStringLiteral("ARKit template not loaded"); return rep; } + + const FaceRigGeometry geo = extractGeometry(entity); + if (!geo.valid()) { + rep.error = QStringLiteral("could not read mesh geometry"); + return rep; + } + + const FaceRigResult res = buildFaceRig(geo.userV, geo.userF, tmpl, opts); + rep.userVertexCount = res.userVertexCount; + rep.fitMeanResidualPct = res.fitMeanResidualPct; + rep.fitMaxResidualPct = res.fitMaxResidualPct; + if (!res.ok) { + rep.error = QString::fromStdString(res.error); + return rep; } - rep.ok = rep.shapesAttached > 0; - if (!rep.ok) - rep.error = QStringLiteral("no blendshapes produced any vertex movement"); + attachShapes(entity, geo, res, rep); return rep; } diff --git a/src/FaceRig/FaceRigAttach.h b/src/FaceRig/FaceRigAttach.h index 8ab37aae4..36113de22 100644 --- a/src/FaceRig/FaceRigAttach.h +++ b/src/FaceRig/FaceRigAttach.h @@ -15,6 +15,9 @@ #include +#include +#include + namespace Ogre { class Entity; } namespace FaceRig { @@ -29,6 +32,37 @@ struct AttachReport { QString templateFallback; // set when the template had to be downloaded/etc. }; +// One geometry owner (shared vertex pool, or a per-submesh vertex data) with +// the morph pose target handle it maps to and its base offset into the combined +// vertex set. Ogre's 1-based convention: 0 = shared, 1..N = submesh index+1. +struct GeometryOwner { + unsigned short handle; + std::uint32_t base; + int count; +}; + +// The user entity's geometry read out of Ogre once, so the heavy Ogre-free +// buildFaceRig() can run OFF the main thread (the GUI path) while extraction + +// attach stay on the main thread. Pure data — no Ogre handles retained. +struct FaceRigGeometry { + std::vector userV; // combined positions (Nu*3) + std::vector userF; // combined triangle indices + std::vector owners; + bool valid() const { return userV.size() >= 9 && userF.size() >= 3; } +}; + +// MAIN-thread: read the entity's combined geometry (locks Ogre hardware +// buffers — milliseconds). Empty/invalid result on failure. +FaceRigGeometry extractGeometry(Ogre::Entity* entity); + +// MAIN-thread: attach a computed FaceRigResult's shapes to `entity` as +// Ogre::Pose + VAT_POSE morph targets (via AddMorphTargetCommand), splitting +// the combined per-vertex deltas back onto each owner's handle. Fills the +// report's shapesAttached / ok. `geo.owners` must match the geometry the +// result was computed from. +void attachShapes(Ogre::Entity* entity, const FaceRigGeometry& geo, + const FaceRigResult& result, AttachReport& report); + // Runs the whole pipeline on `entity` using the given (already-loaded) template // and attaches the shapes as poses + a per-target VAT_POSE clip. Re-initialises // the live entity so the pose buffers exist (mirrors the morph/auto-rig path). diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp new file mode 100644 index 000000000..8aaaf47a4 --- /dev/null +++ b/src/FaceRigController.cpp @@ -0,0 +1,216 @@ +#include "FaceRigController.h" + +#include "FaceRig/ArkitTemplate.h" +#include "FaceRig/FaceRigAttach.h" +#include "GamificationManager.h" +#include "Manager.h" +#include "SelectionSet.h" +#include "SentryReporter.h" +#include "UndoManager.h" +#include "commands/MorphCommands.h" + +#include +#include +#include + +#include +#include + +#include +#include +#include + +FaceRigController* FaceRigController::m_pSingleton = nullptr; + +FaceRigController* FaceRigController::instance() +{ + if (!m_pSingleton) + m_pSingleton = new FaceRigController(); + return m_pSingleton; +} + +FaceRigController* FaceRigController::qmlInstance(QQmlEngine* engine, QJSEngine*) +{ + Q_UNUSED(engine); + auto* inst = instance(); + QQmlEngine::setObjectOwnership(inst, QQmlEngine::CppOwnership); + return inst; +} + +void FaceRigController::kill() +{ + delete m_pSingleton; + m_pSingleton = nullptr; +} + +FaceRigController::FaceRigController() : QObject(nullptr) +{ + connect(SelectionSet::getSingleton(), &SelectionSet::selectionChanged, + this, &FaceRigController::selectionChanged); +} + +void FaceRigController::setStatus(const QString& s) +{ + if (m_status == s) return; + m_status = s; + emit statusChanged(); +} + +bool FaceRigController::hasMeshSelection() const +{ + auto* sel = SelectionSet::getSingleton(); + if (!sel) return false; + const auto entities = sel->getResolvedEntities(); + if (entities.isEmpty()) return false; + Ogre::Entity* first = entities.first(); + return first && first->getMesh(); +} + +bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidualPct) +{ + if (m_busy) { + emit error(QStringLiteral("A face-rig is already running.")); + return false; + } + SentryReporter::addBreadcrumb(QStringLiteral("ui.action"), + QStringLiteral("Add ARKit Blendshapes requested")); + + auto* sel = SelectionSet::getSingleton(); + const auto entities = sel ? sel->getResolvedEntities() + : QList{}; + Ogre::Entity* entity = entities.isEmpty() ? nullptr : entities.first(); + if (!entity || !entity->getMesh()) { + emit error(QStringLiteral("No mesh selected.")); + return false; + } + + // MAIN thread: read the entity geometry (locks Ogre buffers — ms). + auto geo = std::make_shared( + FaceRig::extractGeometry(entity)); + if (!geo->valid()) { + emit error(QStringLiteral("Could not read the mesh geometry.")); + return false; + } + + // MAIN thread: ensure + load the bundled template (may download on first + // use — surface that to the UI). ensureModelBlocking() can take a while on + // a first-run download, so show "Downloading…" first. + m_downloading = !FaceRig::ArkitTemplate::present(); + setStatus(m_downloading ? QStringLiteral("Downloading face template…") + : QStringLiteral("Preparing…")); + const QString path = FaceRig::ArkitTemplate::ensureModelBlocking(); + m_downloading = false; + if (path.isEmpty()) { + setStatus(QString()); + emit error(QStringLiteral( + "ARKit template unavailable (offline and not yet downloaded, or " + "this build has no face-rig model).")); + return false; + } + auto tmpl = std::make_shared(); + QString loadErr; + if (!tmpl->load(path, &loadErr)) { + setStatus(QString()); + emit error(QStringLiteral("Failed to load ARKit template: %1").arg(loadErr)); + return false; + } + + FaceRig::FaceRigOptions opts; + opts.maxShapes = maxShapes; + opts.maxFitResidualPct = maxResidualPct; + + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QStringLiteral("face_rig entity=%1 verts=%2 template=%3v") + .arg(QString::fromStdString(entity->getName())) + .arg(geo->userV.size() / 3).arg(tmpl->vertexCount())); + + m_busy = true; + emit busyChanged(); + setStatus(QStringLiteral("Fitting face template…")); + + const std::string entName = entity->getName(); + QPointer self(this); + + // WORKER thread: the heavy Ogre-free fit + transfer over all 52 shapes. + std::thread([self, geo, tmpl, opts, entName]() { + auto result = std::make_shared( + FaceRig::buildFaceRig(geo->userV, geo->userF, *tmpl, opts)); + + // MAIN thread: attach (touches Ogre + the undo stack). + QMetaObject::invokeMethod(qApp, [self, geo, result, entName]() { + if (!self) return; + self->m_busy = false; + emit self->busyChanged(); + self->setStatus(QString()); + + if (!result->ok) { + emit self->error(QString::fromStdString(result->error)); + return; + } + + // Resolve the entity again by name — the selection may have changed + // while the worker ran; only attach if it's still around. + Ogre::Entity* entity = nullptr; + for (Ogre::Entity* e : Manager::getSingleton()->getEntities()) { + if (e && e->getMovableType() == "Entity" + && e->getName() == entName) { entity = e; break; } + } + if (!entity) { + emit self->error(QStringLiteral( + "The mesh went away before the face-rig could be applied.")); + return; + } + + // Undoable group: one macro so Ctrl+Z removes all shapes at once. + FaceRig::AttachReport rep; + rep.userVertexCount = result->userVertexCount; + rep.fitMeanResidualPct = result->fitMeanResidualPct; + rep.fitMaxResidualPct = result->fitMaxResidualPct; + + auto* undo = UndoManager::getSingleton(); + auto* stack = undo ? undo->stack() : nullptr; + if (stack) stack->beginMacro(QStringLiteral("Add ARKit Blendshapes")); + for (const FaceRig::FaceRigShape& shape : result->shapes) { + std::vector slices; + for (const FaceRig::GeometryOwner& o : geo->owners) { + MorphPoseSlice slice; + slice.submeshHandle = o.handle; + for (int i = 0; i < o.count; ++i) { + const std::uint32_t gv = o.base + std::uint32_t(i); + if (size_t(gv) * 3 + 2 >= shape.userDeltas.size()) break; + const float* d = &shape.userDeltas[size_t(gv) * 3]; + if (d[0] == 0.0f && d[1] == 0.0f && d[2] == 0.0f) continue; + slice.offsets[static_cast(i)] = + Ogre::Vector3f(d[0], d[1], d[2]); + } + if (!slice.offsets.empty()) slices.push_back(std::move(slice)); + } + if (slices.empty()) continue; + undo->push(new AddMorphTargetCommand(entity, shape.name, slices)); + rep.shapesAttached++; + } + if (stack) stack->endMacro(); + rep.ok = rep.shapesAttached > 0; + + if (!rep.ok) { + emit self->error(QStringLiteral( + "No blendshapes produced any vertex movement.")); + return; + } + + GamificationManager::noteOperation( + QStringLiteral("morph"), + {{QStringLiteral("blendshapes_attached"), rep.shapesAttached}}, + GamificationManager::Surface::Gui); + + QVariantMap map; + map["shapesAttached"] = rep.shapesAttached; + map["userVertexCount"] = rep.userVertexCount; + map["fitMeanResidualPct"] = rep.fitMeanResidualPct; + map["fitMaxResidualPct"] = rep.fitMaxResidualPct; + emit self->completed(map); + }, Qt::QueuedConnection); + }).detach(); + + return true; +} diff --git a/src/FaceRigController.h b/src/FaceRigController.h new file mode 100644 index 000000000..4622853d4 --- /dev/null +++ b/src/FaceRigController.h @@ -0,0 +1,70 @@ +#ifndef FACE_RIG_CONTROLLER_H +#define FACE_RIG_CONTROLLER_H + +#include +#include +#include + +// QML-facing singleton for the face auto-rig (#889, Slice F #895). Wraps +// FaceRig::attachFaceRig* plus selection state so the Inspector's "Add ARKit +// Blendshapes" button can enable itself only on a mesh selection and run the +// (heavy) fit on a worker thread while the UI stays responsive. +// +// The pipeline is split across threads: geometry extraction + the pose attach +// touch Ogre and run on the MAIN thread; the heavy Ogre-free buildFaceRig() +// (NRICP + deformation transfer over 52 shapes) runs on a WORKER. The attach +// goes through AddMorphTargetCommand so it is undoable, and it lands the +// shapes in the same Ogre::Pose + VAT_POSE form the Vertex Morph section reads, +// so the #869 face-capture panel drives them immediately. +class FaceRigController : public QObject +{ + Q_OBJECT + QML_ELEMENT + QML_SINGLETON + + Q_PROPERTY(bool hasMeshSelection READ hasMeshSelection NOTIFY selectionChanged) + Q_PROPERTY(bool busy READ busy NOTIFY busyChanged) + // True while the bundled ARKit template is downloading on first use. + Q_PROPERTY(bool downloading READ downloading NOTIFY statusChanged) + // Short human-readable status for the button label / tooltip. + Q_PROPERTY(QString status READ status NOTIFY statusChanged) + +public: + static FaceRigController* instance(); + static FaceRigController* qmlInstance(QQmlEngine* engine, QJSEngine* scriptEngine); + static void kill(); + + bool hasMeshSelection() const; + bool busy() const { return m_busy; } + bool downloading() const { return m_downloading; } + QString status() const { return m_status; } + + /// Run the face auto-rig on the first selected entity and attach the ARKit + /// blendshapes. Prepares (extracts geometry + loads the bundled template) + /// on the main thread, runs the fit on a WORKER, commits the undoable + /// attach back on the main thread. Returns false when it could not start + /// (invalid selection / already busy — `error` is emitted); the real + /// outcome arrives via completed(report) or error(msg). + Q_INVOKABLE bool addArkitBlendshapesAsync(int maxShapes = 0, + double maxResidualPct = 8.0); + +signals: + void selectionChanged(); + void busyChanged(); + void statusChanged(); + void completed(const QVariantMap& report); + void error(const QString& message); + +private: + FaceRigController(); + ~FaceRigController() override = default; + + void setStatus(const QString& s); + + static FaceRigController* m_pSingleton; + bool m_busy = false; + bool m_downloading = false; + QString m_status; +}; + +#endif // FACE_RIG_CONTROLLER_H diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index d11a89be3..b1319e2c6 100755 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -118,6 +118,7 @@ #include "UVEditorController.h" #include "QuadRetopoController.h" #include "SkinWeightsController.h" +#include "FaceRigController.h" #include "LightsController.h" #include "LightRigLibrary.h" #include "SceneLightingController.h" @@ -573,6 +574,7 @@ MainWindow::~MainWindow() UVEditorController::kill(); QuadRetopoController::kill(); SkinWeightsController::kill(); + FaceRigController::kill(); LightsController::kill(); LightPropertiesController::kill(); SceneLightingController::kill(); @@ -766,6 +768,11 @@ void MainWindow::initToolBar() [](QQmlEngine* engine, QJSEngine*) -> QObject* { return SkinWeightsController::qmlInstance(engine, nullptr); }); + qmlRegisterSingletonType( + "PropertiesPanel", 1, 0, "FaceRigController", + [](QQmlEngine* engine, QJSEngine*) -> QObject* { + return FaceRigController::qmlInstance(engine, nullptr); + }); qmlRegisterSingletonType( "PropertiesPanel", 1, 0, "LightsController", [](QQmlEngine* engine, QJSEngine*) -> QObject* { From fbed51409f803945838304f9749060f71e7728c7 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 11:39:03 -0300 Subject: [PATCH 07/38] feat(#889): quality pass, docs, packaging, telemetry (Face-Rig Slice G) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finish the face auto-rig epic. Quality pass — recover ARKit names past the Assimp glTF gap: - FaceRig::writeArkitSidecar() writes a `.arkit.json` sidecar (schema qtmesh-arkit-blendshapes-v1: ordered shape names) next to the exported mesh, because Assimp 6.0's glTF exporter drops mesh.extras.targetNames even when aiAnimMesh::mName is set. `qtmesh mocap --face` / re-import can rebind the mocap-52 vocabulary by index. AttachReport now carries the ordered shapeNames; the CLI + MCP export paths write the sidecar. Verified: 51 ordered names written alongside the glb. Docs: - docs/FACE_RIG.md — user + developer guide (GUI/CLI/MCP usage, the NRICP → deformation-transfer → resample pipeline, quality numbers, limits). - THIRD_PARTY_AI_MODELS.md — ICT-FaceKit MIT licensing verdict (MIT tier only; the USC "full model" tier is REJECTED), hosting + first-use-download notes. - CLAUDE.md — FaceRig architecture section + `qtmesh facerig` CLI reference + added `facerig` to the recognized-subcommand list. - README.md — `qtmesh facerig` quick-start lines. - printUsage() now lists `facerig`. - .gitignore allowlists docs/FACE_RIG.md. Packaging (from Slice B, verified here): scripts/export-arkit-template.py packs the ICT MIT head + 52 shapes → facerig/arkit_template.bin; scripts/upload-facerig-template.sh uploads it (+ the ICT LICENSE) to the HF models repo via `hf upload`; downloads on first use. Telemetry: Sentry `ai.assist.face_rig` on all three surfaces (CLI/MCP/GUI); gamification noteOperation on the auto_rig / morph clusters. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 1 + CLAUDE.md | 5 ++- README.md | 4 ++ THIRD_PARTY_AI_MODELS.md | 28 ++++++++++++ docs/FACE_RIG.md | 85 +++++++++++++++++++++++++++++++++++ src/CLIPipeline.cpp | 11 +++++ src/FaceRig/FaceRigAttach.cpp | 28 ++++++++++++ src/FaceRig/FaceRigAttach.h | 7 +++ src/MCPServer.cpp | 2 + 9 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 docs/FACE_RIG.md diff --git a/.gitignore b/.gitignore index a9421e6f3..e5a11ebc6 100755 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ docs/* !docs/IMAGE_TO_3D_QUALITY.md !docs/TRIPOSG_EXPORT_NOTES.md !docs/MESH_SEGMENTATION_STRATEGY.md +!docs/FACE_RIG.md # minisign — never commit secret keys minisign.key diff --git a/CLAUDE.md b/CLAUDE.md index c14b9e034..3ef98c06e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -140,6 +140,8 @@ qtmesh skin ours.fbx --compare mixamo_ref.fbx [--json] # per-vertex weight diff qtmesh rig model.obj --skeleton humanoid -o rigged.fbx # native auto-rig: embed a skeleton template into an unrigged mesh (#407) qtmesh rig model.obj --skeleton humanoid --skin -o rigged.fbx # one-click rig + skin (chains #402); templates: humanoid|biped|quadruped|generic; --up-axis x|y|z (default y) qtmesh rig model.obj --algo unirig -o rigged.fbx # ML skeleton prediction via ONNX UniRig (#408, MIT model); default --algo pinocchio (offline). UniRig falls back to the template when the model/ONNX is unavailable +qtmesh facerig head.glb -o rigged.glb # #889: auto-generate the 52 ARKit blendshapes on a humanoid FACE mesh (fit ICT template via non-rigid ICP + Sumner-Popović deformation transfer, attach as named morph targets). A poor fit (non-face mesh) is rejected. Bundled template downloads on first use. Feeds `qtmesh mocap --face` +qtmesh facerig head.fbx -o rigged.glb --max-shapes 20 --max-residual 5 --json # cap shape count / tighten the humanoid gate / machine-readable report qtmesh generate3d image.png -o out.glb # AI image-to-3D (#764, TripoSR/ONNX): reconstruct a mesh from a single image (needs ONNX build + model; downloads on first use, clear message if not hosted) qtmesh generate3d photo.png --remove-bg -o out.glb # run U²-Net background removal first (needed for photos with a background; TripoSR wants an isolated subject) qtmesh generate3d image.png --resolution 128 --no-color -o out.glb # faster/preview marching-cubes grid; skip vertex color @@ -173,7 +175,7 @@ qtmesh ps1 dump-vram game.cue --bios scph1001.bin --frames 300 -o vram.png # sn # xvfb-run -a qtmesh ps1 capture game.cue --bios scph1001.bin -o out.gltf ``` -CLI mode is activated by: (1) invoking via the `qtmesh` symlink, (2) passing `--cli`, or (3) using a recognized subcommand (`info`, `fix`, `convert`, `anim`, `validate`, `lod`, `pose`, `turntable`, `isometric`, `scan`, `material`, `hdri`, `light`, `pack-textures`, `normal-from-height`, `atlas`, `atlas-apply`, `memory`, `analyze`, `vertex-cache`, `decimate`, `optimize`, `uv`, `retopo`, `skin`, `rig`, `segment`, `generate3d`, `ps1`, `cloud`) as the first argument. Use `--verbose` to see Ogre/engine debug output. Use `--no-telemetry` to permanently opt out of anonymous usage data collection. +CLI mode is activated by: (1) invoking via the `qtmesh` symlink, (2) passing `--cli`, or (3) using a recognized subcommand (`info`, `fix`, `convert`, `anim`, `validate`, `lod`, `pose`, `turntable`, `isometric`, `scan`, `material`, `hdri`, `light`, `pack-textures`, `normal-from-height`, `atlas`, `atlas-apply`, `memory`, `analyze`, `vertex-cache`, `decimate`, `optimize`, `uv`, `retopo`, `skin`, `rig`, `facerig`, `segment`, `generate3d`, `ps1`, `cloud`) as the first argument. Use `--verbose` to see Ogre/engine debug output. Use `--no-telemetry` to permanently opt out of anonymous usage data collection. If Xcode SDK is updated, clear CMake cache (`rm build_local/CMakeCache.txt`) and reconfigure. @@ -379,6 +381,7 @@ The animation pipeline started skeleton-only; the #517 epic broadens it. Slices - **Skinning v2** (issue #819, Slices A+B): `SkinWeights::Algorithm { InverseDistance, GeodesicVoxel, SkinTokens }` — **SkinTokens (the ML skinner) is the default** on every surface (GUI dialog dropdown, CLI `qtmesh skin --algo skintokens|geodesic-voxel|inverse-distance`, MCP `compute_skin_weights` `algo` param; "unirig" survives as a deprecated alias of skintokens), falling back to GeodesicVoxel when models/ONNX are unavailable — so headless/CI environments transparently get the geodesic bind. `qtmesh rig --skin`, MCP `auto_rig {skin:true}` and the GUI rig+skin checkbox all chain through the same default. **GeodesicVoxelBind** (`src/GeodesicVoxelBind.h/cpp`, Ogre-free + unit-tested) implements Maya's "Geodesic Voxel" bind (Dionne & de Lasa, SCA 2013): voxelize the surface at `--voxel-res` (default 64, max 256; Akenine-Möller tri/box SAT), flood-fill the exterior to classify interior (closes holes at voxel resolution → works on non-watertight/self-intersecting/multi-component meshes), 3D-DDA bone rasterization with snap-to-solid (max 4.5 voxels — far-outside bones get NO seeds and are reported in `bonesWithoutSeeds`), one multi-source Dijkstra over solid voxels (26-conn) keeping the best K=8 `(bone, distance)` pairs per voxel, then per-vertex `(1/d)^falloff` weighting. Distances travel through the volume so cross-limb bleed (hand-near-thigh, inner thighs) is impossible by construction. Degenerate input (planes/cloth — zero interior voxels) falls back to InverseDistance automatically; vertices in bone-less floating islands get inverse-distance fill so they still move with the rig. `SkinTokens` falls back to GeodesicVoxel when its models/ONNX are unavailable (Slice C). **SkinWeightsPost** (`src/SkinWeightsPost.h/cpp`, Ogre-free + unit-tested) runs after EVERY algorithm inside `computeAndApply`: Laplacian relaxation over the vertex adjacency (`--smooth-iterations`, default 3, 0=off; merge-mode manual weights act as Dirichlet constraints — they influence neighbours but are never modified) then prune <0.01 + top-K + renormalize. Report gains `algorithmUsed`, `fallbackReason`, `bleedFraction` (fraction of committed weights not geodesically local — 0 for GVB by construction), `bonesWithoutSeeds`. Sentry breadcrumbs `ai.assist.skin.`. `qtmesh rig --skin` and MCP `auto_rig {skin:true}` chain through the same default. **Slice D — dual-quaternion display toggle**: `SkinningDisplay` (`src/SkinningDisplay.h/cpp`) toggles RTSS hardware skinning per entity (Linear = default path; Dual Quaternion = `HardwareSkinningFactory::prepareEntityForSkinning(ST_DUAL_QUATERNION)` + technique invalidation — kills candy-wrapper collapse on twists). The HS factory + a dormant template SRS are registered in `RTShaderHelper::initialize` (bone cap 96; above-cap entities stay on the default path); Linear mode erases the material's `HS_SRS_DATA` imprint (mirrors Ogre's file-local constant). Display only — exported weights unchanged. Surfaced as the "Display: Linear | Dual Quaternion" row in the Animation-mode Skinning section and MCP `set_skinning_display {mode}`; mode tracked on the entity's UserObjectBindings; Sentry `render.skinning`. **Slice E — evaluation suite**: `SkinMetrics` (`src/SkinMetrics.h/cpp`, pure-data: influence histogram, Laplacian weight-smoothness energy, LBS deform + signed mesh volume) and `SkinEvaluate` (`src/SkinEvaluate.h/cpp`: extract EXISTING weights from an entity, metric report incl. geodesic bleed, position-matched/name-matched comparison vs a reference-skinned copy — equidistant duplicate verts tie-break on minimum weight diff so seams/contact points don't report spurious diffs). CLI `qtmesh skin --evaluate` / `--compare `; acceptance fixtures in `SkinMetrics_test.cpp` (90° elbow capsule volume ≥ 0.9 — measured 0.911; proximity bleed 0 vs inverse-distance >0.1; smoothing strictly reduces energy); Mixamo protocol + thresholds in `docs/SKINNING_QUALITY.md`; env-gated reference test via `QTMESH_SKIN_OURS_FBX`/`QTMESH_SKIN_REF_FBX`. **Slice C — SkinTokens ML skinning (implemented, the DEFAULT per user preference — visually the best skinner in practice)**: `Algorithm::SkinTokens` runs **SkinTokens/TokenRig** (VAST-AI, MIT code + MIT weights; Qwen3-0.6B backbone) — UniRig's own skin head stays blocked on spconv/PTv3 (no ONNX lowering; decision record in THIRD_PARTY_AI_MODELS.md). **SkinTokensPredictor** (`src/SkinTokensPredictor.h/cpp`, the EIGHTH ONNX consumer; pure-data parts unit-tested): surface-sample `num_points` (8192) points+normals, normalise mesh+joints per the upstream AugmentAffine (uniform scale, joints included in the AABB, exact [-1,1] fit), tokenize the skeleton TEACHER-FORCED (DFS stream, multi-root topologies re-parented to the first root, "articulation" cls token), then five graphs: `mesh_cond`/`vae_cond`/`embed`/`decoder` (Qwen3 KV-cache step; ships as proto + `decoder.onnx.data` external weights — ORT 1.20.1 SIGSEGVs parsing the 1.66GB single-file proto)/`skin_decode` (FSQ folded in), greedy skin-token decode constrained to the FSQ vocab, per-joint weight decode, 8-NN IDW transfer to full-res verts. **Geodesic localisation pass** in the dispatch (SkinWeights.cpp): raw SkinTokens weights are diffuse (the upstream demo voxel-masks them by default) — we filter per-vertex to GVB's geodesically-local bone sets + renormalise (bandit: bleed 0.74→0.05, mean L1 vs artist weights 1.72→1.22; GVB baseline 1.09 on that metric, but visual quality favours the ML result, hence the default). Export: `scripts/export-skintokens-onnx.py` (offline; flash-attn shim + eager attention + CPU stubs + traced FPS + RMSNorm decomposition + transformers-5.x cache API; parity vs torch ~1e-5 on every graph); hosting: `scripts/upload-skintokens-models.sh` → HF models repo `skintokens/` (~2.3GB, downloads on first use; `QTMESH_SKINTOKENS_MODEL_BASE_URL`/`ai/skintokensModelBaseUrl`, guard `QTMESH_SKINTOKENS_NO_DOWNLOAD`). ~12 min for 119 bones/90k verts on an M-series CPU — report says `algorithmUsed: "skintokens"`; every failure falls back to GeodesicVoxel with a reason. Ort C++ footgun for future consumers: `GetTensorTypeAndShapeInfo()` is a NON-OWNING view — never chain it off a temporary `TypeInfo`. Debug tracing: `QTMESH_SKINTOKENS_DEBUG=1`. - **SkinWeights** (`src/SkinWeights.h/cpp`, issue #402): inverse-distance ("closest-point-on-bone") automatic skin weights (now the fallback algorithm — see Skinning v2 above). The issue proposed wrapping libigl's bounded biharmonic weights (BBW), but BBW requires tetrahedralization via TetGen — which is **GPL/copyleft**. Adopting it would force the entire binary to GPL and close off Homebrew / Snap / WinGet redistribution under the project's permissive-license stance. This first slice ships a native heuristic with **zero new dependencies**: for each vertex, compute its distance to every bone's segment (line from bone-head to the average of its children, falling back to point distance for leaf bones in the skeleton's bind pose), apply `1/dist^falloff` weighting, keep the top-K bones (default K=4 matches hardware skinning), and normalize. This is the same algorithm Maya / 3dsMax use as their default "smooth bind." Distance cap (`maxInfluenceDistance` × mesh-diagonal) prevents a finger bone from picking up weight on a foot. Optional `skipUnweightedBones` filters Mixamo helper bones. `replaceExisting=false` enables a merge mode for "fill in missing weights" workflows. Surfaced via `qtmesh skin --max-influences N --falloff F -o out`, MCP `compute_skin_weights`, and the **Animation Mode → Mode Tools → "Skinning" section → "Compute Skin Weights…" button** (`qml/SkinWeightsDialog.qml`, driven by `SkinWeightsController` singleton). Lives in Animation Mode (not Edit Mode) because skinning governs how the mesh deforms under animation — a rigging step, not a mesh-topology edit. The button binds to `hasSkinnedSelection` so it disables on static (skeleton-less) meshes. The GUI path runs through `ComputeSkinWeightsCommand` (`src/commands/`) so the auto-skin is **undoable** (Ctrl+Z): the command snapshots every submesh's `VertexBoneAssignmentList` (+ the mesh-level shared list) before the first `redo`, runs `computeAndApply`, and on `undo` restores the snapshot and calls `_compileBoneAssignments` to re-pack the blend buffer. (Unlike the UV-unwrap restore, recompiling is safe here because the vertex buffer object is unchanged — only the blend bytes are rewritten.) Sentry breadcrumb category `ai.assist.skin_weights`. A future slice can plug libigl BBW in behind `-DENABLE_LIBIGL_BBW` for users who accept the GPL implications. Verified on Rumba Dancing.fbx: 69 bones, 5828 verts → 20,129 vertex-bone assignments (avg 3.45 influences/vert), valid glTF round-trip. - **AutoRig** (`src/AutoRig.h/cpp`, issue #407): native automatic rigging — predicts a skeleton for an unrigged mesh. The issue proposed wrapping **Pinocchio** (Baran & Popović, SIGGRAPH 2007), but Pinocchio's **core library is LGPL-2.1-or-later** (only its demo CLI is MIT). Statically vendoring LGPL imposes relink / object-file obligations that conflict with this project's statically-linked, permissively-redistributed binaries (Homebrew / Snap / WinGet / Docker) — the same reason #401 (Instant Meshes / QuadriFlow) and #402 (libigl BBW needs GPL TetGen) shipped native heuristics. Pinocchio's *algorithm* (embed a skeleton template into the mesh interior) is published and unencumbered; only its code is LGPL, so this is a from-scratch native implementation with **zero new dependencies**. Pipeline: (1) read mesh vertices → AABB; (2) each built-in template (humanoid 19-bone / biped / quadruped / generic) is a proportional joint graph in a normalised unit box; map every joint into the AABB; (3) recentre flagged joints (spine, limb roots) toward the centroid of the vertices in a thin slab at the joint's up-height — pulls the spine onto the medial line and lands limb roots inside the silhouette. `rigEntity()` builds an `Ogre::Skeleton` (parent-relative bone positions, `setBindingPose`), binds via `mesh->_notifySkeleton` **+ `entity->_initialise(true)`** — the re-initialise is REQUIRED or both exporters (FBXExporter and the Assimp glTF/FBX path gate on `entity->hasSkeleton()`) silently drop the new rig. Pure-data core (`templateJoints` / `fitTemplate`) is unit-tested without GL. Surfaced via `qtmesh rig [--skeleton T] [--skin] [--up-axis x|y|z] -o out` (`CLIPipeline::cmdRig`, optionally chains `SkinWeights::computeAndApply` for one-click rig+skin), MCP `auto_rig` `{template, skin?, up_axis?, output_path?}` (`MCPServer::toolAutoRig`), and the **Animation Mode → Mode Tools → "Rigging" section → "Auto-Rig…" button** (`qml/AutoRigDialog.qml`, driven by `AutoRigController` singleton, gated on `hasRiggableSelection` — a static/skeleton-less mesh; already-rigged meshes show the "Skinning" section instead). Sentry breadcrumb category `ai.assist.auto_rig`. **Quality limits** (documented per the issue, like Pinocchio): heuristic embedding — works best on roughly upright, single-component, manifold, T/A-pose meshes with +Y up; it does not detect limbs from topology, so exotic proportions or non-upright poses can misplace joints. Verified end-to-end: a static OBJ → 19-bone humanoid + skin → glTF export with 1 skin / 17 joints. **Mixamo-style marker placement** (refinement over the proportional fit): the user clicks the 10 humanoid markers on the mesh surface in the viewport (chin, L/R shoulder, L/R wrist, L/R hip, L/R knee, hips/pelvis — `AutoRig::humanoidMarkerOrder()`), and each placed marker anchors its joint while the limb/spine chains interpolate between the anchors so the rig follows actual body proportions instead of the fixed template. The pure-data core is `AutoRig::fitTemplateWithMarkers` and it does **coherent inference**, not per-marker patching: it runs `fitTemplate` for a proportional baseline (and to read the template's segment vectors / lateral offsets), then **resolves an anchor for every key joint** (Head, L/R Shoulder, L/R Hand, Hips, L/R UpLeg, L/R Knee) as *marked → inferred-from-marked-neighbours → template* and lays the dependent chains (spine, arms, legs) from those anchors — so a partial marker set yields an anatomically-sane skeleton instead of mixing marked anchors with stranded template joints (no shoulder-above-head). Inference: Hips ← midpoint of marked up-legs + template socket→pelvis rise; Head ← template offset above resolved Hips; UpLeg ← mirror the other up-leg across the pelvis, else pelvis + template socket offset; Shoulder ← along the live Hips→Head line at the template shoulder-height fraction + template lateral offset, else mirror the other; Hand ← shoulder + template arm vector (marked shoulder + skipped wrist still lays a full arm); Knee/foot ← clamped to the mesh AABB so an inferred leg never punches through the model: when the knee is skipped, the foot is dropped straight to the mesh FLOOR (`mn[up]`) below the up-leg and the knee placed halfway between (template thigh-vector extrapolation, which used to shoot feet past the lower limit, is only used for the small forward knee nudge); a marked knee keeps its position with the foot extrapolated below but still floor-clamped. Mirroring reflects across the sagittal plane (side axis auto-detected). An empty marker set early-returns `fitTemplate` unchanged; `report.markersApplied` counts only user-placed markers. (Legacy per-marker description retained below for the chain mechanics.) The old behaviour was: Hips→anchor pelvis AND carry the thigh roots (LeftUpLeg/RightUpLeg, children of Hips) by the same delta so the whole pelvis+thigh cluster moves as a unit — unless an explicit hip marker overrides; Chin→anchor Head AND lay the spine straight up from the pelvis — Spine/Chest/Neck distributed evenly between Hips and Head by index (cartoon torso lengths vary too much for a proportional guess); L/R shoulder→anchor the arm-chain attach point (applied before the wrist so the chain lays from the marked shoulder); L/R wrist→`layChain` lays the WHOLE arm straight from the shoulder anchor — Shoulder[anchor]→Arm[⅓]→ForeArm[⅔]→Hand[marker] — distributing every intermediate joint so the entire arm reaches the wrist, not just the hand; L/R hip→anchor the thigh root/hip socket (applied before the knee, overrides the hips-carry — needed for cartoon legs that splay at odd angles); L/R knee→`layLeg` anchors the knee at the marker and continues the foot below it along the thigh→knee direction (so the whole leg — hip socket → knee → foot — follows the marked hip + knee)). `layChain` is generic (anchor-first, marker-last, evens the middle by index) so adding more chain joints is a one-line change. Every marker is OPTIONAL — unset markers keep the template fit (`report.markersApplied` counts the placed ones; an empty marker set is bit-identical to `fitTemplate`). The viewport flow lives in `AutoRigController` (marker-session state machine: `beginMarkerPlacement`/`skipCurrentMarker`/`undoLastMarker`/`cancelMarkerPlacement`/`commitMarkerRig`); clicks are routed in by `TransformOperator::mousePressEvent` (checked **before** the knife/select paths when `markerMode()` is true), ray-cast to the mesh surface (`getCameraToViewportRay` → Möller-Trumbore against world-space triangles), stored in mesh-local space, and shown as unlit-yellow `PT_SPHERE` overlays. **The whole UI is inline in the Inspector's "Rigging" section** (`riggingToolsComponent` in `qml/PropertiesPanel.qml`) — there is no separate dialog (the old `AutoRigDialog.qml` was removed). It show/hides smartly: idle shows the two entry points ("Place markers…" / "Auto-Rig (template)"), a skin checkbox, and an "Advanced options" checkbox that reveals the template + up-axis pickers; while `markerMode` is active it swaps to the per-marker guidance label + Skip/Undo/Cancel/"Rig from markers" controls. Rig state + the `runAutoRig`/`runMarkerRig` helpers live on the `PropertiesPanel` root; the section's `onSectionVisibleChanged` cancels any active marker session if the section disappears (mode change / deselect / re-rig), replacing the dialog's old `onClosing` cancel. No CLI/MCP marker surface — guided placement is inherently interactive. **UniRig ML backend** (issue #408): `AutoRig::Algorithm {Pinocchio, UniRig}` selects the skeleton-prediction backend (default Pinocchio — offline, deterministic). **UniRig** (Zhang et al., *"One Model to Rig Them All"*, SIGGRAPH 2025, VAST-AI-Research/UniRig — **MIT code + MIT weights**, trained on Articulation-XL2.0 **CC-BY-4.0**) is an autoregressive transformer that predicts a skeleton from the mesh geometry, handling arbitrary/non-humanoid topology better than the fixed template. It's the **second ONNX consumer** after #404 PbrMapSynth. **RigNet was rejected** for #408 (GPL code + unlicensed weights + non-public ModelsResource dataset — fails the project's permissive-redistribution bar); UniRig is the clean permissively-licensed alternative (see `THIRD_PARTY_AI_MODELS.md`). `UniRigPredictor` (`src/UniRigPredictor.h/cpp`, Ogre-free + unit-tested) is the C++/ONNX runtime that ports UniRig's skeleton stage: (1) surface-sample up to 65536 points + normals, normalise into a centred unit box (+Y up); (2) run the **Michelangelo encoder** (`encoder.onnx`, pc[1,N,3]+feats[1,N,3] → latent prefix); (3) **greedy/constrained autoregressive decode** over the ~350M causal-LM (`decoder.onnx`) with a manual KV-cache + the tokenizer's next-possible-token validity mask (a documented simplification of UniRig's beam+sampling — deterministic + exportable, still yields a valid tree); (4) the **exact tokenizer FSM** from `src/tokenizer/tokenizer_part.py` (256 coord bins, `continuous_range [-1,1]`, `undiscretize(t)=(t+0.5)/256*2-1`, branch/parent rules, vocab 267) → joints (de-normalised) + parent indices, parent-before-child ordered for Ogre. The detokenizer + `undiscretize` are public statics (`UniRigPredictor::detokenize`/`undiscretize`) so they're unit-tested without ONNX. Everything is `ENABLE_ONNX`-guarded; **two** model files (`AppData/ai_models/unirig/{encoder,decoder}.onnx`) download on first use via `ModelDownloader` (`ensureModelBlocking`, 180s timeout, returns the encoder path only when BOTH exist; base URL override `QTMESH_UNIRIG_MODEL_BASE_URL` / `QSettings ai/unirigModelBaseUrl`, offline guard `QTMESH_UNIRIG_NO_DOWNLOAD`). **UniRig falls back to Pinocchio** (logged in `report.fallbackReason`) when ONNX is off / the models are missing/offline/not-yet-hosted / prediction is unusable — reliable offline. **Design contract / hosting status:** UniRig is an autoregressive HF `AutoModelForCausalLM` + a Michelangelo perceiver — no single-graph ONNX export exists upstream; `scripts/export-unirig-onnx.py` (one-time, offline, NOT shipped) exports the encoder + a KV-cache decoder to the I/O `UniRigPredictor` targets (via `optimum`, with a hand-rolled fallback). Until the exported `.onnx` files are hosted on the HF models repo, the download 404s and the Pinocchio fallback runs — the plumbing + runtime are complete and ship today; hosting the export lights up the ML path with no code change. UniRig is marker-incompatible (markers are a template concept), so a marker-driven call always uses the template. Surfaced via CLI `qtmesh rig --algo pinocchio|unirig` (`CLIPipeline::cmdRig`; `rignet` accepted as a deprecated alias), MCP `auto_rig` `algo` param (`MCPServer::toolAutoRig`), and the Inspector Rigging-section **Algorithm** segmented picker; the report carries `algorithmUsed` + `fallbackReason`, and the Sentry `ai.assist.auto_rig` breadcrumb records the `algo`. (The early `qml/AutoRigDialog.qml` reference above is stale — the UI is inline in `qml/PropertiesPanel.qml`'s Rigging section.) +- **FaceRig** (`src/FaceRig/`, epic #889): auto-generate the 52 **ARKit blendshapes** on an unrigged humanoid **face** mesh so it can be driven by face performance capture (#869). **Deterministic geometry, no ML/ONNX, zero new dependencies** (the house rule, same as #402/#407). Pipeline: **`NonRigidICP`** (`NonRigidICP.{h,cpp}`, Amberg 2007 optimal-step, Slice C #892) fits the ICT-FaceKit template neutral onto the user's neutral → per-template-vertex correspondence; **`DeformationTransfer`** (`DeformationTransfer.{h,cpp}`, Sumner & Popović 2004, Slice D #893) builds each template triangle's deformation gradient `S = [e1' e2' n']·[e1 e2 n]⁻¹` (4th "normal" vertex trick, free per-triangle unknown, single-vertex gauge anchor) and solves one sparse least-squares per shape for the user-identity positions; **`FaceRigger`** (`FaceRigger.{h,cpp}`, Slice E #894) chains them and resamples template-topology deltas onto the real user verts via a dependency-free spatial-hash grid (built once for all 52 shapes) → per-user-vertex deltas per shape, named per `FaceCap::kBlendshapeNames`. The shared sparse solver is **`SparseSolve.{h,cpp}`** (CSR + CG on the normal equations — no Eigen). **`FaceRigAttach`** (`FaceRigAttach.{h,cpp}`) is the only Ogre-touching piece: `extractGeometry()` reads the entity's combined submesh geometry, and `attachShapes()` splits the deltas back per submesh handle and attaches them as `Ogre::Pose` + `VAT_POSE` morph targets via `AddMorphTargetCommand` (the exact MorphCommands pose-build, so face capture drives them with no new playback code). **Humanoid-only gate**: the NRICP fit residual is checked (`--max-residual`, default 8% of the mesh diagonal; non-finite / >5%-diverged fits treated as failed) — a non-face mesh fits poorly and is refused rather than emitting garbage. Surfaced via **CLI `qtmesh facerig -o [--max-shapes N] [--max-residual PCT] [--json]`** (`CLIPipeline::cmdFaceRig`), **MCP `add_arkit_blendshapes`** (`MCPServer::toolAddArkitBlendshapes`, `{max_shapes?, max_residual_pct?, output_path?}`, heavy), and the **Inspector Vertex Morph Animation → "✨ Add ARKit Blendshapes (AI)" button** (`FaceRigController` QML_SINGLETON — extracts + loads template on the main thread, runs the heavy fit on a WORKER thread, commits the attach as one undo macro on the main thread; gated on `hasMeshSelection`, shows "Downloading…/Fitting…" status). Template = **ICT-FaceKit** (MIT, `THIRD_PARTY_AI_MODELS.md`), packed by `scripts/export-arkit-template.py` → `facerig/arkit_template.bin`, hosted on the HF models repo (`scripts/upload-facerig-template.sh`), downloads on first use to `/ai_models/facerig/` (`ArkitTemplate::ensureModelBlocking`; overrides `QTMESH_FACERIG_MODEL_BASE_URL` / `QSettings ai/facerigModelBaseUrl`; offline guard `QTMESH_FACERIG_NO_DOWNLOAD`). Sentry breadcrumb `ai.assist.face_rig`. Verified end-to-end: real ICT template (26719v, 51 shapes) → decimated different-topology face (15755v), mean 0.008% / max 0.61% fit, 51 shapes attached, exported glb carries all 51 morph targets. **glTF export** carries the target geometry but not per-target `extras.targetNames` (a follow-up). Docs: `docs/FACE_RIG.md`; spike/contract: `docs/FACE_RIG_SPIKE.md`. - **QuadRetopo** (`src/QuadRetopo.h/cpp`, issue #401): triangle-pairing quad-dominant retopology. The issue proposed wrapping Instant Meshes (Wenzel Jakob), but Instant Meshes ships as a research GUI app with no clean C++ library API and has been dormant since 2016. QuadriFlow (the production-grade alternative used by Blender 3.0+) requires Boost + Eigen + LEMON — heavy deps the project doesn't currently use. This first slice ships a native triangle-pairing backend with **zero new dependencies**: walks every interior edge whose two adjacent faces are triangles and scores the merge by (1) coplanarity (dot product of triangle normals; default `maxAngleDeg=25°`), (2) quad shape (deviation of interior angles from 90°; default `shapeToleranceDeg=65°`), (3) aspect ratio (longest/shortest edge; default `maxAspectRatio=6.0`). Pairs are taken greedily best-first; each triangle claimed at most once. Quads are emitted with opposing-corner winding `(opposing0, sharedA, opposing1, sharedB)`. Output goes through `EditableSubMesh::faces` → `triangulateFaces` (fan retri for GPU) → `writeNgonFacesToMesh` (n-gon binding for exporters / Edit Mode). **No new vertices** are introduced, so UVs and skin weights survive unchanged. Backends are pluggable via the `Algorithm` enum (only `TrianglePair` implemented; future `QuadriFlow` / `InstantMeshes` slot in here). Surfaced via `qtmesh retopo --target-faces N --max-angle DEG -o out`, MCP `retopologize`, and the **Material Mode → Mode Tools → "Quad Retopology…" button** (`qml/QuadRetopoDialog.qml`, driven by `QuadRetopoController` singleton). Sentry breadcrumb category `ai.assist.retopo`. Verified on Rumba Dancing.fbx: 10,220 tris → 6,032 faces (4188 quads + 1844 tris), 82% quad dominance. Hard lower bound on face count is ~50% of input (every triangle paired); strict gates typically land 60-70%. - **MeshSegmenter** (`src/MeshSegmenter.h/cpp`, issue #410): AI mesh part segmentation — predicts a semantic part label (head/torso/left+right arm/left+right leg) per vertex + per face. The **fourth ONNX consumer**; powers Edit-Mode "Select by part", per-part material assignment, and auto-rig priors. **Geometric fallback is first-class** (always compiled, Ogre-free): `segmentGeometric` does connected-component islands (`connectedComponents`, union-find) + an up-axis/lateral spatial heuristic (top→head, lower→legs, mid-sides→arms, centre→torso), overridable per-vertex by rig bone-proximity hints — used automatically when the build lacks ONNX, the model is missing/un-downloadable, or inference fails (`Result::usedModel`/`fallbackReason` report which ran). The **ONNX path** (`#ifdef ENABLE_ONNX`, PointNet++-style) normalises → deterministic point sample → `[1,N,3]` → per-point argmax over the part channels → scatters labels back to all vertices by nearest sampled point (runtime I/O-name discovery, channels-first/last handling, CoreML EP). `ensureModelBlocking()` downloads `meshseg.onnx` to `AppData/ai_models/segment/` (override `QTMESH_SEGMENT_MODEL_BASE_URL` / `QSettings ai/segmentModelBaseUrl`; offline guard `QTMESH_SEGMENT_NO_DOWNLOAD`; non-ONNX `#ifndef` guard) — the #408/#409 pattern. Pure-data helpers (`connectedComponents`, `facesFromVertexLabels`) are unit-tested without Ogre/GL. Surfaced via **CLI `qtmesh segment [--json] [--no-model] [--up-axis x|y|z]`** (`CLIPipeline::cmdSegment` — text per-part counts or full label arrays), the **MCP `segment_mesh` tool** (`MCPServer::toolSegmentMesh`, args `{entity_name?, no_model?}`, heavy), and the **Edit Mode → "Select by Part (AI)" button** (`EditModeController::selectByPart()` → selects all faces matching the selected face's part, or the largest part if none selected; pushes via `selectFace` so the existing highlight refreshes). Sentry breadcrumb `ai.assist.segment`. **Model: ours (v2), trained on surface-sampled synthetic bodies (humanoid/chibi/quadruped/biped-tail plans) + mined CC0 Quaternius rigs** (rig bone-weight → part; ShapeNet-Part/PartNet are non-commercial and rejected) via `scripts/export-meshseg-onnx.py` (offline, not shipped) — the v2 loader canonicalises arbitrarily-oriented mined clouds from their own labels and geometrically fixes miner side errors; hosted on the HF models repo under `segment/` (see `THIRD_PARTY_AI_MODELS.md` + `docs/MESH_SEGMENTATION_STRATEGY.md` for the v1 failure analysis, accuracy numbers, and the multi-category roadmap). **Three-tier dispatch in `selectByPart`**: (1) **rig-prior** — if the mesh is SKINNED, label each vertex by the part of the bone it's most-weighted to (`AutoRig::rigPriorPartLabels` → `MeshSegmenter::partForBoneName`); EXACT and handles non-human anatomy (ears/snout→head, tail→torso, paws→leg) the coordinate model can't. Used when it resolves ≥70% of vertices. (2) **ONNX model** (UNrigged meshes). (3) **geometric fallback**. The ONNX path also applies `Options::upAxis` by remapping the sampled point cloud to the model's +Y-up training frame before inference (and in the nearest-point scatter), so X/Z-up meshes aren't mislabelled. **Continual-training miner** (the "train further as we gather data" loop): `qtmesh segment --dump-training-data out.json` runs the rig-prior path and writes the normalised point cloud + EXACT per-vertex labels (schema `qtmesh-meshseg-training-v1`) — every rigged asset becomes one free, exactly-labelled sample. `scripts/export-meshseg-onnx.py --real-data ` MIXES those mined JSONs (with yaw/tilt/jitter aug) into the synthetic set and retrains; gains land on the MODEL path used for unrigged meshes (rigged meshes already use the exact rig-prior path in-app). `AutoRig::rigPriorPartLabels` is the shared extractor for the GUI fast-path and the miner, so the in-app selection and mined ground truth are bit-identical. - **Image-to-3D (TripoSR)** (`src/ImageTo3D/`, epic #764): single-image → 3D mesh generation via **TripoSR** (Tripo AI + Stability AI, **MIT code AND MIT weights**, HF `stabilityai/TripoSR`). The **fifth ONNX consumer** (after #404/#408/#409/#410); all files live in the `src/ImageTo3D/` feature folder. MIT code+weights is the deciding factor for redistribution (Homebrew/Snap/WinGet/Docker) — the bar UniRig #408 cleared and non-commercial SF3D failed. **`MeshGenPredictor`** (Ogre-free + unit-tested) runs two exported ONNX graphs — encoder `image[1,3,512,512]→scene_codes[1,3,40,64,64]` (triplane) and per-point decoder `scene_codes+points[1,P,3]→density[1,P,1],color[1,P,3]` — GENERATING query points per chunk (not the whole `res³` grid up front — that would OOM at 512) and extracting the surface with **`MarchingCubes`** (native Lorensen impl, public-domain tables, zero deps; TripoSR's `torchmcubes` is torch/GPU-only). Surface = MC on `density − threshold` at iso 0 (threshold 25.0, radius 0.87); our MC is inside-positive so `extract()` emits `v0,v2,v1` (flipped winding) to keep faces OUTWARD (else the mesh renders inside-out). **Model size tiers** (`MeshGenPredictor::Quality {Fp32,Int8}` → `triposr_encoder{,_int8}.onnx`): fp32 ~1.68 GB (best), int8 ~430 MB (slight quality loss); user-selectable, downloads on demand. (fp16 was dropped — TripoSR's attention has a hardcoded Cast-to-float32 the ONNX fp16 converters can't rewrite; int8 is smaller anyway.) **`MeshGenBuilder`** (the ONLY Ogre-touching piece) turns the arrays into an `Ogre::Mesh` (POSITION + accumulated per-vertex NORMAL + optional DIFFUSE `VET_COLOUR` with a lit vertex-color material; 16-/32-bit index by vertex count; validates index data first), **bakes -90°X + +90°Y** into positions+normals so the model stands upright and faces forward, uses a UNIQUE per-call node/mesh name, and returns the SceneNode for export. **Background removal:** `BackgroundRemover` (6th ONNX consumer) runs **U²-Net** (Apache-2.0, rembg's model) to isolate the subject: `[1,3,320,320]`→`[1,1,320,320]` saliency, then composites over **gray 128** (not white — white → a reconstructed wall) and crops/re-pads to the subject at 0.85 foreground ratio (TripoSR's `resize_foreground`). Model `ai_models/rembg/u2net.onnx` (`QTMESH_REMBG_MODEL_BASE_URL`/`ai/rembgModelBaseUrl`; guard `QTMESH_REMBG_NO_DOWNLOAD`); falls back to the raw image if unavailable. Everything `ENABLE_ONNX`-guarded; **no fallback** (generative), so a non-ONNX build / missing model returns a clear error (never crashes). Models under `ai_models/triposr/` download on first use (`ensureModelBlocking(q)`; `QTMESH_TRIPOSR_MODEL_BASE_URL`/`ai/triposrModelBaseUrl`; guard `QTMESH_TRIPOSR_NO_DOWNLOAD`), OR can be **pre-downloaded from the AI Settings modal's Download tab** (tier picker + progress bar). **Export is `scripts/export-triposr-onnx.py`** (offline, not shipped; `transformers==4.35.0`, `torchmcubes` stub, frozen ViT pos-encoding; emits the int8 variant unless `--no-quant` — see `docs/IMAGE_TO_3D_SPIKE_764.md`). Surfaced via **CLI `qtmesh generate3d [-o out.glb] [--resolution 16..1024] [--no-color] [--remove-bg] [--quality fp32|int8]`** (`CLIPipeline::cmdGenerate3d`), **MCP `generate_mesh_from_image`** (`MCPServer::toolGenerateMeshFromImage`, args `{image_path, output?, resolution?, vertex_color?, remove_bg?, quality?}`, heavy, ONNX-guarded schema), and the **Object Mode Tools → "AI: Image → 3D" inspector section** (`qml/PropertiesPanel.qml` → **`MeshGenController`**, a QML_SINGLETON that runs the whole pipeline on a WORKER THREAD — UI stays responsive — with a select-image→preview→generate flow, resolution + model-tier dropdowns, progress bar, and cancel; mesh construction is marshalled back to the main thread). Sentry breadcrumb `ai.assist.image_to_3d`. Verified end-to-end on macOS. **Models are HOSTED** on the `fernandotonon/QtMeshEditor-models` HF repo (`triposr/triposr_encoder.onnx` + `triposr_encoder_int8.onnx` + `triposr_decoder.onnx`, `rembg/u2net.onnx`) via `scripts/upload-triposr-models.sh` — first use downloads them; if ever absent, every surface reports a clean "not yet hosted" message (no crash). Design/spike note: `docs/IMAGE_TO_3D_SPIKE_764.md`; slices A #765 (spike) → B #766 predictor → C #767 mesh build → D #768 surfaces → E #769 tiers/pre-download/hosting/docs (all in PR #785). **Quality pass (post-#785, ON by default)**: after marching cubes the predictor runs (a) **`MeshRefine::taubinSmooth`** — Taubin λ|μ smoothing (volume-preserving, kills the res³-grid stair-stepping), (b) **`MeshRefine::isoProjectStep`** — one Newton step per vertex back onto the decoder's true iso-surface using forward-difference gradients from 4 extra decoder probes/vertex (recovers grid-quantized detail; both pure-data + unit-tested in `MeshRefine_test.cpp`), and (c) **`MeshGenBaker`** — xatlas auto-unwrap + UV-space triangle rasterization + per-texel decoder colour queries + chart-border dilation, producing UV0 + a real diffuse TEXTURE (default 1024²) instead of per-vertex colour — colour sharpness then scales with texture size, not vertex density (pure-data behind a `ColorSampler` callback; `MeshGenBaker_test.cpp`). `MeshGenBuilder` gained the textured path: saves the baked PNG (AppData/generated_textures/ or the export dir when given), registers the dir as a resource location, and binds a lit material with a named `diffuse_map` TUS. Bake failure falls back to vertex colours with `Result::warning` set (never fails the generation). **PBR stage (d, ON by default)**: `MeshGenBuilder::BuildOptions::generatePbrMaps` chains **#404 PBR map synthesis** onto the baked diffuse — normal + roughness PNGs written next to it (height skipped, no consumer) and bound into the material via the same recipe as the Material Editor's "Generate PBR maps from diffuse" button (canonical `normal_map`/`roughness` TUS + `wirePbrSlotsForFFP` + `RTShaderHelper::applyNormalMap` — without applyNormalMap the bind is invisible in the viewport — + recompile). This is what turns the flat diffuse-only result into a polished, surface-detailed one; fails soft to diffuse-only when the PBRify models are unavailable. The exported material references all three maps (FBX embeds them; the PNGs land next to the export). **Every stage is user-selectable**: GUI checkboxes in the AI section (Remove background / Smooth / Refine / Bake texture / PBR maps / Upscale 2×) feed an options QVariantMap into `MeshGenController::generateSelected`; CLI `--no-smooth --no-refine --no-bake-texture --no-pbr --texture-size N --upscale-texture`; MCP `smooth/refine/bake_texture/generate_pbr/texture_size/upscale_texture`. The GUI runs the upscale on the WORKER thread (model pre-ensured on the main thread) and the PBR synthesis on the main thread inside buildSceneNode (small models, Material-Editor precedent). **TripoSG backend** (`src/ImageTo3D/TripoSGPredictor.{h,cpp}`, the SEVENTH ONNX consumer): `MeshGenPredictor::Options::backend {TripoSR|TripoSG}` dispatches to **TripoSG** (VAST-AI, SIGGRAPH 2025, **MIT code + MIT weights**, geometry ≈ commercial Tripo 2.0) — a 1.5B rectified-flow DiT over an SDF VAE, run as FOUR exported graphs (`scripts/export-triposg-onnx.py`, offline dev tool; measured contract in `docs/TRIPOSG_EXPORT_NOTES.md`): DINOv2-224 image encoder (mean/std baked in; CFG uncond = zeros) → **C++ Euler flow loop** over the DiT step graph (σᵢ = 1−i/N, timestep = 1000·σ, update `x += (σᵢ−σᵢ₊₁)·v` — sign is OPPOSITE of stock diffusers FlowMatchEuler; CFG as two B=1 calls, guidance 7.0, steps knob default 25) → VAE latent kv-cache graph (run ONCE per generation) → per-point field decoder (already inside-positive, iso 0, bounds ±1.005) → the same native MarchingCubes + smooth/reproject polish. Geometry-only (no colour decoder): bake/PBR/upscale stages are TripoSR-only; background removal for TripoSG composites over WHITE (its reference pipeline) vs TripoSR's gray-128. fp32 DiT ships as `.onnx`+`.onnx.data` (>2 GB external weights) with an int8 single-file tier mapped from `Quality::Int8`. Models under `ai_models/triposg/` download on first use (`QTMESH_TRIPOSG_MODEL_BASE_URL`/`ai/triposgModelBaseUrl`; guard `QTMESH_TRIPOSG_NO_DOWNLOAD`); clean "not hosted yet" error until the export is run + hosted. Surfaced via CLI `--backend triposr|triposg --flow-steps N`, MCP `backend`/`flow_steps` args, and the GUI Backend dropdown (the step list gains a "Denoise (flow steps)" row via `Stage::Denoise`). Roadmap/audit: `docs/IMAGE_TO_3D_QUALITY.md`. **TripoSG post-integration updates (supersede the "geometry-only / int8 tier / white-bg / disabled texture checkboxes" claims above):** (1) **int8 tier DROPPED** — even per-channel-quantized, the 1.5B DiT degrades to blobs over the 25-step CFG flow loop (live-verified), and dynamic-int8 MatMuls are no faster than fp32 on ARM; all surfaces force fp32 (CLI prints a note; the GUI Model picker collapses to "fp32 (only option for TripoSG)" and locks; the `quality` param now only selects the TripoSR tier used for the colour bake). (2) **Colour** — TripoSG has no colour decoder, so `MeshGenPredictor::colorizeWithTripoSR` bakes colour by (a) projecting the actual input PHOTO onto the visible front (depth-buffer-gated front-most-surface test; camera looks toward +Z so nearest = max z; soft depth-band crossfade to the field) and (b) filling occluded/back texels from **TripoSR's image-conditioned colour field** (the TripoSG mesh mapped into TripoSR's native frame + per-axis affine-fit onto its occupied bounds). The front is photo-accurate; the back is inferred/approximate. Falls soft to a shared neutral **lit clay material** (`MeshGen/NeutralClay`) on any failure. Texture/PBR/upscale stages + their GUI checkboxes are ENABLED for TripoSG (route through the colour bake). (3) **AI texture (GUI, `ENABLE_STABLE_DIFFUSION`)** — a "Generate texture (AI, front photo + generated back)" checkbox runs the existing **multi-view depth-ControlNet bake** (`MaterialEditorQML::generateMeshTextureMultiView`, `MultiViewTextureBaker`) after the mesh builds, with the input photo PINNED as the front view (img2img is disabled on Metal, so the photo is injected as a filled view rather than an init image) and back/sides SD-generated; needs a loaded SD model. (4) **Orientation** — TripoSG output is already +Y-up (`Result::bakeTripoSROrientation=false` skips the TripoSR -90°X/+90°Y bake); its decoder field is negated at the sample site (exported graph lands OUTSIDE-positive → inverted winding otherwise). (5) **Memory/speed** — decoder chunk hard-capped at 8192 pts (cross-attention to 2048 kv tokens; TripoSR's 262144 chunk OOM-killed at ~90 GB); ONNX sessions staged (opened/released per stage, ~1 GB peak vs the >4 GB sum); the ~48 MB point decoder can run on the CoreML GPU via `QTMESH_TRIPOSG_COREML_DECODER=1` (default CPU — per-call kv re-upload made GPU slower); `--guidance` knob (CLI/MCP). Next speed win: hierarchical extraction (coarse grid → refine near surface). SF3D (non-commercial) and Hunyuan3D (EU-excluded) rejected for the texture upgrade; MV-Adapter (VAST-AI, Apache-2.0) is the tracked multi-view candidate. diff --git a/README.md b/README.md index 0ef1683b9..72884ad4b 100755 --- a/README.md +++ b/README.md @@ -186,6 +186,10 @@ qtmesh retopo model.fbx --max-angle 15 -o conservative.glb # tighter coplanarit qtmesh skin model.fbx -o skinned.glb # default 4 influences, falloff 4 qtmesh skin model.fbx --max-influences 8 --falloff 6 -o skinned.glb qtmesh skin model.fbx --skip-unweighted --merge -o filled.glb # fill missing weights only + +# Auto-generate the 52 ARKit blendshapes on a humanoid FACE mesh (for face capture) +qtmesh facerig head.glb -o rigged.glb # fit ARKit template + transfer 52 shapes +qtmesh facerig head.fbx -o rigged.glb --max-shapes 20 --json # cap shapes / machine-readable report ``` --- diff --git a/THIRD_PARTY_AI_MODELS.md b/THIRD_PARTY_AI_MODELS.md index 5384598a6..ab32db948 100644 --- a/THIRD_PARTY_AI_MODELS.md +++ b/THIRD_PARTY_AI_MODELS.md @@ -271,6 +271,34 @@ the binary). Attribution + licenses for the models and their training data: checkbox; the template library remains the default and the automatic fallback. Same CMU licensing basis as above. +## ICT-FaceKit — ARKit blendshape template for face auto-rig (epic #889) + +- **Asset (not a learned model):** the ICT-FaceKit generic neutral head + (`generic_neutral_mesh.obj`) + its per-expression meshes named after the + ARKit blendshapes (`jawOpen`, `mouthSmile_L`, `eyeBlink_L`, `browInnerUp_L`, + …), all sharing one topology (26,719 verts) so each shape = expr − neutral. +- **Source / license:** [USC-ICT/ICT-FaceKit](https://github.com/USC-ICT/ICT-FaceKit) + — **MIT** (Copyright 2020 USC Institute for Creative Technologies). The + standard/released model is MIT; a separate "full model" tier under a + USC-specific license is **REJECTED** (we ship only the MIT tier). MIT clears + the permissive-redistribution bar, so the template + shapes are hostable on + the `fernandotonon/QtMeshEditor-models` HF repo (Slice B) — packed by + `scripts/export-arkit-template.py` into `facerig/arkit_template.bin` and + uploaded by `scripts/upload-facerig-template.sh`; it downloads on first use. +- **How it is used:** the template is the *source* for **deformation transfer** + (Sumner & Popović 2004) — QtMeshEditor fits it to the user's neutral head via + native non-rigid ICP (Amberg 2007), then transfers each of the 52 ARKit + expressions onto the user's topology, attaching them as `Ogre::Pose` morph + targets so face performance capture (#869) works on the mesh. **No ML model, + no ONNX** — it is a deterministic geometry algorithm (sparse linear solve), + implemented natively in `src/FaceRig/` (Slices C/D/E). Verified end-to-end on + a decimated, different-topology face: mean 0.008% / max 0.61% NRICP fit and + 51 attached shapes. Surfaced via `qtmesh facerig`, MCP `add_arkit_blendshapes`, + and the Inspector "Add ARKit Blendshapes" button. See `docs/FACE_RIG.md`. +- **Rejected alternatives:** Wrap3D (commercial, used by the reference impl for + NRICP — we implement NRICP natively instead), FLAME-based 3DMMs + (research-only), any generative expression model on non-commercial data. + All of the above clear QtMeshEditor's permissive-redistribution bar (MIT app, distributed via Homebrew / WinGet / Snap / Docker). GPL/CC-BY-NC/unlicensed models are deliberately excluded (e.g. RigNet was rejected for #408 — GPL code + diff --git a/docs/FACE_RIG.md b/docs/FACE_RIG.md new file mode 100644 index 000000000..481a8a841 --- /dev/null +++ b/docs/FACE_RIG.md @@ -0,0 +1,85 @@ +# Face auto-rig: ARKit blendshapes on any humanoid face + +Epic [#889](https://github.com/fernandotonon/QtMeshEditor/issues/889). Given an +unrigged neutral **face** mesh, QtMeshEditor generates the 52 **ARKit** +blendshapes (`jawOpen`, `mouthSmileLeft`, `eyeBlinkLeft`, `browInnerUp`, …) and +attaches them as morph targets, so the mesh can be driven by face performance +capture ([#869](https://github.com/fernandotonon/QtMeshEditor/issues/869), +`qtmesh mocap --face`) — no manual sculpting of blend shapes. + +It is **deterministic geometry**, not an ML model: a non-rigid fit of a +permissively-licensed template face onto yours, followed by deformation +transfer of each expression. No ONNX, no GPU, no network beyond a one-time +template download. + +## Using it + +### GUI +Select a face mesh → Inspector → **Vertex Morph Animation** (Edit Mode) → +**"✨ Add ARKit Blendshapes (AI)"**. The fit runs on a worker thread (the button +shows *Downloading… / Fitting…*); when it finishes the 52 shapes appear in the +Shapes list and the whole batch is a single undo step. The +[#869](https://github.com/fernandotonon/QtMeshEditor/issues/869) Performance +Capture panel then drives them. + +### CLI +```bash +qtmesh facerig neutral_head.glb -o rigged.glb +qtmesh facerig head.fbx -o rigged.glb --max-shapes 20 # cap the shape count +qtmesh facerig head.glb -o rigged.glb --max-residual 5 # stricter humanoid gate +qtmesh facerig head.glb -o rigged.glb --json # machine-readable report +``` + +### MCP +`add_arkit_blendshapes` — `{max_shapes?, max_residual_pct?, output_path?}`, +operates on the selected entity; re-exports when `output_path` is given. + +## How it works + +``` +ArkitTemplate (ICT-FaceKit neutral + 52 expression deltas, one topology) + │ + ▼ NonRigidICP (Amberg 2007 optimal-step) src/FaceRig/NonRigidICP + correspondence X — template verts fitted onto the USER surface + │ + ▼ DeformationTransfer (Sumner & Popović 2004) src/FaceRig/DeformationTransfer + per-template-vertex delta per shape, on the user identity + │ + ▼ resample template topology → the real user vertices src/FaceRig/FaceRigger + 52 × per-user-vertex deltas + │ + ▼ Ogre::Pose + VAT_POSE morph targets, named per FaceCap::kBlendshapeNames +``` + +- **`FaceRigger` / `FaceRigAttach`** (`src/FaceRig/`) orchestrate the pipeline; + the pure-data core is Ogre-free and headless-unit-tested. +- The linear solves use a self-contained sparse CG (`SparseSolve`) — **no Eigen, + no external solver, zero new dependencies** (the house rule for these + features, same as skinning #402 and auto-rig #407). +- The template (`facerig/arkit_template.bin`, ICT-FaceKit MIT — see + `THIRD_PARTY_AI_MODELS.md`) downloads on first use to + `/ai_models/facerig/`. Overrides: `QTMESH_FACERIG_MODEL_BASE_URL` / + `QSettings ai/facerigModelBaseUrl`; offline guard `QTMESH_FACERIG_NO_DOWNLOAD`. + +## Quality & limits + +- **Humanoid faces only.** The fit residual is a gate: a non-face mesh fits + poorly and is **rejected** (`--max-residual`, default 8% of the mesh + diagonal), rather than emitting garbage shapes. This mirrors the + AutoRig/Pinocchio precedent. +- **Measured** (decimated, different-topology ICT head, 15 755 verts): NRICP + fit **mean 0.008% / max 0.61%** of the diagonal; **51 shapes** attached; + jawOpen drops the lower face while the forehead stays still; mouthSmile / + eyeBlink / browInnerUp localise to their regions. +- **Orientation:** the mesh should be roughly upright, +Y up, facing the + template's orientation. A wildly rotated head may fit poorly. +- **glTF export** carries the morph-target geometry on the primitive; per-target + *names* in glTF `extras.targetNames` are a follow-up (the in-editor targets + and the mocap hand-off use the correct names regardless). + +## Related + +- `THIRD_PARTY_AI_MODELS.md` — ICT-FaceKit licensing verdict. +- `docs/FACE_RIG_SPIKE.md` — the offline feasibility spike + the C/D contract. +- Epic [#889](https://github.com/fernandotonon/QtMeshEditor/issues/889); + slices B–G (#891–#895 + the polish slice). diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 18707bfcb..68e7a7d06 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -862,6 +862,13 @@ void CLIPipeline::printUsage() " Per-vertex weight diff vs a reference-skinned copy of the same asset\n" " (e.g. Mixamo) — vertices matched by position, bones by name. See\n" " docs/SKINNING_QUALITY.md for the comparison protocol.\n" + " facerig -o [--max-shapes N] [--max-residual PCT] [--json]\n" + " Auto-generate the 52 ARKit blendshapes on a humanoid\n" + " FACE mesh: fit the ARKit template (non-rigid ICP) and\n" + " transfer each expression (Sumner-Popovic) onto the\n" + " mesh, attaching them as named morph targets. A poor\n" + " fit (non-face mesh) is rejected. The bundled template\n" + " downloads on first use. Feeds `qtmesh mocap --face`.\n" " morph --list [--json] List morph targets / blend shapes on a mesh. (Set/add/delete\n" " land in follow-up slices once authoring is in place.)\n" " nodeanim --list [--json] List node-animation clips on a scene (props, doors, machinery,\n" @@ -9548,6 +9555,10 @@ int CLIPipeline::cmdFaceRig(int argc, char* argv[]) err() << "Error: export failed." << Qt::endl; return 1; } + // Sidecar with the ordered ARKit names (Assimp's glTF exporter drops + // targetNames), so `qtmesh mocap --face` / re-import can rebind by index. + FaceRig::writeArkitSidecar(QFileInfo(outputPath).absoluteFilePath(), + rep.shapeNames); if (jsonOutput) { QJsonObject j; diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp index 7387df9d0..4644249e5 100644 --- a/src/FaceRig/FaceRigAttach.cpp +++ b/src/FaceRig/FaceRigAttach.cpp @@ -9,6 +9,11 @@ #include #include +#include +#include +#include +#include + #include #include @@ -129,6 +134,7 @@ void attachShapes(Ogre::Entity* entity, const FaceRigGeometry& geo, AddMorphTargetCommand cmd(entity, shape.name, slices); cmd.redo(); report.shapesAttached++; + report.shapeNames.push_back(shape.name); } report.ok = report.shapesAttached > 0; if (!report.ok) @@ -184,4 +190,26 @@ AttachReport attachFaceRigWithBundledTemplate(Ogre::Entity* entity, return attachFaceRig(entity, tmpl, opts); } +bool writeArkitSidecar(const QString& meshPath, + const std::vector& shapeNames) +{ + if (meshPath.isEmpty() || shapeNames.empty()) return false; + QJsonArray names; + for (const QString& n : shapeNames) names.append(n); + QJsonObject root; + root["schema"] = QStringLiteral("qtmesh-arkit-blendshapes-v1"); + root["count"] = int(shapeNames.size()); + root["names"] = names; // ordered, matches the mesh's morph targets + + // .arkit.json alongside the exported mesh — recovers the ARKit names + // that Assimp 6.0's glTF exporter drops (it doesn't emit targetNames even + // though we set aiAnimMesh::mName). Face capture (#869) and re-import can + // read this to rebind the mocap-52 vocabulary by index. + QFile f(meshPath + QStringLiteral(".arkit.json")); + if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) return false; + f.write(QJsonDocument(root).toJson(QJsonDocument::Indented)); + f.close(); + return true; +} + } // namespace FaceRig diff --git a/src/FaceRig/FaceRigAttach.h b/src/FaceRig/FaceRigAttach.h index 36113de22..d9ccd9319 100644 --- a/src/FaceRig/FaceRigAttach.h +++ b/src/FaceRig/FaceRigAttach.h @@ -30,6 +30,7 @@ struct AttachReport { double fitMeanResidualPct = 0.0; double fitMaxResidualPct = 0.0; QString templateFallback; // set when the template had to be downloaded/etc. + std::vector shapeNames; // attached shape names, in order }; // One geometry owner (shared vertex pool, or a per-submesh vertex data) with @@ -77,6 +78,12 @@ AttachReport attachFaceRig(Ogre::Entity* entity, AttachReport attachFaceRigWithBundledTemplate(Ogre::Entity* entity, const FaceRigOptions& opts = {}); +// Write a `.arkit.json` sidecar with the ordered ARKit shape names, +// so downstream tools recover the names even though Assimp 6.0's glTF exporter +// drops mesh.extras.targetNames. Returns false on write failure. +bool writeArkitSidecar(const QString& meshPath, + const std::vector& shapeNames); + } // namespace FaceRig #endif // FACERIGATTACH_H diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 70ecefc7d..56703ea1a 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -2348,6 +2348,8 @@ QJsonObject MCPServer::toolAddArkitBlendshapes(const QJsonObject &args) return makeErrorResult( QStringLiteral("Error: blendshapes attached but export to '%1' " "failed (code %2)").arg(outputPath).arg(rc)); + // Sidecar with the ordered ARKit names (Assimp drops targetNames). + FaceRig::writeArkitSidecar(outputPath, rep.shapeNames); } } catch (const Ogre::Exception& e) { return makeErrorResult(QStringLiteral("Ogre error: %1") From f87968f5e132240da03501ccc87f8ce79518634e Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 19:43:46 -0300 Subject: [PATCH 08/38] fix(#889): head isolation + attach-crash fix for face auto-rig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two field-reported problems on full-body character meshes: 1) Wrong fit — the ARKit template (a FACE) was fitted against the WHOLE body, so "mouth" shapes landed on an arm. Fix: isolate the head region before fitting. FaceRigAttach::extractGeometry now computes a per-vertex headMask via the rig-prior (AutoRig::rigPriorPartLabels, exact on skinned meshes — handles the fox's snout/ears and Mixamo proportions) or MeshSegmenter's geometric fallback; buildFaceRig fits ONLY the head sub-mesh and scatters deltas back to the head vertices (body stays at zero). Only isolates when the head is a real minority region (50..75% of verts) — a bare-face crop still fits whole. Verified on Hip Hop Dancing (fox): jawOpen now moves 546 head verts at the top of the mesh (height frac 0.89) vs smearing across the torso (0.50) before. 2) GUI crash during attach — the batch adds poses + a VAT_POSE clip to a LIVE entity and re-initialises it per shape; the render loop's _updateAnimation could run against half-rebuilt pose buffers mid-batch and crash (skeletal + pose combined). Fix: FaceRigController disables every enabled animation state for the attach batch and restores them after. CLI (no render loop) was already stable; this closes the GUI-only crash. Known limitation (follow-up): on meshes whose head proportions differ sharply from the ICT template (e.g. Rumba), NRICP can converge to a low-residual but mis-oriented correspondence, so some shapes attach sparsely or not at all — landmark/PCA pre-align is the next step. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/FaceRig/FaceRigAttach.cpp | 46 +++++++++++++++++- src/FaceRig/FaceRigAttach.h | 6 +++ src/FaceRig/FaceRigger.cpp | 91 ++++++++++++++++++++++++++++++----- src/FaceRig/FaceRigger.h | 21 ++++++-- src/FaceRigController.cpp | 63 ++++++++++++++++++++++-- 5 files changed, 208 insertions(+), 19 deletions(-) diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp index 4644249e5..dfe5e7ca9 100644 --- a/src/FaceRig/FaceRigAttach.cpp +++ b/src/FaceRig/FaceRigAttach.cpp @@ -2,6 +2,8 @@ #include "ArkitTemplate.h" +#include "../AutoRig.h" +#include "../MeshSegmenter.h" #include "../commands/MorphCommands.h" #include @@ -104,6 +106,47 @@ FaceRigGeometry extractGeometry(Ogre::Entity* entity) if (!sm || sm->useSharedVertices) continue; addOwner(static_cast(si + 1), sm->vertexData, sm->indexData); } + + // ── HEAD ISOLATION (the fix for full-body characters) ──────────────────── + // The ARKit template is a FACE. Fitting it against a whole dancing body + // smears the face over the body (mouth shapes end up on an arm). So we + // isolate the head region and only fit / deform those vertices. Preference: + // 1) rig-prior — if the mesh is SKINNED, label each vertex by the body + // region of the bone it's most weighted to (EXACT; handles the fox's + // snout/ears and Mixamo's exaggerated proportions the coordinate model + // can't). This is AutoRig::rigPriorPartLabels, ordered to match our + // combined gather. + // 2) geometric fallback — MeshSegmenter's spatial head/torso/limb split. + // If neither finds a plausible head (e.g. the mesh really IS just a face), + // leave headMask empty and fit the whole thing (previous behaviour). + const int nv = int(geo.userV.size() / 3); + const int headPart = int(MeshSegmenter::Part::Head); + std::vector labels; + if (mesh->hasSkeleton()) { + int resolved = 0; + labels = AutoRig::rigPriorPartLabels(entity, nv, &resolved); + // Require the rig prior to resolve a decent share; else fall through. + if (resolved < nv / 2) labels.clear(); + } + if (labels.empty() && geo.userF.size() >= 3) { + std::vector idx(geo.userF.begin(), geo.userF.end()); + MeshSegmenter::Result seg = MeshSegmenter::segmentGeometric( + geo.userV.data(), nv, idx.data(), int(idx.size())); + if (seg.ok) labels = seg.vertexLabels; + } + if (int(labels.size()) == nv) { + int headCount = 0; + geo.headMask.assign(size_t(nv), 0); + for (int v = 0; v < nv; ++v) + if (labels[size_t(v)] == headPart) { geo.headMask[size_t(v)] = 1; ++headCount; } + // Only isolate when the head is a real, minority region of the mesh — + // i.e. this looks like a full body, not a bare face. A head that IS + // most of the mesh means it's already a face crop; fit it whole. + if (headCount >= 50 && headCount < nv * 3 / 4) + geo.headVertexCount = headCount; + else + geo.headMask.clear(); // treat as a face crop + } return geo; } @@ -156,7 +199,8 @@ AttachReport attachFaceRig(Ogre::Entity* entity, return rep; } - const FaceRigResult res = buildFaceRig(geo.userV, geo.userF, tmpl, opts); + const FaceRigResult res = buildFaceRig(geo.userV, geo.userF, tmpl, opts, + geo.headMask); rep.userVertexCount = res.userVertexCount; rep.fitMeanResidualPct = res.fitMeanResidualPct; rep.fitMaxResidualPct = res.fitMaxResidualPct; diff --git a/src/FaceRig/FaceRigAttach.h b/src/FaceRig/FaceRigAttach.h index d9ccd9319..3afc920d6 100644 --- a/src/FaceRig/FaceRigAttach.h +++ b/src/FaceRig/FaceRigAttach.h @@ -49,6 +49,12 @@ struct FaceRigGeometry { std::vector userV; // combined positions (Nu*3) std::vector userF; // combined triangle indices std::vector owners; + // Per-combined-vertex head flag (size Nu). Empty = no head isolation (the + // whole mesh is treated as the face). When populated, only head vertices + // participate in the fit + receive blendshape deltas — so a full-body + // character rigs correctly instead of smearing the face over the body. + std::vector headMask; + int headVertexCount = 0; bool valid() const { return userV.size() >= 9 && userF.size() >= 3; } }; diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index c698c2870..9315d397e 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -125,7 +125,9 @@ double bboxDiag(const std::vector& v) FaceRigResult buildFaceRig(const std::vector& userV, const std::vector& userF, const ArkitTemplate& tmpl, - const FaceRigOptions& opts) + const FaceRigOptions& opts, + const std::vector& headMask, + const FaceRigProgressFn& progress) { FaceRigResult r; if (userV.size() < 9 || userF.size() < 3) { @@ -136,16 +138,76 @@ FaceRigResult buildFaceRig(const std::vector& userV, r.error = "ARKit template not loaded"; return r; } - r.userVertexCount = int(userV.size() / 3); + const int nuFull = int(userV.size() / 3); + r.userVertexCount = nuFull; - // 1) NRICP: template neutral → user neutral = correspondence X. - const NricpResult fit = FaceRig::fit(tmpl.neutral(), tmpl.faces(), userV, userF); + // Head isolation: if a mask is supplied, build a head-only sub-mesh and fit + // THAT (so the face template lands on the face, not the whole body). We + // keep a fit→full-mesh vertex index map so the resampled deltas scatter + // back to the right original vertices; non-head vertices get zero delta. + // Without a mask, fit the whole mesh (a bare-face crop). + std::vector subV; + std::vector subF; + std::vector subToFull; // fit vertex index → full-mesh index + const bool isolate = int(headMask.size()) == nuFull; + if (isolate) { + std::vector fullToSub(size_t(nuFull), -1); + for (int v = 0; v < nuFull; ++v) { + if (!headMask[size_t(v)]) continue; + fullToSub[size_t(v)] = int(subToFull.size()); + subToFull.push_back(v); + subV.insert(subV.end(), + {userV[size_t(v)*3], userV[size_t(v)*3+1], userV[size_t(v)*3+2]}); + } + // keep faces whose 3 verts are all head; remap to sub indices + for (size_t f = 0; f + 2 < userF.size(); f += 3) { + const int a = userF[f], b = userF[f+1], c = userF[f+2]; + if (a < 0 || b < 0 || c < 0) continue; + const int sa = fullToSub[size_t(a)], sb = fullToSub[size_t(b)], + sc = fullToSub[size_t(c)]; + if (sa >= 0 && sb >= 0 && sc >= 0) + subF.insert(subF.end(), {sa, sb, sc}); + } + if (subV.size() < 9 || subF.size() < 3) { + // head region had no usable surface — fall back to whole-mesh fit + subV.clear(); subF.clear(); subToFull.clear(); + } + } + const bool useSub = !subToFull.empty(); + const std::vector& fitV = useSub ? subV : userV; + const std::vector& fitF = useSub ? subF : userF; + const int nu = int(fitV.size() / 3); + + // Progress model: the NRICP fit anneals over N stiffness levels (each a + // step under "Fitting…"), then one step per transferred shape. Total = + // fitLevels + shapeCount so the bar advances through BOTH phases. + NricpOptions fitOpts; + const int fitLevels = int(fitOpts.stiffness.size()); + const int shapeTotal = opts.maxShapes > 0 + ? std::min(opts.maxShapes, tmpl.shapeCount()) + : tmpl.shapeCount(); + const int total = fitLevels + shapeTotal; + bool cancelled = false; + auto tick = [&](int done, const char* phase) -> bool { + return progress ? progress(done, total, phase) : true; + }; + + // 1) NRICP: template neutral → user neutral = correspondence X. Report each + // annealing level so the (long) fit phase visibly advances. + const NricpResult fit = FaceRig::fit( + tmpl.neutral(), tmpl.faces(), fitV, fitF, fitOpts, + [&](int level, int /*levelCount*/) -> bool { + if (!tick(level, "Fitting face template…")) { cancelled = true; return false; } + return true; + }); + if (cancelled) { r.error = "cancelled"; return r; } if (!fit.ok || fit.diag <= 0.0) { r.error = "non-rigid fit failed"; return r; } r.fitMeanResidualPct = 100.0 * fit.meanResidual / fit.diag; r.fitMaxResidualPct = 100.0 * fit.maxResidual / fit.diag; + if (!tick(fitLevels, "Transferring shapes…")) { r.error = "cancelled"; return r; } // humanoid-only gate: a bad fit means this isn't a face — refuse. A NRICP // fit that couldn't converge onto the surface reports non-finite or huge @@ -180,15 +242,16 @@ FaceRigResult buildFaceRig(const std::vector& userV, return r; } - // 3) resample map: user vertex → nearest correspondence vertex (built once). + // 3) resample map: fit vertex → nearest correspondence vertex (built once). PointGrid grid; grid.build(fitted); - const int nu = r.userVertexCount; std::vector userToTmpl(size_t(nu), -1); for (int i = 0; i < nu; ++i) - userToTmpl[size_t(i)] = grid.nearest(&userV[size_t(i)*3]); + userToTmpl[size_t(i)] = grid.nearest(&fitV[size_t(i)*3]); - const double diag = bboxDiag(userV); + // noise floor scaled by the FIT region diagonal (a head is smaller than a + // whole body, so scaling on the full-body diag would swallow real motion). + const double diag = bboxDiag(fitV); const double eps = opts.deltaEpsPct / 100.0 * diag; const auto& shapes = tmpl.shapes(); @@ -197,6 +260,7 @@ FaceRigResult buildFaceRig(const std::vector& userV, : int(shapes.size()); for (int s = 0; s < maxShapes; ++s) { + if (!tick(fitLevels + s, "Transferring shapes…")) { r.error = "cancelled"; return r; } // per-TEMPLATE-vertex delta on the user identity const std::vector tmplDelta = dt.transfer(shapes[size_t(s)].deltas); if (int(tmplDelta.size() / 3) != tmpl.vertexCount()) { @@ -206,7 +270,9 @@ FaceRigResult buildFaceRig(const std::vector& userV, FaceRigShape out; out.name = shapes[size_t(s)].name; - out.userDeltas.assign(size_t(nu) * 3, 0.0f); + // Deltas are always full-mesh sized; head isolation writes only the + // head vertices (via subToFull), leaving the body at zero. + out.userDeltas.assign(size_t(nuFull) * 3, 0.0f); for (int i = 0; i < nu; ++i) { const int t = userToTmpl[size_t(i)]; if (t < 0) continue; @@ -216,9 +282,10 @@ FaceRigResult buildFaceRig(const std::vector& userV, double(dvec[1])*dvec[1] + double(dvec[2])*dvec[2]); if (mag < eps) continue; // noise floor → keep sparse - out.userDeltas[size_t(i)*3] = dvec[0]; - out.userDeltas[size_t(i)*3+1] = dvec[1]; - out.userDeltas[size_t(i)*3+2] = dvec[2]; + const int dst = useSub ? subToFull[size_t(i)] : i; + out.userDeltas[size_t(dst)*3] = dvec[0]; + out.userDeltas[size_t(dst)*3+1] = dvec[1]; + out.userDeltas[size_t(dst)*3+2] = dvec[2]; out.nonZeroVerts++; out.maxDisp = std::max(out.maxDisp, float(mag)); } diff --git a/src/FaceRig/FaceRigger.h b/src/FaceRig/FaceRigger.h index de24aa14e..3e468a8cd 100644 --- a/src/FaceRig/FaceRigger.h +++ b/src/FaceRig/FaceRigger.h @@ -25,6 +25,7 @@ #include +#include #include #include @@ -61,13 +62,27 @@ struct FaceRigOptions { int maxShapes = 0; }; +// Progress callback: (done, total, phase). `phase` is a short static label +// ("Fitting…", "Transferring shapes…"). Return false to request cancellation — +// buildFaceRig then returns ok=false with error "cancelled". May be called from +// a worker thread, so the callee must marshal any UI work itself. +using FaceRigProgressFn = + std::function; + // userV/userF: the user neutral mesh (Nu*3 verts + Fu*3 tris). tmpl: a loaded -// ArkitTemplate. Returns 52 per-user-vertex delta sets (or fewer if the -// template has fewer), or ok=false + error on failure / a poor fit. +// ArkitTemplate. Returns per-user-vertex delta sets (size Nu*3 each), or +// ok=false + error on failure / a poor fit. +// `headMask` (optional, size Nu): when non-empty, ONLY vertices flagged 1 are +// fitted against the face template and receive blendshape deltas — the fix for +// full-body characters (the face template must not smear over the body). Empty +// = fit the whole mesh (a bare-face crop). +// `progress` (optional) reports fit + per-shape steps and can cancel. FaceRigResult buildFaceRig(const std::vector& userV, const std::vector& userF, const ArkitTemplate& tmpl, - const FaceRigOptions& opts = {}); + const FaceRigOptions& opts = {}, + const std::vector& headMask = {}, + const FaceRigProgressFn& progress = {}); } // namespace FaceRig diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 8aaaf47a4..e75454737 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -126,25 +126,51 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu m_busy = true; emit busyChanged(); + m_progress = 0; + m_progressTotal = 0; + emit progressChanged(); setStatus(QStringLiteral("Fitting face template…")); + m_cancel = std::make_shared(false); + auto cancel = m_cancel; const std::string entName = entity->getName(); QPointer self(this); // WORKER thread: the heavy Ogre-free fit + transfer over all 52 shapes. - std::thread([self, geo, tmpl, opts, entName]() { + std::thread([self, geo, tmpl, opts, entName, cancel]() { + // Progress callback: marshal the counters to the main thread for the + // progress bar; return false to stop the worker when cancel is set. + auto progress = [self, cancel](int done, int total, + const char* phase) -> bool { + if (cancel->load()) return false; + const QString ph = QString::fromUtf8(phase); + QMetaObject::invokeMethod(qApp, [self, done, total, ph]() { + if (!self) return; + self->m_progress = done; + self->m_progressTotal = total; + emit self->progressChanged(); + self->setStatus(ph); + }, Qt::QueuedConnection); + return true; + }; auto result = std::make_shared( - FaceRig::buildFaceRig(geo->userV, geo->userF, *tmpl, opts)); + FaceRig::buildFaceRig(geo->userV, geo->userF, *tmpl, opts, + geo->headMask, progress)); // MAIN thread: attach (touches Ogre + the undo stack). QMetaObject::invokeMethod(qApp, [self, geo, result, entName]() { if (!self) return; self->m_busy = false; emit self->busyChanged(); + self->m_progress = 0; + self->m_progressTotal = 0; + emit self->progressChanged(); self->setStatus(QString()); if (!result->ok) { - emit self->error(QString::fromStdString(result->error)); + emit self->error(result->error == "cancelled" + ? QStringLiteral("Face-rig cancelled.") + : QString::fromStdString(result->error)); return; } @@ -167,6 +193,22 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu rep.fitMeanResidualPct = result->fitMeanResidualPct; rep.fitMaxResidualPct = result->fitMaxResidualPct; + // The attach adds poses + a VAT_POSE clip to a LIVE entity and + // re-initialises it per shape. If the entity is mid-skeletal- + // animation, the render loop's _updateAnimation can run against the + // half-rebuilt pose buffers between our steps and crash. Disable + // every enabled animation state for the batch, then restore them — + // so the frame loop leaves the entity static while we mutate it. + std::vector reEnable; + if (auto* ass = entity->getAllAnimationStates()) { + for (const auto& kv : ass->getAnimationStates()) { + if (kv.second && kv.second->getEnabled()) { + reEnable.push_back(QString::fromStdString(kv.first)); + kv.second->setEnabled(false); + } + } + } + auto* undo = UndoManager::getSingleton(); auto* stack = undo ? undo->stack() : nullptr; if (stack) stack->beginMacro(QStringLiteral("Add ARKit Blendshapes")); @@ -190,6 +232,16 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu rep.shapesAttached++; } if (stack) stack->endMacro(); + + // Restore the animation states we disabled (refreshAvailable... in + // the attach may have recreated the state set, so re-resolve). + if (auto* ass = entity->getAllAnimationStates()) { + for (const QString& n : reEnable) { + const std::string sn = n.toStdString(); + if (ass->hasAnimationState(sn)) + ass->getAnimationState(sn)->setEnabled(true); + } + } rep.ok = rep.shapesAttached > 0; if (!rep.ok) { @@ -214,3 +266,8 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu return true; } + +void FaceRigController::cancel() +{ + if (m_cancel) m_cancel->store(true); +} From 3475c1b6fd1190620d4754124c3e219e926d5a3f Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 15 Jul 2026 02:37:06 -0400 Subject: [PATCH 09/38] fix(#889): batch face-rig attach so the GUI doesn't freeze on the last step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ARKit attach adds 51 poses + VAT_POSE clips to a LIVE entity and re-initialised the entity after EACH shape (entity->_initialise(true)). On a big / multi-submesh mesh (e.g. Rumba Dancing, 11 submeshes) that is O(shapes × mesh) of main-thread work and froze the UI at the final progress step (56/57 = the attach phase). Fix: AddMorphTargetCommand gains setDeferInit(). FaceRigController builds all per-shape commands, defers the re-init on every one EXCEPT the last, and lets that last command do a single _initialise — so redo (and Ctrl+Shift+Z redo) rebuilds the pose/animation buffers exactly once at the end instead of 51 times. undo is unchanged (per-command removal is cheap). The CLI/MCP attachShapes path is untouched (no render loop, process exits). Verified: Rumba attaches all 51 shapes at 0.086% residual with every moved vertex on the head (height fraction 0.73–1.00, mean 0.82) — no body smear. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/FaceRig/FaceRigger.cpp | 2 ++ src/FaceRigController.cpp | 19 ++++++++++++++----- src/commands/MorphCommands.cpp | 9 ++++++--- src/commands/MorphCommands.h | 9 +++++++++ 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index 9315d397e..60294d217 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index e75454737..279620bee 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -209,9 +209,12 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu } } - auto* undo = UndoManager::getSingleton(); - auto* stack = undo ? undo->stack() : nullptr; - if (stack) stack->beginMacro(QStringLiteral("Add ARKit Blendshapes")); + // Build the per-shape commands first so we know which is LAST — only + // the last re-initialises the entity (deferInit=false on it), the + // rest defer. This makes redo (Ctrl+Shift+Z) rebuild the pose buffers + // exactly once at the end too, not just the initial attach; a + // per-shape re-init would freeze the UI on a multi-submesh mesh. + std::vector cmds; for (const FaceRig::FaceRigShape& shape : result->shapes) { std::vector slices; for (const FaceRig::GeometryOwner& o : geo->owners) { @@ -228,9 +231,15 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu if (!slice.offsets.empty()) slices.push_back(std::move(slice)); } if (slices.empty()) continue; - undo->push(new AddMorphTargetCommand(entity, shape.name, slices)); - rep.shapesAttached++; + cmds.push_back(new AddMorphTargetCommand(entity, shape.name, slices)); } + for (size_t ci = 0; ci + 1 < cmds.size(); ++ci) + cmds[ci]->setDeferInit(true); // all but the last defer re-init + + auto* undo = UndoManager::getSingleton(); + auto* stack = undo ? undo->stack() : nullptr; + if (stack) stack->beginMacro(QStringLiteral("Add ARKit Blendshapes")); + for (auto* cmd : cmds) { undo->push(cmd); rep.shapesAttached++; } if (stack) stack->endMacro(); // Restore the animation states we disabled (refreshAvailable... in diff --git a/src/commands/MorphCommands.cpp b/src/commands/MorphCommands.cpp index 4ab6853ca..e31ac73d1 100644 --- a/src/commands/MorphCommands.cpp +++ b/src/commands/MorphCommands.cpp @@ -105,7 +105,8 @@ void removePosesByName(Ogre::Mesh* mesh, const QString& name, Ogre::Entity* enti void buildPosesFromSlices(Ogre::Mesh* mesh, const QString& name, const std::vector& slices, - Ogre::Entity* entity) + Ogre::Entity* entity, + bool deferInit = false) { if (!mesh || slices.empty()) return; const std::string sn = name.toStdString(); @@ -141,7 +142,7 @@ void buildPosesFromSlices(Ogre::Mesh* mesh, kf->addPoseReference(poseIndices[i], 1.0f); } - if (entity) { + if (entity && !deferInit) { // Adding poses / a VAT_POSE animation to an already-loaded mesh means // the entity's pose (software + hardware) vertex-animation buffers were // never allocated — the importer sets poses up BEFORE the entity is @@ -150,6 +151,8 @@ void buildPosesFromSlices(Ogre::Mesh* mesh, // animation against null buffers and crashes (skinned meshes especially, // where skeletal + pose animation combine). Mirrors the AutoRig path, // which likewise re-initialises after mutating a live entity's mesh. + // deferInit lets a batch attach (face auto-rig) skip this per-shape and + // re-initialise ONCE after the last shape — O(shapes×mesh) → O(mesh). entity->_initialise(true); entity->refreshAvailableAnimationState(); } @@ -261,7 +264,7 @@ void AddMorphTargetCommand::redo() if (!mEntity) return; Ogre::MeshPtr mesh = mEntity->getMesh(); if (!mesh) return; - buildPosesFromSlices(mesh.get(), mName, mSlices, mEntity); + buildPosesFromSlices(mesh.get(), mName, mSlices, mEntity, mDeferInit); SentryReporter::addBreadcrumb("scene.anim.morph", QStringLiteral("add target '%1'").arg(mName)); } diff --git a/src/commands/MorphCommands.h b/src/commands/MorphCommands.h index b7ca7d213..13f08b7e0 100644 --- a/src/commands/MorphCommands.h +++ b/src/commands/MorphCommands.h @@ -51,10 +51,19 @@ class AddMorphTargetCommand : public QUndoCommand void undo() override; void redo() override; + // Batch attach optimisation (face auto-rig #889 attaches 51 shapes at once): + // re-initialising the live entity after EVERY shape is O(shapes × mesh) and + // freezes the UI on big/multi-submesh meshes. When deferInit is set, redo() + // builds the poses but skips entity->_initialise; the caller must call it + // ONCE after the last command (Ogre rebuilds the pose buffers in one pass). + // undo() always re-inits (single removal — cheap). + void setDeferInit(bool defer) { mDeferInit = defer; } + private: Ogre::Entity* mEntity = nullptr; QString mName; std::vector mSlices; + bool mDeferInit = false; }; // DeleteMorphTargetCommand: remove all same-named poses + the matching From 577617abc064fbdfb28b33bceeca8574a774fc68 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 15 Jul 2026 14:10:03 -0400 Subject: [PATCH 10/38] feat(#889): facial-landmark anchoring for the face auto-rig (fixes wrong shape placement) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NRICP fit had no anatomical anchoring, so it converged to a low-residual but MIS-ORIENTED drape — ARKit "mouth" shapes landed on the wrong features (and worse on full-body / oddly-proportioned humanoids like the fox and Rumba). This adds automatic facial-landmark detection to anchor the fit onto the real eyes/nose/mouth, per the user's suggestion. Pipeline (all ENABLE_ONNX-guarded, graceful unanchored fallback): - FaceLandmarkDetector (src/FaceRig): runs MediaPipe FaceMesh V2 (face_landmarks.onnx, Apache-2.0 — the SAME model the mocap face-capture uses, reused not re-converted) on a centred 256 crop → 478 image-space landmarks. We render the head ourselves so the upstream face detector is skipped. Model downloads on first use to ai_models/facerig/. - MeshDepthRenderer::renderShadedView: renders the entity with REAL materials + a head-on light (photo-like face for the detector), with an optional focus AABB so a full-body character is framed TIGHTLY on the head — without this the face is a few pixels and MediaPipe can't detect it (7 landmarks → 166 once framed on the head). - FaceRigLandmarks: render + detect + back-project (Möller–Trumbore ray from the render camera to the mesh surface) on BOTH the template (temp entity, cached) and the user head, pair by MediaPipe index → NRICP landmark anchors. - NonRigidICP: landmark constraint rows (weight rides alpha so orientation/scale lock first, then relax). FaceRigger/FaceRigAttach/FaceRigController thread the anchors through; detection runs on the MAIN thread (Ogre render), the fit on the worker. headSubmesh() gives the detector the head region to frame/raycast. Also lands the previously-uncommitted GUI responsiveness work: worker-thread progress bar + Cancel (FaceRigController progress/progressTotal/cancel) and the batch-attach deferred re-init. Verified via CLI on both problem meshes: Hip Hop (fox) 151 anchors, Rumba (human) 166 anchors; both fit at <0.15% mean / <11% max with all moved verts on the head. Full FaceRig suite green (18 tests). Docs + THIRD_PARTY entry + upload-script hook added; the landmark model hosts under facerig/ (falls back to the unanchored fit until hosted). Co-Authored-By: Claude Opus 4.8 (1M context) --- THIRD_PARTY_AI_MODELS.md | 11 + docs/FACE_RIG.md | 17 +- qml/PropertiesPanel.qml | 51 +++++ scripts/upload-facerig-template.sh | 8 + src/CMakeLists.txt | 2 + src/FaceRig/FaceLandmarkDetector.cpp | 272 +++++++++++++++++++++++++ src/FaceRig/FaceLandmarkDetector.h | 69 +++++++ src/FaceRig/FaceRigAttach.cpp | 43 +++- src/FaceRig/FaceRigAttach.h | 6 + src/FaceRig/FaceRigLandmarks.cpp | 287 +++++++++++++++++++++++++++ src/FaceRig/FaceRigLandmarks.h | 57 ++++++ src/FaceRig/FaceRigger.cpp | 2 + src/FaceRig/FaceRigger.h | 7 + src/FaceRig/NonRigidICP.cpp | 40 +++- src/FaceRig/NonRigidICP.h | 25 ++- src/FaceRigController.cpp | 14 +- src/FaceRigController.h | 16 ++ src/MeshDepthRenderer.cpp | 111 +++++++++++ src/MeshDepthRenderer.h | 17 ++ 19 files changed, 1046 insertions(+), 9 deletions(-) create mode 100644 src/FaceRig/FaceLandmarkDetector.cpp create mode 100644 src/FaceRig/FaceLandmarkDetector.h create mode 100644 src/FaceRig/FaceRigLandmarks.cpp create mode 100644 src/FaceRig/FaceRigLandmarks.h diff --git a/THIRD_PARTY_AI_MODELS.md b/THIRD_PARTY_AI_MODELS.md index ab32db948..45610c6f2 100644 --- a/THIRD_PARTY_AI_MODELS.md +++ b/THIRD_PARTY_AI_MODELS.md @@ -295,9 +295,20 @@ the binary). Attribution + licenses for the models and their training data: a decimated, different-topology face: mean 0.008% / max 0.61% NRICP fit and 51 attached shapes. Surfaced via `qtmesh facerig`, MCP `add_arkit_blendshapes`, and the Inspector "Add ARKit Blendshapes" button. See `docs/FACE_RIG.md`. +- **Facial-landmark anchoring (landmark pass):** the NRICP fit is anchored to + real face features by **MediaPipe Face Mesh V2** (`face_landmarks.onnx`, + **Apache-2.0** — the same model the mocap face-capture uses, #869). We render + the head front-on, detect the 478 landmarks, back-project them to the mesh + surface, and pin the matching template vertices — so the template lands on the + actual eyes/nose/mouth instead of a low-residual-but-mis-oriented drape. Hosted + under `facerig/face_landmarks.onnx` (a copy of the mocap graph); downloads on + first use; `ENABLE_ONNX`-guarded with a graceful unanchored-fit fallback. - **Rejected alternatives:** Wrap3D (commercial, used by the reference impl for NRICP — we implement NRICP natively instead), FLAME-based 3DMMs (research-only), any generative expression model on non-commercial data. + Landmark detectors trained on 300W / WFLW / InsightFace (dlib, PIPNet, + 2d106det) were rejected — their weights carry research-only / non-commercial + terms; MediaPipe FaceMesh (Apache-2.0) is the clean choice. All of the above clear QtMeshEditor's permissive-redistribution bar (MIT app, distributed via Homebrew / WinGet / Snap / Docker). GPL/CC-BY-NC/unlicensed diff --git a/docs/FACE_RIG.md b/docs/FACE_RIG.md index 481a8a841..71484397f 100644 --- a/docs/FACE_RIG.md +++ b/docs/FACE_RIG.md @@ -39,7 +39,16 @@ operates on the selected entity; re-exports when `output_path` is given. ``` ArkitTemplate (ICT-FaceKit neutral + 52 expression deltas, one topology) │ - ▼ NonRigidICP (Amberg 2007 optimal-step) src/FaceRig/NonRigidICP + ▼ HEAD ISOLATION — rig-prior (skinned) or MeshSegmenter picks the head + │ region, so a full-body character fits the face template only on the + │ head (not smeared across the body). + │ + ▼ FACE-LANDMARK ANCHORS — render the head front-on, detect 478 MediaPipe + │ landmarks (FaceLandmarkDetector), back-project to the surface on BOTH + │ template + user, pair by index → anchor constraints. This locks the + │ fit onto the real eyes/nose/mouth (the fix for wrong shape placement). + │ + ▼ NonRigidICP (Amberg 2007 optimal-step, landmark-anchored) src/FaceRig/NonRigidICP correspondence X — template verts fitted onto the USER surface │ ▼ DeformationTransfer (Sumner & Popović 2004) src/FaceRig/DeformationTransfer @@ -60,6 +69,12 @@ ArkitTemplate (ICT-FaceKit neutral + 52 expression deltas, one topology) `THIRD_PARTY_AI_MODELS.md`) downloads on first use to `/ai_models/facerig/`. Overrides: `QTMESH_FACERIG_MODEL_BASE_URL` / `QSettings ai/facerigModelBaseUrl`; offline guard `QTMESH_FACERIG_NO_DOWNLOAD`. +- **Landmark anchoring** (`FaceLandmarkDetector` + `FaceRigLandmarks`, + `ENABLE_ONNX`): renders the head and runs MediaPipe FaceMesh V2 + (`facerig/face_landmarks.onnx`, Apache-2.0) to anchor the fit to real face + features. When ONNX is off, the model is missing, or no face is detected, the + fit runs **unanchored** (the previous behaviour) — the feature degrades + gracefully, it never blocks a rig. ## Quality & limits diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index dd300bf0b..3df615f03 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -9424,6 +9424,57 @@ Rectangle { : "Select a face mesh first." } } + // Progress bar + Cancel, shown only while the worker runs. + RowLayout { + width: parent.width + visible: FaceRigController.busy + spacing: 6 + // Track + determinate fill (indeterminate look when total==0). + Rectangle { + Layout.fillWidth: true + Layout.preferredHeight: 6 + radius: 3 + color: PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + Rectangle { + height: parent.height + radius: 3 + color: PropertiesPanelController.highlightColor + width: FaceRigController.progressTotal > 0 + ? parent.width * FaceRigController.progress + / FaceRigController.progressTotal + : parent.width * 0.15 + } + } + Text { + visible: FaceRigController.progressTotal > 0 + text: FaceRigController.progress + "/" + FaceRigController.progressTotal + color: PropertiesPanelController.textColor + font.pixelSize: 9 + } + Rectangle { + Layout.preferredWidth: 48 + Layout.preferredHeight: 18 + radius: 3 + color: cancelMa.containsMouse + ? Qt.lighter(PropertiesPanelController.headerColor, 1.3) + : PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: "Cancel" + color: PropertiesPanelController.textColor + font.pixelSize: 9 + } + MouseArea { + id: cancelMa + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + onClicked: FaceRigController.cancel() + } + } + } Connections { target: FaceRigController function onError(message) { diff --git a/scripts/upload-facerig-template.sh b/scripts/upload-facerig-template.sh index 6f32acd49..779642910 100755 --- a/scripts/upload-facerig-template.sh +++ b/scripts/upload-facerig-template.sh @@ -37,4 +37,12 @@ upload() { # upload "$OUT/arkit_template.bin" "facerig/arkit_template.bin" [ -n "$ICT_LICENSE" ] && upload "$ICT_LICENSE" "facerig/ICT-FaceKit-LICENSE.txt" +# Facial-landmark model (MediaPipe FaceMesh V2, Apache-2.0) that anchors the +# NRICP fit to real face features (#889 landmark pass). It is the SAME graph the +# mocap face-capture uses (mocap/face/face_landmarks.onnx); we host a copy under +# facerig/ so the face-rig feature is self-contained. Point FACE_LMK at the +# converted onnx (e.g. .mocap_work/out/face/face_landmarks.onnx). +FACE_LMK="${FACE_LMK:-}" +[ -n "$FACE_LMK" ] && upload "$FACE_LMK" "facerig/face_landmarks.onnx" + echo "done. Verify: curl -sI https://huggingface.co/$REPO/resolve/main/facerig/arkit_template.bin | head -1" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 140603c9e..afb9271af 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -133,6 +133,8 @@ FaceRig/SparseSolve.cpp FaceRig/DeformationTransfer.cpp FaceRig/FaceRigger.cpp FaceRig/FaceRigAttach.cpp +FaceRig/FaceLandmarkDetector.cpp +FaceRig/FaceRigLandmarks.cpp MotionInbetween.cpp MotionLibrary.cpp MotionGenerator.cpp diff --git a/src/FaceRig/FaceLandmarkDetector.cpp b/src/FaceRig/FaceLandmarkDetector.cpp new file mode 100644 index 000000000..ed5072d1f --- /dev/null +++ b/src/FaceRig/FaceLandmarkDetector.cpp @@ -0,0 +1,272 @@ +#include "FaceLandmarkDetector.h" + +#include "ArkitTemplate.h" // reuse its model dir + base-url convention +#include "../ModelDownloader.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef ENABLE_ONNX +#include +#endif + +namespace FaceRig { + +namespace { +constexpr const char* kModelFile = "face_landmarks.onnx"; +constexpr int kInputSize = 256; // MediaPipe FaceMesh V2 crop +constexpr int kMinLandmarkFloats = 468 * 3; // 468/478 landmarks × xyz +} // namespace + +struct FaceLandmarkDetector::Impl { +#ifdef ENABLE_ONNX + std::unique_ptr env; + std::unique_ptr session; + std::vector inputNames; + std::vector outputNames; + std::vector inputNamesC; + std::vector outputNamesC; +#endif + bool loaded = false; +}; + +FaceLandmarkDetector::FaceLandmarkDetector() : d(std::make_unique()) {} +FaceLandmarkDetector::~FaceLandmarkDetector() = default; + +QString FaceLandmarkDetector::modelPath() +{ + const QString dataPath = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + return QDir(dataPath).filePath(QStringLiteral("ai_models/facerig/") + + QString::fromLatin1(kModelFile)); +} + +bool FaceLandmarkDetector::present() { return QFileInfo::exists(modelPath()); } + +bool FaceLandmarkDetector::backendAvailable() +{ +#ifdef ENABLE_ONNX + return true; +#else + return false; +#endif +} + +QString FaceLandmarkDetector::ensureModelBlocking() +{ + const QString dest = modelPath(); + if (QFileInfo::exists(dest)) + return dest; + if (!qEnvironmentVariableIsEmpty("QTMESH_FACERIG_NO_DOWNLOAD")) + return {}; + + // Same base URL as the ARKit template (they live together in facerig/). + QString base; + { + QSettings s; + base = s.value(QStringLiteral("ai/facerigModelBaseUrl")).toString(); + if (base.isEmpty()) { + const QByteArray env = qgetenv("QTMESH_FACERIG_MODEL_BASE_URL"); + base = env.isEmpty() + ? QStringLiteral("https://huggingface.co/fernandotonon/" + "QtMeshEditor-models/resolve/main/facerig/") + : QString::fromUtf8(env); + } + } + if (base.isEmpty()) + return {}; + if (!base.endsWith('/')) + base += '/'; + + auto* dl = ModelDownloader::instance(); + if (!dl) + return {}; + + QDir().mkpath(QFileInfo(dest).absolutePath()); + const QString url = base + QString::fromLatin1(kModelFile); + const QString label = QStringLiteral("face landmark model"); + + QEventLoop loop; + bool ok = false, timedOut = false; + auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = true; loop.quit(); } + }); + auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = false; loop.quit(); } + }); + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&timeout, &QTimer::timeout, &loop, + [&]() { timedOut = true; loop.quit(); }); + timeout.start(120000); // 2 min — the model is small (~3 MB) + + dl->startDownload(url, dest, label); + loop.exec(); + + QObject::disconnect(onDone); + QObject::disconnect(onErr); + if (timedOut && dl) + dl->cancelDownload(); + + return (ok && !timedOut && QFileInfo::exists(dest)) ? dest : QString(); +} + +bool FaceLandmarkDetector::isAvailable() const { return d && d->loaded; } + +#ifdef ENABLE_ONNX + +bool FaceLandmarkDetector::load(const QString& path) +{ + m_error.clear(); + d->loaded = false; + const QString p = path.isEmpty() ? modelPath() : path; + if (!QFileInfo::exists(p)) { + m_error = QStringLiteral("face landmark model not found at %1").arg(p); + return false; + } + try { + d->env = std::make_unique(ORT_LOGGING_LEVEL_WARNING, + "qtmesh_facelmk"); + Ort::SessionOptions so; + so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); +#if defined(__APPLE__) + try { + std::unordered_map coreml; + so.AppendExecutionProvider("CoreML", coreml); + } catch (...) { /* CPU fallback */ } +#endif +#ifdef _WIN32 + const std::wstring wp = p.toStdWString(); + d->session = std::make_unique(*d->env, wp.c_str(), so); +#else + const std::string sp = p.toStdString(); + d->session = std::make_unique(*d->env, sp.c_str(), so); +#endif + Ort::AllocatorWithDefaultOptions alloc; + const size_t ni = d->session->GetInputCount(); + for (size_t i = 0; i < ni; ++i) { + auto n = d->session->GetInputNameAllocated(i, alloc); + d->inputNames.emplace_back(n.get()); + } + const size_t no = d->session->GetOutputCount(); + for (size_t i = 0; i < no; ++i) { + auto n = d->session->GetOutputNameAllocated(i, alloc); + d->outputNames.emplace_back(n.get()); + } + for (auto& s : d->inputNames) d->inputNamesC.push_back(s.c_str()); + for (auto& s : d->outputNames) d->outputNamesC.push_back(s.c_str()); + d->loaded = !d->inputNames.empty() && !d->outputNames.empty(); + if (!d->loaded) + m_error = QStringLiteral("model has no inputs/outputs"); + return d->loaded; + } catch (const std::exception& e) { + m_error = QStringLiteral("failed to load face landmark model: %1") + .arg(QString::fromUtf8(e.what())); + d->loaded = false; + return false; + } +} + +LandmarkResult FaceLandmarkDetector::detect(const QImage& image) +{ + LandmarkResult r; + if (!isAvailable() || image.isNull()) return r; + + // Centre-square crop of the render, then resize to 256 — the head fills the + // frame (we rendered it), so the upstream face detector is unnecessary. We + // track the crop rect to map landmark px back into the ORIGINAL image space. + const int W = image.width(), H = image.height(); + const int side = std::min(W, H); + const int ox = (W - side) / 2, oy = (H - side) / 2; + QImage crop = image.copy(ox, oy, side, side) + .convertToFormat(QImage::Format_RGB888) + .scaled(kInputSize, kInputSize, Qt::IgnoreAspectRatio, + Qt::SmoothTransformation); + + // NHWC float [0,1] + std::vector input(size_t(kInputSize) * kInputSize * 3); + for (int y = 0; y < kInputSize; ++y) { + const uchar* line = crop.constScanLine(y); + for (int x = 0; x < kInputSize; ++x) { + const uchar* px = line + x * 3; + const size_t o = (size_t(y) * kInputSize + x) * 3; + input[o + 0] = px[0] / 255.0f; + input[o + 1] = px[1] / 255.0f; + input[o + 2] = px[2] / 255.0f; + } + } + + try { + Ort::MemoryInfo mem = Ort::MemoryInfo::CreateCpu( + OrtArenaAllocator, OrtMemTypeDefault); + const std::array shape{1, kInputSize, kInputSize, 3}; + Ort::Value in = Ort::Value::CreateTensor( + mem, input.data(), input.size(), shape.data(), shape.size()); + auto outs = d->session->Run( + Ort::RunOptions{nullptr}, d->inputNamesC.data(), &in, 1, + d->outputNamesC.data(), d->outputNamesC.size()); + + const float* rawLandmarks = nullptr; + size_t landmarkFloats = 0; + float presenceLogit = 0.f; + bool presenceFound = false; + for (auto& o : outs) { + const auto info = o.GetTensorTypeAndShapeInfo(); + const size_t count = info.GetElementCount(); + if (count >= size_t(kMinLandmarkFloats)) { + rawLandmarks = o.GetTensorData(); + landmarkFloats = count; + } else if (count == 1 && !presenceFound) { + presenceLogit = o.GetTensorData()[0]; + presenceFound = true; + } + } + if (!rawLandmarks) { + m_error = QStringLiteral("unexpected face landmark outputs"); + return r; + } + r.confidence = presenceFound + ? 1.f / (1.f + std::exp(-std::clamp(presenceLogit, -50.f, 50.f))) + : 1.f; + + const int n = int(landmarkFloats / 3); + r.points.reserve(size_t(n)); + const float scale = float(side) / float(kInputSize); + for (int i = 0; i < n; ++i) { + // model outputs 256-space px (x,y) + relative z; map back to the + // ORIGINAL image pixel space via the crop offset + scale. + const float lx = rawLandmarks[i * 3 + 0] * scale + ox; + const float ly = rawLandmarks[i * 3 + 1] * scale + oy; + const float lz = rawLandmarks[i * 3 + 2] * scale; + r.points.push_back({lx, ly, lz}); + } + r.ok = true; + return r; + } catch (const std::exception& e) { + m_error = QStringLiteral("face landmark inference failed: %1") + .arg(QString::fromUtf8(e.what())); + return r; + } +} + +#else // !ENABLE_ONNX + +bool FaceLandmarkDetector::load(const QString&) +{ + m_error = QStringLiteral("built without ONNX (ENABLE_ONNX off)"); + return false; +} +LandmarkResult FaceLandmarkDetector::detect(const QImage&) { return {}; } + +#endif // ENABLE_ONNX + +} // namespace FaceRig diff --git a/src/FaceRig/FaceLandmarkDetector.h b/src/FaceRig/FaceLandmarkDetector.h new file mode 100644 index 000000000..20023b4cd --- /dev/null +++ b/src/FaceRig/FaceLandmarkDetector.h @@ -0,0 +1,69 @@ +#ifndef FACELANDMARKDETECTOR_H +#define FACELANDMARKDETECTOR_H + +// Facial-landmark detection for the face auto-rig (#889). Runs the MediaPipe +// Face Mesh V2 landmark graph (face_landmarks.onnx — Apache-2.0, the same model +// the mocap face-capture uses, #869) on a rendered head image to get the 478 +// canonical face landmarks. Those anchor the non-rigid ICP fit so the ARKit +// template lands on the ACTUAL face features (eyes/nose/mouth) instead of a +// low-residual-but-mis-oriented drape — the fix for wrong shape placement. +// +// We render the head ourselves (centred, front-facing, evenly lit), so the +// upstream face DETECTOR is unnecessary — we feed a plain centred 256×256 crop +// straight to the landmark graph. Ogre-free + ENABLE_ONNX-guarded; the model +// downloads on first use to ai_models/facerig/ (same dir/base-url as the ARKit +// template). Without ONNX or the model, isAvailable() stays false and the +// caller falls back to the landmark-free fit. + +#include +#include + +#include +#include +#include + +namespace FaceRig { + +struct LandmarkResult { + // 478 landmarks as (x,y,z), x/y in the INPUT IMAGE's pixel space (0..W/H), + // z a relative depth (unused for back-projection). Empty when no face. + std::vector> points; + float confidence = 0.0f; // presence sigmoid, 0 = no face + bool ok = false; +}; + +class FaceLandmarkDetector { +public: + FaceLandmarkDetector(); + ~FaceLandmarkDetector(); + + // ---- model management (mirrors ArkitTemplate) ---- + static QString modelPath(); // AppData/ai_models/facerig/face_landmarks.onnx + static bool present(); + // Blocking first-use download; returns the path or empty (offline guard + // QTMESH_FACERIG_NO_DOWNLOAD; base URL QTMESH_FACERIG_MODEL_BASE_URL / + // QSettings ai/facerigModelBaseUrl — same as the ARKit template). + static QString ensureModelBlocking(); + + // True only when built with ENABLE_ONNX. + static bool backendAvailable(); + + // Load the ONNX session from `path` (default: modelPath()). Returns false + // if ONNX is off / the file is missing / the session can't be created. + bool load(const QString& path = {}); + bool isAvailable() const; + QString lastError() const { return m_error; } + + // Detect landmarks on `image` (any format; the head should fill the frame, + // centred and front-facing). Points come back in `image`'s pixel space. + LandmarkResult detect(const QImage& image); + +private: + struct Impl; + std::unique_ptr d; + QString m_error; +}; + +} // namespace FaceRig + +#endif // FACELANDMARKDETECTOR_H diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp index dfe5e7ca9..c6b223b57 100644 --- a/src/FaceRig/FaceRigAttach.cpp +++ b/src/FaceRig/FaceRigAttach.cpp @@ -1,6 +1,7 @@ #include "FaceRigAttach.h" #include "ArkitTemplate.h" +#include "FaceRigLandmarks.h" #include "../AutoRig.h" #include "../MeshSegmenter.h" @@ -150,6 +151,37 @@ FaceRigGeometry extractGeometry(Ogre::Entity* entity) return geo; } +void headSubmesh(const FaceRigGeometry& geo, + std::vector& outV, std::vector& outF) +{ + outV.clear(); + outF.clear(); + const int nv = int(geo.userV.size() / 3); + if (int(geo.headMask.size()) != nv) { + // no head isolation — the whole mesh IS the face crop. + outV = geo.userV; + outF = geo.userF; + return; + } + std::vector fullToSub(size_t(nv), -1); + for (int v = 0; v < nv; ++v) { + if (!geo.headMask[size_t(v)]) continue; + fullToSub[size_t(v)] = int(outV.size() / 3); + outV.insert(outV.end(), {geo.userV[size_t(v)*3], + geo.userV[size_t(v)*3+1], + geo.userV[size_t(v)*3+2]}); + } + for (size_t f = 0; f + 2 < geo.userF.size(); f += 3) { + const int a = geo.userF[f], b = geo.userF[f+1], c = geo.userF[f+2]; + if (a < 0 || b < 0 || c < 0) continue; + const int sa = fullToSub[size_t(a)], sb = fullToSub[size_t(b)], + sc = fullToSub[size_t(c)]; + if (sa >= 0 && sb >= 0 && sc >= 0) + outF.insert(outF.end(), {sa, sb, sc}); + } + if (outV.size() < 9 || outF.size() < 3) { outV = geo.userV; outF = geo.userF; } +} + void attachShapes(Ogre::Entity* entity, const FaceRigGeometry& geo, const FaceRigResult& result, AttachReport& report) { @@ -199,8 +231,17 @@ AttachReport attachFaceRig(Ogre::Entity* entity, return rep; } + // Facial-landmark anchors (render + detect on template AND user, pair by + // MediaPipe index). Frame/raycast the HEAD sub-mesh so the face fills the + // detector's frame. Empty when ONNX/model/face-detection unavailable — the + // fit then runs unanchored (previous behaviour). + std::vector headV; std::vector headF; + headSubmesh(geo, headV, headF); + const std::vector anchors = + buildLandmarkAnchors(entity, headV, headF, tmpl); + const FaceRigResult res = buildFaceRig(geo.userV, geo.userF, tmpl, opts, - geo.headMask); + geo.headMask, anchors); rep.userVertexCount = res.userVertexCount; rep.fitMeanResidualPct = res.fitMeanResidualPct; rep.fitMaxResidualPct = res.fitMaxResidualPct; diff --git a/src/FaceRig/FaceRigAttach.h b/src/FaceRig/FaceRigAttach.h index 3afc920d6..8cb645558 100644 --- a/src/FaceRig/FaceRigAttach.h +++ b/src/FaceRig/FaceRigAttach.h @@ -62,6 +62,12 @@ struct FaceRigGeometry { // buffers — milliseconds). Empty/invalid result on failure. FaceRigGeometry extractGeometry(Ogre::Entity* entity); +// Extract the HEAD sub-mesh (local V + remapped F) from a geometry + its +// headMask — the region the landmark detector should frame/raycast. Returns the +// full mesh when there's no head mask (a bare-face crop). Pure data. +void headSubmesh(const FaceRigGeometry& geo, + std::vector& outV, std::vector& outF); + // MAIN-thread: attach a computed FaceRigResult's shapes to `entity` as // Ogre::Pose + VAT_POSE morph targets (via AddMorphTargetCommand), splitting // the combined per-vertex deltas back onto each owner's handle. Fills the diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp new file mode 100644 index 000000000..dec9d6d28 --- /dev/null +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -0,0 +1,287 @@ +#include "FaceRigLandmarks.h" + +#include "ArkitTemplate.h" +#include "FaceLandmarkDetector.h" + +#include "../Manager.h" +#include "../MeshDepthRenderer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace FaceRig { + +namespace { + +constexpr int kRenderSize = 512; // render + detect resolution + +// Möller–Trumbore ray/triangle intersection. Returns t (ray param) > 0 on hit. +bool rayTri(const Ogre::Vector3& o, const Ogre::Vector3& d, + const Ogre::Vector3& a, const Ogre::Vector3& b, + const Ogre::Vector3& c, float& tOut) +{ + const Ogre::Vector3 e1 = b - a, e2 = c - a; + const Ogre::Vector3 p = d.crossProduct(e2); + const float det = e1.dotProduct(p); + if (std::fabs(det) < 1e-9f) return false; + const float inv = 1.0f / det; + const Ogre::Vector3 tv = o - a; + const float u = tv.dotProduct(p) * inv; + if (u < -1e-4f || u > 1.0001f) return false; + const Ogre::Vector3 q = tv.crossProduct(e1); + const float v = d.dotProduct(q) * inv; + if (v < -1e-4f || u + v > 1.0001f) return false; + const float t = e2.dotProduct(q) * inv; + if (t <= 1e-5f) return false; + tOut = t; + return true; +} + +} // namespace + +MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, + const std::vector& localV, + const std::vector& localF) +{ + MeshLandmarks out; + if (!entity || localV.size() < 9 || localF.size() < 3) return out; + + FaceLandmarkDetector det; + if (!det.load()) return out; // ONNX off / model missing → caller falls back + + // World transform for building the head focus box + later ray/tri tests. + Ogre::Node* node = entity->getParentNode(); + const Ogre::Matrix4 world = node ? node->_getFullTransform() + : Ogre::Matrix4::IDENTITY; + + // Head focus box (WORLD space) from the local head verts, so the render + // frames tightly on the FACE — a full-body character's face would be a few + // pixels if we framed the whole entity, and MediaPipe wouldn't detect it. + Ogre::AxisAlignedBox focus; + { const int fnv = int(localV.size() / 3); + for (int i = 0; i < fnv; ++i) + focus.merge(world * Ogre::Vector3(localV[size_t(i)*3], + localV[size_t(i)*3+1], + localV[size_t(i)*3+2])); } + + // 1) render the head front-on (materials intact, lit) for the detector. + QString err; + const MeshDepthRenderer::RenderResult rr = + MeshDepthRenderer::renderShadedView( + entity, kRenderSize, MeshDepthRenderer::front(), &err, + focus.isNull() ? nullptr : &focus); + if (rr.depth.isNull()) return out; +#ifndef NDEBUG + if (const char* dp = std::getenv("QTMESH_FACERIG_DUMP_RENDER")) + rr.depth.save(QString::fromUtf8(dp)); +#endif + + // 2) detect landmarks (image-pixel space). + const LandmarkResult lr = det.detect(rr.depth); + if (!lr.ok || lr.points.empty()) return out; + out.confidence = lr.confidence; + + // 3) build world-space head triangles (local verts × node world transform). + // The render framed the WORLD bounding box, so rays are in world space; + // we intersect world triangles and transform the hit back to LOCAL (the + // frame the fit uses). `world` was computed above for the focus box. + const Ogre::Matrix4 worldInv = world.inverse(); + const int nv = int(localV.size() / 3); + std::vector wv(size_t(nv), Ogre::Vector3::ZERO); + for (int i = 0; i < nv; ++i) + wv[size_t(i)] = world * Ogre::Vector3(localV[size_t(i)*3], + localV[size_t(i)*3+1], + localV[size_t(i)*3+2]); + + // Inverse of (proj * view) to unproject pixels into world rays. + const Ogre::Matrix4 vp = rr.projMatrix * rr.viewMatrix; + const Ogre::Matrix4 vpInv = vp.inverse(); + const float W = float(rr.depth.width()), H = float(rr.depth.height()); + + const int nl = int(lr.points.size()); + out.points.assign(size_t(nl), {0, 0, 0}); + out.valid.assign(size_t(nl), 0); + + for (int i = 0; i < nl; ++i) { + // pixel → NDC ([-1,1], y up). Landmarks are in image pixels (y down). + const float ndcX = (lr.points[size_t(i)][0] / W) * 2.0f - 1.0f; + const float ndcY = 1.0f - (lr.points[size_t(i)][1] / H) * 2.0f; + // unproject near (z=-1) and far (z=1) → world ray. + const Ogre::Vector3 nearW = vpInv * Ogre::Vector3(ndcX, ndcY, -1.0f); + const Ogre::Vector3 farW = vpInv * Ogre::Vector3(ndcX, ndcY, 1.0f); + const Ogre::Vector3 o = nearW; + Ogre::Vector3 dir = farW - nearW; + if (dir.isZeroLength()) continue; + dir.normalise(); + + // nearest triangle hit along the ray. + float bestT = std::numeric_limits::max(); + Ogre::Vector3 hit; + bool found = false; + for (size_t f = 0; f + 2 < localF.size(); f += 3) { + const int ia = localF[f], ib = localF[f+1], ic = localF[f+2]; + if (ia < 0 || ib < 0 || ic < 0 || ia >= nv || ib >= nv || ic >= nv) + continue; + float t; + if (rayTri(o, dir, wv[size_t(ia)], wv[size_t(ib)], wv[size_t(ic)], t) + && t < bestT) { + bestT = t; hit = o + dir * t; found = true; + } + } + if (!found) continue; + const Ogre::Vector3 local = worldInv * hit; // back to mesh-local + out.points[size_t(i)] = {local.x, local.y, local.z}; + out.valid[size_t(i)] = 1; + } + + // ok only if we anchored a useful number of landmarks. + int good = 0; + for (char v : out.valid) good += v ? 1 : 0; + out.ok = good >= 20; // need a meaningful anchor set + return out; +} + +namespace { + +// Build a throwaway Ogre entity for the template neutral so we can render + +// detect its landmarks. Cached by vertex/face count so repeated rigs in one +// session reuse it (the template is fixed). Returns nullptr on failure. +Ogre::Entity* templateEntity(const std::vector& v, + const std::vector& f) +{ + auto* mgr = Manager::getSingletonPtr(); + if (!mgr) return nullptr; + Ogre::SceneManager* sm = mgr->getSceneMgr(); + if (!sm) return nullptr; + + static Ogre::Entity* cached = nullptr; + static size_t cachedKey = 0; + const size_t key = v.size() * 1000003u + f.size(); + if (cached && cachedKey == key) return cached; + + const std::string meshName = "QtMeshFaceRigTemplateMesh"; + const std::string entName = "QtMeshFaceRigTemplateEnt"; + auto& mm = Ogre::MeshManager::getSingleton(); + if (sm->hasEntity(entName)) sm->destroyEntity(entName); + if (mm.resourceExists(meshName)) mm.remove(meshName); + + Ogre::MeshPtr mesh = mm.createManual( + meshName, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + auto* sub = mesh->createSubMesh(); + sub->useSharedVertices = false; + const int vc = int(v.size() / 3); + sub->vertexData = new Ogre::VertexData(); + sub->vertexData->vertexCount = size_t(vc); + auto* decl = sub->vertexData->vertexDeclaration; + decl->addElement(0, 0, Ogre::VET_FLOAT3, Ogre::VES_POSITION); + auto vbuf = Ogre::HardwareBufferManager::getSingleton().createVertexBuffer( + decl->getVertexSize(0), vc, Ogre::HardwareBuffer::HBU_STATIC_WRITE_ONLY); + vbuf->writeData(0, v.size() * sizeof(float), v.data()); + sub->vertexData->vertexBufferBinding->setBinding(0, vbuf); + + const bool use32 = vc > 65535; + auto ibuf = Ogre::HardwareBufferManager::getSingleton().createIndexBuffer( + use32 ? Ogre::HardwareIndexBuffer::IT_32BIT + : Ogre::HardwareIndexBuffer::IT_16BIT, + f.size(), Ogre::HardwareBuffer::HBU_STATIC_WRITE_ONLY); + if (use32) { + std::vector i32(f.begin(), f.end()); + ibuf->writeData(0, i32.size() * sizeof(uint32_t), i32.data()); + } else { + std::vector i16(f.size()); + for (size_t i = 0; i < f.size(); ++i) i16[i] = uint16_t(f[i]); + ibuf->writeData(0, i16.size() * sizeof(uint16_t), i16.data()); + } + sub->indexData->indexBuffer = ibuf; + sub->indexData->indexCount = f.size(); + + Ogre::Vector3 mn(1e30f, 1e30f, 1e30f), mx(-1e30f, -1e30f, -1e30f); + for (int i = 0; i < vc; ++i) { + Ogre::Vector3 p(v[size_t(i)*3], v[size_t(i)*3+1], v[size_t(i)*3+2]); + mn.makeFloor(p); mx.makeCeil(p); + } + mesh->_setBounds(Ogre::AxisAlignedBox(mn, mx)); + mesh->_setBoundingSphereRadius(0.5f * (mx - mn).length()); + mesh->load(); + + Ogre::Entity* ent = sm->createEntity(entName, meshName); + // Parent to a detached node so it's in the scene graph for world transform + // but hidden (renderShadedView hides all other entities anyway; this one is + // the render target when we pass it in). + Ogre::SceneNode* node = sm->getRootSceneNode()->createChildSceneNode(); + node->attachObject(ent); + node->setVisible(false); // only shown during its own render pass + + cached = ent; + cachedKey = key; + return ent; +} + +} // namespace + +std::vector buildLandmarkAnchors( + Ogre::Entity* userEntity, + const std::vector& userLocalV, + const std::vector& userLocalF, + const ArkitTemplate& tmpl) +{ + std::vector anchors; + if (!userEntity || !tmpl.valid()) return anchors; + if (!FaceLandmarkDetector::backendAvailable()) return anchors; + // Cheap pre-check: no model → skip the renders entirely. + { FaceLandmarkDetector probe; if (!probe.load()) return anchors; } + + // template side: build a temp entity, detect, map each landmark → nearest + // template vertex. + Ogre::Entity* tent = templateEntity(tmpl.neutral(), tmpl.faces()); + if (!tent) return anchors; + // render pass needs it visible; renderShadedView hides OTHERS, so show it. + if (auto* tn = tent->getParentSceneNode()) tn->setVisible(true); + const MeshLandmarks tlm = detectMeshLandmarks(tent, tmpl.neutral(), + std::vector(tmpl.faces())); + if (auto* tn = tent->getParentSceneNode()) tn->setVisible(false); + if (!tlm.ok) return anchors; + + // user side. + const MeshLandmarks ulm = detectMeshLandmarks(userEntity, userLocalV, userLocalF); + if (!ulm.ok) return anchors; + + const int nl = int(std::min(tlm.points.size(), ulm.points.size())); + const int tvc = tmpl.vertexCount(); + const auto& tn = tmpl.neutral(); + for (int i = 0; i < nl; ++i) { + if (!tlm.valid[size_t(i)] || !ulm.valid[size_t(i)]) continue; + // nearest template vertex to the template landmark point. + const auto& tp = tlm.points[size_t(i)]; + int best = -1; float bestD = std::numeric_limits::max(); + for (int vtx = 0; vtx < tvc; ++vtx) { + const float dx = tn[size_t(vtx)*3] - tp[0]; + const float dy = tn[size_t(vtx)*3+1] - tp[1]; + const float dz = tn[size_t(vtx)*3+2] - tp[2]; + const float d = dx*dx + dy*dy + dz*dz; + if (d < bestD) { bestD = d; best = vtx; } + } + if (best < 0) continue; + anchors.push_back({best, ulm.points[size_t(i)]}); + } +#ifndef NDEBUG + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] landmark anchors: tmpl.ok=%d user.ok=%d " + "anchors=%zu\n", tlm.ok, ulm.ok, anchors.size()); +#endif + return anchors; +} + +} // namespace FaceRig diff --git a/src/FaceRig/FaceRigLandmarks.h b/src/FaceRig/FaceRigLandmarks.h new file mode 100644 index 000000000..6d4cfc03f --- /dev/null +++ b/src/FaceRig/FaceRigLandmarks.h @@ -0,0 +1,57 @@ +#ifndef FACERIGLANDMARKS_H +#define FACERIGLANDMARKS_H + +// Landmark acquisition for the face auto-rig (#889): render a head mesh +// front-on, detect the MediaPipe 478 face landmarks (FaceLandmarkDetector), and +// back-project each 2D landmark through the render camera onto the mesh surface +// to get a 3D landmark ON THE MESH. Running this on BOTH the ARKit template and +// the user head, then pairing by MediaPipe index, yields the correspondences +// that anchor NRICP so the fit lands on the real face features. +// +// Ogre-touching (renders via MeshDepthRenderer::renderShadedView) — main thread. + +#include "NonRigidICP.h" // NricpLandmark + +#include +#include + +namespace Ogre { class Entity; } + +namespace FaceRig { + +class ArkitTemplate; + +struct MeshLandmarks { + // 3D landmark positions in MESH-LOCAL space (same frame as the geometry the + // fit uses), one per MediaPipe index; `valid[i]` is false when landmark i + // didn't hit the surface (skipped as a correspondence). + std::vector> points; + std::vector valid; + float confidence = 0.0f; + bool ok = false; +}; + +// Render `entity` front-on, detect face landmarks, back-project to the surface. +// `localV`/`localF` are the MESH-LOCAL vertices/indices the fit operates on +// (the head sub-mesh in local space) — the ray cast intersects these so the +// returned points live in the same frame as the fit. Returns ok=false when the +// detector is unavailable / no face is found. +MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, + const std::vector& localV, + const std::vector& localF); + +// Build NRICP landmark anchors (template vertex → user position) by detecting +// face landmarks on BOTH the ARKit template and the user head and pairing them +// by MediaPipe index. `userEntity` is rendered for the user side; `userLocalV/F` +// are the head sub-mesh (local frame) the fit uses, both for the user raycast +// and the frame the returned targets live in. Returns empty when the detector +// is unavailable / either face isn't found (caller then fits without anchors). +std::vector buildLandmarkAnchors( + Ogre::Entity* userEntity, + const std::vector& userLocalV, + const std::vector& userLocalF, + const ArkitTemplate& tmpl); + +} // namespace FaceRig + +#endif // FACERIGLANDMARKS_H diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index 60294d217..18b3a3bb9 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -129,6 +129,7 @@ FaceRigResult buildFaceRig(const std::vector& userV, const ArkitTemplate& tmpl, const FaceRigOptions& opts, const std::vector& headMask, + const std::vector& landmarks, const FaceRigProgressFn& progress) { FaceRigResult r; @@ -184,6 +185,7 @@ FaceRigResult buildFaceRig(const std::vector& userV, // step under "Fitting…"), then one step per transferred shape. Total = // fitLevels + shapeCount so the bar advances through BOTH phases. NricpOptions fitOpts; + fitOpts.landmarks = landmarks; // anchor the fit to detected face features const int fitLevels = int(fitOpts.stiffness.size()); const int shapeTotal = opts.maxShapes > 0 ? std::min(opts.maxShapes, tmpl.shapeCount()) diff --git a/src/FaceRig/FaceRigger.h b/src/FaceRig/FaceRigger.h index 3e468a8cd..7b21b5f43 100644 --- a/src/FaceRig/FaceRigger.h +++ b/src/FaceRig/FaceRigger.h @@ -23,6 +23,8 @@ // quality gate — a non-face mesh fits poorly and is rejected with a clear // reason, mirroring the AutoRig/Pinocchio precedent. +#include "NonRigidICP.h" // NricpLandmark + #include #include @@ -76,12 +78,17 @@ using FaceRigProgressFn = // fitted against the face template and receive blendshape deltas — the fix for // full-body characters (the face template must not smear over the body). Empty // = fit the whole mesh (a bare-face crop). +// `landmarks` (optional): facial-landmark correspondences pinning template +// vertices to USER positions (in the SAME frame as userV), computed externally +// by the Ogre landmark layer. When present they anchor the NRICP fit so the +// template lands on the real eyes/nose/mouth instead of a mis-oriented drape. // `progress` (optional) reports fit + per-shape steps and can cancel. FaceRigResult buildFaceRig(const std::vector& userV, const std::vector& userF, const ArkitTemplate& tmpl, const FaceRigOptions& opts = {}, const std::vector& headMask = {}, + const std::vector& landmarks = {}, const FaceRigProgressFn& progress = {}); } // namespace FaceRig diff --git a/src/FaceRig/NonRigidICP.cpp b/src/FaceRig/NonRigidICP.cpp index 2188b773c..2bad23508 100644 --- a/src/FaceRig/NonRigidICP.cpp +++ b/src/FaceRig/NonRigidICP.cpp @@ -182,7 +182,8 @@ void cgnr(const Sparse& A, const std::vector& b, NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, const std::vector& userV, const std::vector& userF, - const NricpOptions& opts) + const NricpOptions& opts, + const NricpProgressFn& progress) { NricpResult res; const int Nt = int(tmplV.size()/3); @@ -255,6 +256,8 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, // coord for every vertex), sharing the SAME sparse matrix. const int cols = 4*Nt; + const int levelCount = int(opts.stiffness.size()); + int levelIdx = 0; for (double alpha : opts.stiffness) { for (int iter = 0; iter < opts.itersPerLevel; ++iter) { // find closest surface point per current X_i @@ -267,9 +270,21 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, target[i] = closestPointTriangle(X[i], utri[cf][0], utri[cf][1], utri[cf][2]); } - // assemble A (rows = Nt data + E stiffness) once; rhs differs per axis + // Landmark anchors that reference a valid template vertex. Weight + // rides alpha so it dominates while the fit is still rigid (locking + // orientation/scale), then relaxes as alpha anneals down. + std::vector lms; + lms.reserve(opts.landmarks.size()); + for (const auto& lm : opts.landmarks) + if (lm.tmplVertex >= 0 && lm.tmplVertex < Nt) + lms.push_back(&lm); + const int L = int(lms.size()); + const double lw = opts.landmarkWeight * alpha; + + // assemble A (rows = Nt data + E stiffness + L landmark) once; + // rhs differs per axis std::vector> trip; - trip.reserve(size_t(Nt)*4 + size_t(E)*8); + trip.reserve(size_t(Nt)*4 + size_t(E)*8 + size_t(L)*4); // data rows: row i uses cols [4i..4i+3] with [vx,vy,vz,1] for (int i = 0; i < Nt; ++i) { trip.push_back({double(i), double(4*i+0), vhat[i][0]}); @@ -287,14 +302,25 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, trip.push_back({double(row), double(4*j+k), -alpha}); } } + // landmark rows: lw*[vx,vy,vz,1]·A_L = lw*target[axis] (rhs per axis) + for (int l = 0; l < L; ++l) { + const int i = lms[l]->tmplVertex; + const int row = Nt + E*4 + l; + trip.push_back({double(row), double(4*i+0), lw*vhat[i][0]}); + trip.push_back({double(row), double(4*i+1), lw*vhat[i][1]}); + trip.push_back({double(row), double(4*i+2), lw*vhat[i][2]}); + trip.push_back({double(row), double(4*i+3), lw}); + } Sparse A; - A.fromTriplets(Nt + E*4, cols, trip); + A.fromTriplets(Nt + E*4 + L, cols, trip); std::vector b(A.rows, 0.0); std::vector x(cols, 0.0); for (int axis = 0; axis < 3; ++axis) { for (int i = 0; i < Nt; ++i) b[i] = target[i][axis]; // stiffness rhs stays 0 + for (int l = 0; l < L; ++l) + b[Nt + E*4 + l] = lw * double(lms[l]->target[size_t(axis)]); // warm start x from the current affine estimate for this axis for (int i = 0; i < Nt; ++i) { // recover current A_i row for this axis from X_i & vhat_i is @@ -309,6 +335,12 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, } } } + ++levelIdx; + if (progress && !progress(levelIdx, levelCount)) { + // caller aborted — bail with the best fit so far (ok stays false + // below because the residual pass computes finiteCount honestly). + break; + } } // final residuals to the user surface. A vertex whose affine solve diff --git a/src/FaceRig/NonRigidICP.h b/src/FaceRig/NonRigidICP.h index dc63e0f25..a5cec93af 100644 --- a/src/FaceRig/NonRigidICP.h +++ b/src/FaceRig/NonRigidICP.h @@ -18,7 +18,9 @@ // point-to-triangle projection against the user mesh (KD-tree over triangle // centroids for the broad phase). Contract proven in docs/FACE_RIG_SPIKE.md. +#include #include +#include #include namespace FaceRig { @@ -35,21 +37,42 @@ struct NricpResult { bool ok = false; }; +// A landmark correspondence: template vertex `tmplVertex` should map onto the +// user-space position `target`. Feeding a handful of these (from facial-landmark +// detection) ANCHORS the fit so it can't converge to a low-residual but +// mis-oriented/mis-scaled drape — the fix for the ARKit template landing on the +// wrong face features. (#889) +struct NricpLandmark { + int tmplVertex = -1; + std::array target{0, 0, 0}; +}; + struct NricpOptions { // annealed stiffness weights (high = rigid, low = free); ~3 inner iters each std::vector stiffness = {50, 20, 8, 3, 1, 0.5}; int itersPerLevel = 3; int cgIters = 400; // CG cap per axis solve double cgTol = 1e-6; + // Optional landmark anchors (template vertex → user position). Added as + // high-weight data rows; the weight is strongest at the rigid (high-alpha) + // levels so orientation/scale lock first, then relaxes as the fit refines. + std::vector landmarks; + double landmarkWeight = 10.0; // base weight (× alpha at each level) }; +// Progress callback for the annealing loop: (level, levelCount). Return false +// to abort the fit early (fit() then returns the best-so-far with ok=false). +using NricpProgressFn = std::function; + // tmplV/tmplF: template neutral verts (Nt*3) + tris (Ft*3). // userV/userF: user neutral verts (Nu*3) + tris (Fu*3). // Both assumed roughly aligned in orientation (+Y up); the fit does a // centroid+bbox-scale rigid pre-align, then the non-rigid warp. +// `progress` (optional) fires once per completed stiffness level. NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, const std::vector& userV, const std::vector& userF, - const NricpOptions& opts = {}); + const NricpOptions& opts = {}, + const NricpProgressFn& progress = {}); } // namespace FaceRig diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 279620bee..6d6be39b0 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -2,6 +2,7 @@ #include "FaceRig/ArkitTemplate.h" #include "FaceRig/FaceRigAttach.h" +#include "FaceRig/FaceRigLandmarks.h" #include "GamificationManager.h" #include "Manager.h" #include "SelectionSet.h" @@ -119,6 +120,15 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu opts.maxShapes = maxShapes; opts.maxFitResidualPct = maxResidualPct; + // MAIN thread: facial-landmark anchors (renders template + user — Ogre) so + // the worker's fit lands on the real face features. Empty when ONNX/model/ + // face-detection unavailable → the fit runs unanchored (previous behaviour). + setStatus(QStringLiteral("Detecting face landmarks…")); + std::vector headV; std::vector headF; + FaceRig::headSubmesh(*geo, headV, headF); + auto anchors = std::make_shared>( + FaceRig::buildLandmarkAnchors(entity, headV, headF, *tmpl)); + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), QStringLiteral("face_rig entity=%1 verts=%2 template=%3v") .arg(QString::fromStdString(entity->getName())) @@ -137,7 +147,7 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu QPointer self(this); // WORKER thread: the heavy Ogre-free fit + transfer over all 52 shapes. - std::thread([self, geo, tmpl, opts, entName, cancel]() { + std::thread([self, geo, tmpl, opts, entName, cancel, anchors]() { // Progress callback: marshal the counters to the main thread for the // progress bar; return false to stop the worker when cancel is set. auto progress = [self, cancel](int done, int total, @@ -155,7 +165,7 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu }; auto result = std::make_shared( FaceRig::buildFaceRig(geo->userV, geo->userF, *tmpl, opts, - geo->headMask, progress)); + geo->headMask, *anchors, progress)); // MAIN thread: attach (touches Ogre + the undo stack). QMetaObject::invokeMethod(qApp, [self, geo, result, entName]() { diff --git a/src/FaceRigController.h b/src/FaceRigController.h index 4622853d4..1f4a70594 100644 --- a/src/FaceRigController.h +++ b/src/FaceRigController.h @@ -5,6 +5,9 @@ #include #include +#include +#include + // QML-facing singleton for the face auto-rig (#889, Slice F #895). Wraps // FaceRig::attachFaceRig* plus selection state so the Inspector's "Add ARKit // Blendshapes" button can enable itself only on a mesh selection and run the @@ -28,6 +31,9 @@ class FaceRigController : public QObject Q_PROPERTY(bool downloading READ downloading NOTIFY statusChanged) // Short human-readable status for the button label / tooltip. Q_PROPERTY(QString status READ status NOTIFY statusChanged) + // Worker progress for a progress bar: done / total steps (0 total = idle). + Q_PROPERTY(int progress READ progress NOTIFY progressChanged) + Q_PROPERTY(int progressTotal READ progressTotal NOTIFY progressChanged) public: static FaceRigController* instance(); @@ -38,6 +44,8 @@ class FaceRigController : public QObject bool busy() const { return m_busy; } bool downloading() const { return m_downloading; } QString status() const { return m_status; } + int progress() const { return m_progress; } + int progressTotal() const { return m_progressTotal; } /// Run the face auto-rig on the first selected entity and attach the ARKit /// blendshapes. Prepares (extracts geometry + loads the bundled template) @@ -48,10 +56,15 @@ class FaceRigController : public QObject Q_INVOKABLE bool addArkitBlendshapesAsync(int maxShapes = 0, double maxResidualPct = 8.0); + /// Request cancellation of an in-flight fit (no-op otherwise). The worker + /// stops at the next progress step; `error("cancelled")` follows. + Q_INVOKABLE void cancel(); + signals: void selectionChanged(); void busyChanged(); void statusChanged(); + void progressChanged(); void completed(const QVariantMap& report); void error(const QString& message); @@ -65,6 +78,9 @@ class FaceRigController : public QObject bool m_busy = false; bool m_downloading = false; QString m_status; + int m_progress = 0; + int m_progressTotal = 0; + std::shared_ptr m_cancel; }; #endif // FACE_RIG_CONTROLLER_H diff --git a/src/MeshDepthRenderer.cpp b/src/MeshDepthRenderer.cpp index 98b855861..ec7058864 100644 --- a/src/MeshDepthRenderer.cpp +++ b/src/MeshDepthRenderer.cpp @@ -290,6 +290,117 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( return result; } +MeshDepthRenderer::RenderResult MeshDepthRenderer::renderShadedView( + Ogre::Entity* entity, int size, const View& view, QString* errorOut, + const Ogre::AxisAlignedBox* focusAabb) +{ + RenderResult result; + if (!entity) { + if (errorOut) *errorOut = QStringLiteral("null entity"); + return result; + } + size = std::clamp(size, 64, 2048); + if (!ensureRenderTarget(size, errorOut)) + return result; + + auto* sm = sceneMgr(); + DepthState& st = state(); + + // Frame on the focus box (head) when given, else the whole entity. + const Ogre::AxisAlignedBox aabb = + (focusAabb && !focusAabb->isNull()) ? *focusAabb + : entity->getWorldBoundingBox(true); + const Ogre::Vector3 center = aabb.getCenter(); + const Ogre::Real radius = aabb.getHalfSize().length(); + if (radius <= 0.0f) { + if (errorOut) *errorOut = QStringLiteral("entity has zero-size bounding box"); + return result; + } + const Ogre::Real fovY = st.camera->getFOVy().valueRadians(); + const Ogre::Real dist = radius / std::sin(fovY * 0.5f) * 1.15f; + Ogre::Vector3 dir = view.dir; + if (dir.isZeroLength()) dir = Ogre::Vector3(0, 0, 1); + dir.normalise(); + const Ogre::Vector3 camPos = center - dir * dist; + st.cameraNode->setPosition(camPos); + const Ogre::Vector3 up = view.up.isZeroLength() ? Ogre::Vector3::UNIT_Y : view.up; + st.cameraNode->setFixedYawAxis(true, up); + st.cameraNode->lookAt(center, Ogre::Node::TS_WORLD, + Ogre::Vector3::NEGATIVE_UNIT_Z); + + // Neutral flat lighting so MediaPipe sees an evenly-lit, photo-like face + // (no fog, materials intact). Bright ambient + a head-on light fills in + // feature shading without harsh shadows that would confuse the detector. + const Ogre::ColourValue savedAmbient = sm->getAmbientLight(); + const Ogre::FogMode savedFogMode = sm->getFogMode(); + const Ogre::ColourValue savedFogColour = sm->getFogColour(); + const Ogre::Real savedFogStart = sm->getFogStart(); + const Ogre::Real savedFogEnd = sm->getFogEnd(); + sm->setFog(Ogre::FOG_NONE); + sm->setAmbientLight(Ogre::ColourValue(0.75f, 0.75f, 0.75f)); + + Ogre::Light* light = nullptr; + Ogre::SceneNode* lightNode = nullptr; + try { + light = sm->createLight("QtMeshFaceRigLight"); + light->setType(Ogre::Light::LT_DIRECTIONAL); + light->setDiffuseColour(Ogre::ColourValue(0.5f, 0.5f, 0.5f)); + light->setSpecularColour(Ogre::ColourValue::Black); + lightNode = sm->getRootSceneNode()->createChildSceneNode(); + lightNode->attachObject(light); + lightNode->setDirection(dir, Ogre::Node::TS_WORLD); + } catch (...) { /* lighting is best-effort */ } + + Ogre::SceneNode* gridNode = nullptr; + bool gridWasVisible = false; + if (Manager::getSingletonPtr() + && Manager::getSingleton()->hasSceneNode("GridLine_node")) { + gridNode = Manager::getSingleton()->getSceneNode("GridLine_node"); + if (gridNode) { + gridWasVisible = gridNode->getAttachedObject(0) + ? gridNode->getAttachedObject(0)->getVisible() : true; + gridNode->setVisible(false); + } + } + Ogre::SceneNode* targetNode = entity->getParentSceneNode(); + if (targetNode) targetNode->showBoundingBox(false); + std::vector> hiddenNodes; + if (Manager::getSingletonPtr()) { + for (Ogre::Entity* other : Manager::getSingleton()->getEntities()) { + if (!other || other == entity) continue; + if (other->getMovableType() != "Entity") continue; + Ogre::SceneNode* n = other->getParentSceneNode(); + if (n && n->getAttachedObject(0) && n->getAttachedObject(0)->getVisible()) { + hiddenNodes.emplace_back(n, true); + n->setVisible(false); + } + } + } + + auto restore = [&]() { + sm->setAmbientLight(savedAmbient); + sm->setFog(savedFogMode, savedFogColour, 0.0f, savedFogStart, savedFogEnd); + if (lightNode) { lightNode->detachAllObjects(); + sm->getRootSceneNode()->removeAndDestroyChild(lightNode); } + if (light) sm->destroyLight(light); + if (gridNode) gridNode->setVisible(gridWasVisible); + if (targetNode) targetNode->showBoundingBox(true); + for (auto& [n, wasVisible] : hiddenNodes) n->setVisible(wasVisible); + }; + struct Restorer { std::function fn; ~Restorer() { fn(); } } restorer{restore}; + + st.renderTarget->update(); + QImage rgba = readRenderTarget(size); + OgreRenderTargetUtil::restoreEditorRenderTarget(); + + result.viewMatrix = st.camera->getViewMatrix(); + result.projMatrix = st.camera->getProjectionMatrix(); + result.camPosition = camPos; + result.camDirection = dir; + result.depth = rgba.convertToFormat(QImage::Format_RGB888); // RGB, not gray + return result; +} + void MeshDepthRenderer::shutdown() { DepthState& st = state(); diff --git a/src/MeshDepthRenderer.h b/src/MeshDepthRenderer.h index d7bc01341..4b26b6f64 100644 --- a/src/MeshDepthRenderer.h +++ b/src/MeshDepthRenderer.h @@ -83,6 +83,23 @@ class MeshDepthRenderer { int size, QString* errorOut = nullptr); + // Render the entity with its REAL materials (not the flat depth material), + // lit by a temporary head-on light so faces show feature shading — for the + // face auto-rig's landmark detection (#889), which feeds a photo-like image + // to MediaPipe FaceMesh. Same framing / hide-others / RTT reuse as the depth + // path; RenderResult.depth holds the RGB888 image (name kept for reuse) and + // the view/proj/cam fields let the caller back-project 2D landmarks to the + // mesh surface. Main/render thread only. + // `focusAabb` (optional, WORLD space): when non-null, frame the camera on + // THIS box instead of the entity's full bounds — so a full-body character + // can be rendered tightly around the HEAD (the face fills the frame for the + // landmark detector). Pass nullptr to frame the whole entity. + static RenderResult renderShadedView(Ogre::Entity* entity, + int size, + const View& view, + QString* errorOut = nullptr, + const Ogre::AxisAlignedBox* focusAabb = nullptr); + // Release the cached RTT / camera / scene nodes. Safe to call // when Ogre is shutting down. static void shutdown(); From 6e45700d752788d1122164f19075e51169ae85c0 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 15 Jul 2026 15:43:08 -0400 Subject: [PATCH 11/38] fix(#889): dedup morph rows on multi-submesh + face-marker foundation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two field-reported issues + the groundwork for the fix you asked for (auto-place markers, user adjusts): - Duplicate morph-target rows: a target that spans N submeshes has one Ogre pose PER submesh, all same-named (Rumba's face = 11 submeshes → 11 "jawOpen" rows). morphTargetsFor() now coalesces by name (every by-name op already operates across all matching poses), so the Inspector lists each target once. - Attach robustness: hide the entity for the pose-attach batch so the render loop can't touch its pose/skin buffers mid-rebuild (belt-and-braces with the animation-state disable + the single deferred _initialise). - Face-marker foundation (FaceRigLandmarks): a canonical marker catalog (nose, chin, eye corners, mouth corners, brows, …) mapped to stable MediaPipe FaceMesh indices; seedFaceMarkers() resolves each marker's TEMPLATE vertex from the (reliable) template detection and seeds the editable USER position from auto-detect when confident, else a head-box-projected default for the user to drag; anchorsFromMarkers() turns the (edited) markers into the same NRICP landmark anchors. This is the base for the click-to-adjust UI — needed because MediaPipe FaceMesh returns a garbage point-cloud on cartoon/stylized faces (verified via a landmark overlay), so auto-detect must be a SEED the user corrects, not the final answer. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/FaceRig/FaceRigLandmarks.cpp | 159 +++++++++++++++++++++++++++++++ src/FaceRig/FaceRigLandmarks.h | 37 +++++++ src/FaceRigController.cpp | 8 ++ src/MorphAnimationManager.cpp | 14 ++- 4 files changed, 217 insertions(+), 1 deletion(-) diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index dec9d6d28..175bc186b 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -92,6 +92,21 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, const LandmarkResult lr = det.detect(rr.depth); if (!lr.ok || lr.points.empty()) return out; out.confidence = lr.confidence; +#ifndef NDEBUG + if (const char* dp = std::getenv("QTMESH_FACERIG_DUMP_LANDMARKS")) { + QImage vis = rr.depth.convertToFormat(QImage::Format_RGB888); + for (const auto& p : lr.points) { + const int px = int(p[0]), py = int(p[1]); + for (int dy = -1; dy <= 1; ++dy) + for (int dx = -1; dx <= 1; ++dx) { + const int x = px+dx, y = py+dy; + if (x >= 0 && y >= 0 && x < vis.width() && y < vis.height()) + vis.setPixel(x, y, qRgb(0, 255, 0)); + } + } + vis.save(QString::fromUtf8(dp)); + } +#endif // 3) build world-space head triangles (local verts × node world transform). // The render framed the WORLD bounding box, so rays are in world space; @@ -284,4 +299,148 @@ std::vector buildLandmarkAnchors( return anchors; } +// Canonical MediaPipe FaceMesh indices for the few anatomical anchors the fit +// needs. These indices are stable in the FaceMesh topology (documented in the +// MediaPipe canonical face model). +const std::vector>& faceMarkerCatalog() +{ + static const std::vector> kCatalog = { + {QStringLiteral("Nose tip"), 1}, + {QStringLiteral("Chin"), 152}, + {QStringLiteral("Left eye outer"), 33}, + {QStringLiteral("Right eye outer"), 263}, + {QStringLiteral("Left eye inner"), 133}, + {QStringLiteral("Right eye inner"), 362}, + {QStringLiteral("Left mouth corner"), 61}, + {QStringLiteral("Right mouth corner"),291}, + {QStringLiteral("Upper lip"), 13}, + {QStringLiteral("Lower lip"), 14}, + {QStringLiteral("Left brow"), 105}, + {QStringLiteral("Right brow"), 334}, + {QStringLiteral("Forehead"), 10}, + }; + return kCatalog; +} + +namespace { +int nearestTemplateVertex(const std::vector& tn, int tvc, + const std::array& p) +{ + int best = -1; float bestD = std::numeric_limits::max(); + for (int vtx = 0; vtx < tvc; ++vtx) { + const float dx = tn[size_t(vtx)*3] - p[0]; + const float dy = tn[size_t(vtx)*3+1] - p[1]; + const float dz = tn[size_t(vtx)*3+2] - p[2]; + const float d = dx*dx + dy*dy + dz*dz; + if (d < bestD) { bestD = d; best = vtx; } + } + return best; +} +} // namespace + +std::vector seedFaceMarkers( + Ogre::Entity* userEntity, + const std::vector& userLocalV, + const std::vector& userLocalF, + const ArkitTemplate& tmpl, + bool* outConfident) +{ + std::vector markers; + if (outConfident) *outConfident = false; + if (!userEntity || !tmpl.valid()) return markers; + + const auto& cat = faceMarkerCatalog(); + markers.reserve(cat.size()); + for (const auto& [label, idx] : cat) { + FaceMarker m; m.label = label; m.mediapipeIndex = idx; + markers.push_back(std::move(m)); + } + + if (!FaceLandmarkDetector::backendAvailable()) return markers; + { FaceLandmarkDetector probe; if (!probe.load()) return markers; } + + // Template detection resolves each marker's TEMPLATE vertex (reliable — the + // ICT template is a real human face). + Ogre::Entity* tent = templateEntity(tmpl.neutral(), tmpl.faces()); + if (!tent) return markers; + if (auto* tnode = tent->getParentSceneNode()) tnode->setVisible(true); + const MeshLandmarks tlm = detectMeshLandmarks(tent, tmpl.neutral(), + std::vector(tmpl.faces())); + if (auto* tnode = tent->getParentSceneNode()) tnode->setVisible(false); + + const int tvc = tmpl.vertexCount(); + const auto& tn = tmpl.neutral(); + if (tlm.ok) { + for (auto& m : markers) { + const int i = m.mediapipeIndex; + if (i >= 0 && i < int(tlm.points.size()) && tlm.valid[size_t(i)]) + m.tmplVertex = nearestTemplateVertex(tn, tvc, tlm.points[size_t(i)]); + } + } + + // User detection seeds the editable positions. If it's confident, use its + // points; otherwise leave markers unplaced at a sensible default (projected + // template landmark into the user head box) for the user to drag. + const MeshLandmarks ulm = detectMeshLandmarks(userEntity, userLocalV, userLocalF); + + // Confidence heuristic: enough of the catalog markers hit the surface AND + // the landmark cloud isn't a degenerate blob. We approximate "trustworthy" + // by how many catalog markers are valid. + int seeded = 0; + if (ulm.ok) { + for (auto& m : markers) { + const int i = m.mediapipeIndex; + if (i >= 0 && i < int(ulm.points.size()) && ulm.valid[size_t(i)]) { + m.userPos = ulm.points[size_t(i)]; + m.placed = true; + ++seeded; + } + } + } + // If auto-seed was weak, place the rest at the head-box-projected template + // landmark so every marker starts SOMEWHERE on the face for the user to + // nudge (rather than at the origin). + if (seeded < int(markers.size())) { + // user head AABB + std::array lo{1e30f,1e30f,1e30f}, hi{-1e30f,-1e30f,-1e30f}; + const int unv = int(userLocalV.size()/3); + for (int i = 0; i < unv; ++i) + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], userLocalV[size_t(i)*3+a]); + hi[a] = std::max(hi[a], userLocalV[size_t(i)*3+a]); + } + // template head AABB (its neutral) + std::array tlo{1e30f,1e30f,1e30f}, thi{-1e30f,-1e30f,-1e30f}; + for (int i = 0; i < tvc; ++i) + for (int a = 0; a < 3; ++a) { + tlo[a] = std::min(tlo[a], tn[size_t(i)*3+a]); + thi[a] = std::max(thi[a], tn[size_t(i)*3+a]); + } + for (auto& m : markers) { + if (m.placed || m.tmplVertex < 0) continue; + // map the template vertex position into the user head box. + for (int a = 0; a < 3; ++a) { + const float tv = tn[size_t(m.tmplVertex)*3+a]; + const float f = (thi[a]-tlo[a]) > 1e-6f + ? (tv - tlo[a]) / (thi[a]-tlo[a]) : 0.5f; + m.userPos[size_t(a)] = lo[a] + f * (hi[a]-lo[a]); + } + // left unplaced: it's a default guess the user should confirm/drag. + } + } + + const bool confident = ulm.ok && seeded >= int(markers.size()) * 3 / 4; + if (outConfident) *outConfident = confident; + return markers; +} + +std::vector anchorsFromMarkers(const std::vector& markers) +{ + std::vector anchors; + for (const auto& m : markers) + if (m.placed && m.tmplVertex >= 0) + anchors.push_back({m.tmplVertex, m.userPos}); + return anchors; +} + } // namespace FaceRig diff --git a/src/FaceRig/FaceRigLandmarks.h b/src/FaceRig/FaceRigLandmarks.h index 6d4cfc03f..af88463db 100644 --- a/src/FaceRig/FaceRigLandmarks.h +++ b/src/FaceRig/FaceRigLandmarks.h @@ -12,7 +12,10 @@ #include "NonRigidICP.h" // NricpLandmark +#include + #include +#include #include namespace Ogre { class Entity; } @@ -21,6 +24,22 @@ namespace FaceRig { class ArkitTemplate; +// A user-adjustable face marker: a canonical facial point (eye/nose/mouth/…) +// with the TEMPLATE vertex it anchors and the current USER-mesh position (seeded +// from auto-detection, then draggable). label is a short guidance string. +struct FaceMarker { + QString label; + int mediapipeIndex = -1; // canonical MediaPipe FaceMesh index + int tmplVertex = -1; // template vertex it pins (from tmpl detect) + std::array userPos{0, 0, 0}; // user-mesh position (editable) + bool placed = false; // seeded-or-user-set (vs unresolved) +}; + +// The canonical marker set (label + MediaPipe index), fixed order. These are +// the anatomical anchors the fit needs to lock orientation/scale; the user only +// has to get these few roughly right, not all 478. +const std::vector>& faceMarkerCatalog(); + struct MeshLandmarks { // 3D landmark positions in MESH-LOCAL space (same frame as the geometry the // fit uses), one per MediaPipe index; `valid[i]` is false when landmark i @@ -52,6 +71,24 @@ std::vector buildLandmarkAnchors( const std::vector& userLocalF, const ArkitTemplate& tmpl); +// Seed the editable face-marker set: detect on the template (reliable — a real +// human face) to resolve each marker's template vertex, then AUTO-DETECT on the +// user head to seed userPos when it works (marker.placed=true) or fall back to +// the template landmark position mapped into the user's head box when it +// doesn't (cartoon faces — the user then drags to correct). Always returns the +// full catalog with template verts resolved; `outConfident` reports whether the +// user auto-detect looked trustworthy. +std::vector seedFaceMarkers( + Ogre::Entity* userEntity, + const std::vector& userLocalV, + const std::vector& userLocalF, + const ArkitTemplate& tmpl, + bool* outConfident = nullptr); + +// Build NRICP anchors from the (possibly user-edited) markers — one per placed +// marker with a resolved template vertex. +std::vector anchorsFromMarkers(const std::vector& markers); + } // namespace FaceRig #endif // FACERIGLANDMARKS_H diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 6d6be39b0..96b48176e 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -218,6 +219,12 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu } } } + // Hide the entity for the batch so the render loop doesn't touch its + // pose/skin buffers while we rebuild them — restored after the single + // _initialise. (Belt-and-braces with the animation-state disable.) + Ogre::SceneNode* enode = entity->getParentSceneNode(); + const bool wasVisible = enode ? entity->getVisible() : true; + if (enode) entity->setVisible(false); // Build the per-shape commands first so we know which is LAST — only // the last re-initialises the entity (deferInit=false on it), the @@ -251,6 +258,7 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu if (stack) stack->beginMacro(QStringLiteral("Add ARKit Blendshapes")); for (auto* cmd : cmds) { undo->push(cmd); rep.shapesAttached++; } if (stack) stack->endMacro(); + if (enode) entity->setVisible(wasVisible); // Restore the animation states we disabled (refreshAvailable... in // the attach may have recreated the state set, so re-resolve). diff --git a/src/MorphAnimationManager.cpp b/src/MorphAnimationManager.cpp index 555941f44..2bf38b5a8 100644 --- a/src/MorphAnimationManager.cpp +++ b/src/MorphAnimationManager.cpp @@ -21,6 +21,7 @@ The MIT License #include "commands/MorphCommands.h" #include +#include #include #include @@ -101,11 +102,22 @@ QStringList MorphAnimationManager::morphTargetsFor(Ogre::Entity* entity) const if (!entity) return out; Ogre::MeshPtr mesh = entity->getMesh(); if (!mesh) return out; + // A morph target that spans multiple submeshes has ONE pose per submesh + // handle, all sharing the same name (common on multi-submesh characters — + // e.g. a face split across 11 submeshes yields 11 "jawOpen" poses). The + // UI + every by-name op (weight/key/delete) treat a target as its NAME, so + // list each distinct name once — otherwise the Inspector shows a duplicate + // row per submesh. + QSet seen; const auto& poseList = mesh->getPoseList(); for (const Ogre::Pose* p : poseList) { if (!p) continue; const Ogre::String n = p->getName(); - if (!n.empty()) out << QString::fromStdString(n); + if (n.empty()) continue; + const QString qn = QString::fromStdString(n); + if (seen.contains(qn)) continue; // coalesce same-named poses + seen.insert(qn); + out << qn; } return out; } From e0fe24a725c46835bff927f81ddf3dd82f3be5fe Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 15 Jul 2026 15:51:45 -0400 Subject: [PATCH 12/38] =?UTF-8?q?feat(#889):=20editable=20face=20markers?= =?UTF-8?q?=20=E2=80=94=20auto-seed=20+=20user-adjust=20(cartoon=20faces)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MediaPipe FaceMesh returns garbage on cartoon/stylized faces (verified via a landmark overlay), so pure auto-detection can't anchor them. Per the chosen design, auto-detection now SEEDS a small set of draggable markers that the user corrects — the reliable path for the stylized characters people actually rig. - FaceRigController marker session: beginFaceMarkers() extracts geometry, loads the template, and seeds the markers (template detection resolves each marker's template vertex reliably — the ICT template is a real face; user detection seeds positions when confident, else a head-box-projected default). Draggable PT_SPHERE overlays (cyan = selected, yellow = placed, grey = default). selectMarker/handleMarkerClick reposition a marker on the mesh surface (Ogre::Math ray/tri); rigFromMarkers() builds NRICP anchors from the edited markers and runs the SAME fit+attach (shared runRigAsync, refactored out of addArkitBlendshapesAsync). TransformOperator routes viewport clicks (same priority as AutoRig markers). - QML: "Place / adjust face markers…" button + an in-session panel — marker chips (click to select, colour-coded by state), "Rig from markers", "Cancel". Shows whether the auto-seed was confident so the user knows how much to fix. Both paths (direct auto + marker-anchored) share runRigAsync, so all the prior fixes (head isolation, deferred attach, animation-disable, progress+cancel) apply to both. App loads with no QML errors; full FaceRig + morph suites green. Co-Authored-By: Claude Opus 4.8 (1M context) --- qml/PropertiesPanel.qml | 133 ++++++++++++++++++ src/FaceRigController.cpp | 286 ++++++++++++++++++++++++++++++++++++-- src/FaceRigController.h | 68 +++++++++ src/TransformOperator.cpp | 8 ++ 4 files changed, 487 insertions(+), 8 deletions(-) diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 3df615f03..50d7bbb00 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -9424,6 +9424,139 @@ Rectangle { : "Select a face mesh first." } } + + // ── Face markers (auto-seed, user adjusts) — the reliable path + // for cartoon/stylized faces MediaPipe can't detect. #889. + Rectangle { + id: markerBtn + width: parent.width + height: 22 + visible: !FaceRigController.markerMode + property bool canRun: FaceRigController.hasMeshSelection + && !FaceRigController.busy + opacity: canRun ? 1.0 : 0.5 + radius: 3 + color: markerMa.containsMouse && canRun + ? Qt.lighter(PropertiesPanelController.headerColor, 1.3) + : PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: "◎ Place / adjust face markers…" + color: PropertiesPanelController.textColor + font.pixelSize: 9 + } + MouseArea { + id: markerMa + anchors.fill: parent + hoverEnabled: true + enabled: markerBtn.canRun + cursorShape: enabled ? Qt.PointingHandCursor : Qt.ForbiddenCursor + onClicked: FaceRigController.beginFaceMarkers() + ToolTip.visible: containsMouse + ToolTip.text: "Auto-place face anchors (eyes, nose, mouth, " + + "chin) and drag any that are off, then rig from them. " + + "Use this for cartoon/stylized faces where auto-detect " + + "struggles." + } + } + + // Marker-editing panel — shown only during a marker session. + Column { + width: parent.width + visible: FaceRigController.markerMode + spacing: 4 + Text { + width: parent.width + wrapMode: Text.Wrap + font.pixelSize: 9 + color: PropertiesPanelController.textColor + text: (FaceRigController.markersSeededFromDetection + ? "Auto-detected. " : "Auto-detect was weak — ") + + "Click a marker below, then click on the face to move " + + "it. Cyan = selected." + } + // Marker chips — click to select which one the next mesh + // click will move. + Flow { + width: parent.width + spacing: 3 + Repeater { + model: FaceRigController.markerLabels + Rectangle { + height: 18 + width: chipText.implicitWidth + 12 + radius: 3 + property bool sel: index === FaceRigController.selectedMarker + property bool placed: FaceRigController.markerPlaced(index) + color: sel ? "#2ae6ff" + : placed ? PropertiesPanelController.highlightColor + : PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + opacity: placed || sel ? 1.0 : 0.6 + Text { + id: chipText + anchors.centerIn: parent + text: modelData + font.pixelSize: 8 + color: sel ? "#003" : PropertiesPanelController.textColor + } + MouseArea { + anchors.fill: parent + cursorShape: Qt.PointingHandCursor + onClicked: FaceRigController.selectMarker(index) + } + } + } + } + RowLayout { + width: parent.width + spacing: 4 + Rectangle { + Layout.fillWidth: true + Layout.preferredHeight: 22 + radius: 3 + color: rigMkMa.containsMouse + ? Qt.lighter(PropertiesPanelController.highlightColor, 1.1) + : PropertiesPanelController.highlightColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: "✓ Rig from markers" + font.pixelSize: 9; font.bold: true + color: PropertiesPanelController.textColor + } + MouseArea { + id: rigMkMa + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + onClicked: FaceRigController.rigFromMarkers(0, 8.0) + } + } + Rectangle { + Layout.preferredWidth: 56 + Layout.preferredHeight: 22 + radius: 3 + color: cancelMkMa.containsMouse + ? Qt.lighter(PropertiesPanelController.headerColor, 1.3) + : PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: "Cancel"; font.pixelSize: 9 + color: PropertiesPanelController.textColor + } + MouseArea { + id: cancelMkMa + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + onClicked: FaceRigController.cancelFaceMarkers() + } + } + } + } // Progress bar + Cancel, shown only while the worker runs. RowLayout { width: parent.width diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 96b48176e..003e7b7f6 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -5,7 +5,9 @@ #include "FaceRig/FaceRigLandmarks.h" #include "GamificationManager.h" #include "Manager.h" +#include "OgreWidget.h" #include "SelectionSet.h" +#include "SpaceCamera.h" #include "SentryReporter.h" #include "UndoManager.h" #include "commands/MorphCommands.h" @@ -117,23 +119,46 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu return false; } - FaceRig::FaceRigOptions opts; - opts.maxShapes = maxShapes; - opts.maxFitResidualPct = maxResidualPct; - // MAIN thread: facial-landmark anchors (renders template + user — Ogre) so // the worker's fit lands on the real face features. Empty when ONNX/model/ // face-detection unavailable → the fit runs unanchored (previous behaviour). setStatus(QStringLiteral("Detecting face landmarks…")); std::vector headV; std::vector headF; FaceRig::headSubmesh(*geo, headV, headF); - auto anchors = std::make_shared>( - FaceRig::buildLandmarkAnchors(entity, headV, headF, *tmpl)); + const std::vector anchors = + FaceRig::buildLandmarkAnchors(entity, headV, headF, *tmpl); + + m_geo = geo; + return runRigAsync(tmpl, maxShapes, maxResidualPct, anchors); +} + +bool FaceRigController::runRigAsync( + const std::shared_ptr& tmpl, + int maxShapes, double maxResidualPct, + const std::vector& anchorsIn) +{ + auto geo = m_geo; + if (!geo || !geo->valid()) { + emit error(QStringLiteral("Could not read the mesh geometry.")); + return false; + } + auto* sel = SelectionSet::getSingleton(); + const auto entities = sel ? sel->getResolvedEntities() + : QList{}; + Ogre::Entity* entity = entities.isEmpty() ? nullptr : entities.first(); + if (!entity) { emit error(QStringLiteral("No mesh selected.")); return false; } + + FaceRig::FaceRigOptions opts; + opts.maxShapes = maxShapes; + opts.maxFitResidualPct = maxResidualPct; + auto anchors = + std::make_shared>(anchorsIn); SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), - QStringLiteral("face_rig entity=%1 verts=%2 template=%3v") + QStringLiteral("face_rig entity=%1 verts=%2 template=%3v anchors=%4") .arg(QString::fromStdString(entity->getName())) - .arg(geo->userV.size() / 3).arg(tmpl->vertexCount())); + .arg(geo->userV.size() / 3).arg(tmpl->vertexCount()) + .arg(anchors->size())); m_busy = true; emit busyChanged(); @@ -298,3 +323,248 @@ void FaceRigController::cancel() { if (m_cancel) m_cancel->store(true); } + +// ─────────────────── Face-marker editing session ──────────────────── + +QStringList FaceRigController::markerLabels() const +{ + QStringList out; + for (const auto& m : m_markers) out << m.label; + return out; +} + +bool FaceRigController::markerPlaced(int index) const +{ + return index >= 0 && index < int(m_markers.size()) + && m_markers[size_t(index)].placed; +} + +bool FaceRigController::beginFaceMarkers() +{ + if (m_busy) { emit error(QStringLiteral("Busy.")); return false; } + auto* sel = SelectionSet::getSingleton(); + const auto entities = sel ? sel->getResolvedEntities() + : QList{}; + Ogre::Entity* entity = entities.isEmpty() ? nullptr : entities.first(); + if (!entity || !entity->getMesh()) { + emit error(QStringLiteral("No mesh selected.")); + return false; + } + + // Geometry + template (same prep as the direct rig). + m_geo = std::make_shared( + FaceRig::extractGeometry(entity)); + if (!m_geo->valid()) { + emit error(QStringLiteral("Could not read the mesh geometry.")); + return false; + } + m_downloading = !FaceRig::ArkitTemplate::present(); + setStatus(m_downloading ? QStringLiteral("Downloading face template…") + : QStringLiteral("Preparing…")); + const QString path = FaceRig::ArkitTemplate::ensureModelBlocking(); + m_downloading = false; + if (path.isEmpty()) { + setStatus(QString()); + emit error(QStringLiteral("ARKit template unavailable.")); + return false; + } + m_markerTmpl = std::make_shared(); + QString loadErr; + if (!m_markerTmpl->load(path, &loadErr)) { + setStatus(QString()); + emit error(QStringLiteral("Failed to load ARKit template: %1").arg(loadErr)); + return false; + } + + // Seed markers: template detection resolves template verts (reliable), + // user detection seeds positions when confident, else sensible defaults. + setStatus(QStringLiteral("Detecting face landmarks…")); + std::vector headV; std::vector headF; + FaceRig::headSubmesh(*m_geo, headV, headF); + m_markers = FaceRig::seedFaceMarkers(entity, headV, headF, *m_markerTmpl, + &m_seededConfident); + setStatus(QString()); + if (m_markers.empty()) { + emit error(QStringLiteral("Could not prepare face markers.")); + return false; + } + + m_markerEntityName = entity->getName(); + m_markerMode = true; + m_selMarker = 0; + refreshMarkerOverlays(); + emit markerModeChanged(); + emit markersChanged(); + return true; +} + +void FaceRigController::selectMarker(int index) +{ + if (!m_markerMode) return; + m_selMarker = (index >= 0 && index < int(m_markers.size())) ? index : -1; + refreshMarkerOverlays(); + emit markersChanged(); +} + +void FaceRigController::cancelFaceMarkers() +{ + if (!m_markerMode) return; + m_markerMode = false; + m_markers.clear(); + m_selMarker = -1; + clearMarkerOverlays(); + setStatus(QString()); + emit markerModeChanged(); + emit markersChanged(); +} + +bool FaceRigController::handleMarkerClick(OgreWidget* widget, const QPoint& screenPos) +{ + if (!m_markerMode || !widget) return false; + + Ogre::Entity* entity = nullptr; + for (Ogre::Entity* e : Manager::getSingleton()->getEntities()) + if (e && e->getMovableType() == "Entity" + && e->getName() == m_markerEntityName) { entity = e; break; } + if (!entity) { cancelFaceMarkers(); return false; } + + auto* spaceCam = widget->getSpaceCamera(); + auto* cam = spaceCam ? spaceCam->getCamera() : nullptr; + if (!cam) return true; + int vw = 0, vh = 0; + widget->pixelSizeForCameraPicking(vw, vh); + if (vw <= 0 || vh <= 0) return true; + const Ogre::Real nx = Ogre::Real(screenPos.x()) / vw; + const Ogre::Real ny = Ogre::Real(screenPos.y()) / vh; + const Ogre::Ray ray = cam->getCameraToViewportRay(nx, ny); + + // Ray-cast to the mesh surface (world-space triangles from the geometry). + Ogre::Node* node = entity->getParentSceneNode(); + const Ogre::Matrix4 world = node ? node->_getFullTransform() + : Ogre::Matrix4::IDENTITY; + const auto& V = m_geo->userV; const auto& F = m_geo->userF; + const int nv = int(V.size()/3); + float bestT = std::numeric_limits::max(); + Ogre::Vector3 hitLocal; bool found = false; + for (size_t f = 0; f + 2 < F.size(); f += 3) { + const int ia=F[f], ib=F[f+1], ic=F[f+2]; + if (ia<0||ib<0||ic<0||ia>=nv||ib>=nv||ic>=nv) continue; + const Ogre::Vector3 a = world*Ogre::Vector3(V[size_t(ia)*3],V[size_t(ia)*3+1],V[size_t(ia)*3+2]); + const Ogre::Vector3 b = world*Ogre::Vector3(V[size_t(ib)*3],V[size_t(ib)*3+1],V[size_t(ib)*3+2]); + const Ogre::Vector3 c = world*Ogre::Vector3(V[size_t(ic)*3],V[size_t(ic)*3+1],V[size_t(ic)*3+2]); + auto res = Ogre::Math::intersects(ray, a, b, c, true, false); + if (res.first && res.second < bestT) { + bestT = res.second; + const Ogre::Vector3 w = ray.getPoint(res.second); + hitLocal = world.inverse() * w; + found = true; + } + } + if (!found) return true; // missed the mesh — consume + + if (m_selMarker < 0 || m_selMarker >= int(m_markers.size())) { + // no selection → pick the nearest marker to the hit, don't move it. + int best = -1; float bd = std::numeric_limits::max(); + for (int i = 0; i < int(m_markers.size()); ++i) { + const auto& p = m_markers[size_t(i)].userPos; + const float d = (Ogre::Vector3(p[0],p[1],p[2]) - hitLocal).squaredLength(); + if (d < bd) { bd = d; best = i; } + } + m_selMarker = best; + } else { + // move the selected marker to the hit point. + auto& m = m_markers[size_t(m_selMarker)]; + m.userPos = { hitLocal.x, hitLocal.y, hitLocal.z }; + m.placed = true; + // auto-advance to the next unplaced marker for a smooth flow. + int next = -1; + for (int i = 1; i <= int(m_markers.size()); ++i) { + const int idx = (m_selMarker + i) % int(m_markers.size()); + if (!m_markers[size_t(idx)].placed) { next = idx; break; } + } + m_selMarker = next >= 0 ? next : m_selMarker; + } + refreshMarkerOverlays(); + emit markersChanged(); + return true; +} + +bool FaceRigController::rigFromMarkers(int maxShapes, double maxResidualPct) +{ + if (!m_markerMode) { emit error(QStringLiteral("Not in marker mode.")); return false; } + auto tmpl = m_markerTmpl; + const auto anchors = FaceRig::anchorsFromMarkers(m_markers); + // Leave marker mode (clears overlays) before the rig runs. + m_markerMode = false; + m_selMarker = -1; + clearMarkerOverlays(); + emit markerModeChanged(); + emit markersChanged(); + if (!tmpl) { emit error(QStringLiteral("Template not loaded.")); return false; } + return runRigAsync(tmpl, maxShapes, maxResidualPct, anchors); +} + +void FaceRigController::clearMarkerOverlays() +{ + auto* mgr = Manager::getSingletonPtr(); + Ogre::SceneManager* scene = mgr ? mgr->getSceneMgr() : nullptr; + for (Ogre::SceneNode* n : m_markerNodes) { + if (!n) continue; + if (scene) { + auto objs = n->getAttachedObjects(); + for (auto* o : objs) scene->destroyMovableObject(o); + scene->destroySceneNode(n); + } + } + m_markerNodes.clear(); +} + +void FaceRigController::refreshMarkerOverlays() +{ + clearMarkerOverlays(); + auto* mgr = Manager::getSingletonPtr(); + Ogre::SceneManager* scene = mgr ? mgr->getSceneMgr() : nullptr; + if (!scene) return; + Ogre::Entity* entity = nullptr; + for (Ogre::Entity* e : mgr->getEntities()) + if (e && e->getMovableType() == "Entity" + && e->getName() == m_markerEntityName) { entity = e; break; } + if (!entity) return; + Ogre::Node* node = entity->getParentSceneNode(); + + auto& mm = Ogre::MaterialManager::getSingleton(); + auto ensureMat = [&](const std::string& n, const Ogre::ColourValue& c) { + if (!mm.resourceExists(n)) { + auto mat = mm.create(n, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + auto* pass = mat->getTechnique(0)->getPass(0); + pass->setLightingEnabled(false); + pass->setDiffuse(c); pass->setAmbient(c); + pass->setDepthCheckEnabled(false); + } + }; + ensureMat("__FaceMarkerMat__", Ogre::ColourValue(1.0f, 0.85f, 0.1f, 1.0f)); // placed + ensureMat("__FaceMarkerMatSel__", Ogre::ColourValue(0.2f, 0.9f, 1.0f, 1.0f)); // selected + ensureMat("__FaceMarkerMatUnset__", Ogre::ColourValue(0.6f, 0.6f, 0.6f, 1.0f)); // default/unplaced + + const Ogre::Real r = entity->getBoundingRadius() * 0.02f; + for (int i = 0; i < int(m_markers.size()); ++i) { + const auto& m = m_markers[size_t(i)]; + const Ogre::Vector3 localPos(m.userPos[0], m.userPos[1], m.userPos[2]); + const Ogre::Vector3 worldPos = node ? node->_getFullTransform()*localPos : localPos; + Ogre::SceneNode* sn = scene->getRootSceneNode()->createChildSceneNode(); + Ogre::Entity* sphere = nullptr; + try { sphere = scene->createEntity(Ogre::SceneManager::PT_SPHERE); } catch (...) {} + if (sphere) { + sphere->setMaterialName(i == m_selMarker ? "__FaceMarkerMatSel__" + : m.placed ? "__FaceMarkerMat__" + : "__FaceMarkerMatUnset__"); + sphere->setRenderQueueGroup(Ogre::RENDER_QUEUE_OVERLAY - 1); + sn->attachObject(sphere); + const Ogre::Real s = (r > 1e-4f ? r : 0.02f) / 100.0f + * (i == m_selMarker ? 1.5f : 1.0f); + sn->setScale(s, s, s); + } + sn->setPosition(worldPos); + m_markerNodes.push_back(sn); + } +} diff --git a/src/FaceRigController.h b/src/FaceRigController.h index 1f4a70594..2284d005b 100644 --- a/src/FaceRigController.h +++ b/src/FaceRigController.h @@ -1,12 +1,20 @@ #ifndef FACE_RIG_CONTROLLER_H #define FACE_RIG_CONTROLLER_H +#include "FaceRig/FaceRigLandmarks.h" // FaceMarker, NricpLandmark + #include #include +#include #include #include #include +#include +#include + +namespace Ogre { class SceneNode; } +namespace FaceRig { class ArkitTemplate; struct FaceRigGeometry; } // QML-facing singleton for the face auto-rig (#889, Slice F #895). Wraps // FaceRig::attachFaceRig* plus selection state so the Inspector's "Add ARKit @@ -34,6 +42,12 @@ class FaceRigController : public QObject // Worker progress for a progress bar: done / total steps (0 total = idle). Q_PROPERTY(int progress READ progress NOTIFY progressChanged) Q_PROPERTY(int progressTotal READ progressTotal NOTIFY progressChanged) + // Face-marker editing session (auto-seed → user adjusts → rig). + Q_PROPERTY(bool markerMode READ markerMode NOTIFY markerModeChanged) + Q_PROPERTY(QStringList markerLabels READ markerLabels NOTIFY markersChanged) + Q_PROPERTY(int selectedMarker READ selectedMarker NOTIFY markersChanged) + Q_PROPERTY(bool markersSeededFromDetection READ markersSeededFromDetection + NOTIFY markersChanged) public: static FaceRigController* instance(); @@ -60,11 +74,44 @@ class FaceRigController : public QObject /// stops at the next progress step; `error("cancelled")` follows. Q_INVOKABLE void cancel(); + // ---- Face-marker editing (auto-seed, user adjusts, then rig) -------- + bool markerMode() const { return m_markerMode; } + QStringList markerLabels() const; + int selectedMarker() const { return m_selMarker; } + bool markersSeededFromDetection() const { return m_seededConfident; } + + /// Enter marker mode on the selected face mesh: loads the template, seeds + /// the markers (auto-detect when it works, sensible defaults otherwise), + /// and shows draggable overlays. Emits markersChanged / error. Runs the + /// (main-thread) detection renders inline — quick for a head. + Q_INVOKABLE bool beginFaceMarkers(); + /// Select a marker (index into markerLabels) to reposition on the next + /// mesh click. -1 = none. + Q_INVOKABLE void selectMarker(int index); + /// Leave marker mode, discarding overlays (does NOT rig). + Q_INVOKABLE void cancelFaceMarkers(); + /// Whether a marker is currently "placed" (has a position). QML dims the + /// unplaced ones. + Q_INVOKABLE bool markerPlaced(int index) const; + + /// Commit: build NRICP anchors from the (edited) markers and run the rig + /// (worker thread), same flow as addArkitBlendshapesAsync but anchored to + /// the user-corrected markers. Returns false if it couldn't start. + Q_INVOKABLE bool rigFromMarkers(int maxShapes = 0, double maxResidualPct = 8.0); + + /// Called by TransformOperator on a viewport left-click while markerMode is + /// active: ray-casts to the mesh surface and moves the SELECTED marker + /// there (or picks the nearest marker if none selected). Returns true if + /// the click was consumed. + bool handleMarkerClick(class OgreWidget* widget, const QPoint& screenPos); + signals: void selectionChanged(); void busyChanged(); void statusChanged(); void progressChanged(); + void markerModeChanged(); + void markersChanged(); void completed(const QVariantMap& report); void error(const QString& message); @@ -74,6 +121,16 @@ class FaceRigController : public QObject void setStatus(const QString& s); + // Shared async rig runner: extract geometry + template already loaded on the + // main thread; `anchors` are the landmark constraints (auto or marker-based, + // possibly empty). Runs the fit on a worker, attaches on the main thread. + bool runRigAsync(const std::shared_ptr& tmpl, + int maxShapes, double maxResidualPct, + const std::vector& anchors); + + void clearMarkerOverlays(); + void refreshMarkerOverlays(); + static FaceRigController* m_pSingleton; bool m_busy = false; bool m_downloading = false; @@ -81,6 +138,17 @@ class FaceRigController : public QObject int m_progress = 0; int m_progressTotal = 0; std::shared_ptr m_cancel; + + // Marker session. + bool m_markerMode = false; + int m_selMarker = -1; + bool m_seededConfident = false; + std::string m_markerEntityName; + std::vector m_markers; + std::vector m_markerNodes; + std::shared_ptr m_markerTmpl; + // Geometry extracted on the main thread, handed to the worker rig run. + std::shared_ptr m_geo; }; #endif // FACE_RIG_CONTROLLER_H diff --git a/src/TransformOperator.cpp b/src/TransformOperator.cpp index 57c11b76f..afd4ffb4c 100755 --- a/src/TransformOperator.cpp +++ b/src/TransformOperator.cpp @@ -29,6 +29,7 @@ #include "BoneDragRelease.h" #include "EditModeController.h" #include "AutoRigController.h" +#include "FaceRigController.h" #include "TexturePaintController.h" #include "AnimationControlController.h" #include "PropertiesPanelController.h" @@ -1028,6 +1029,13 @@ void TransformOperator::mousePressEvent(QMouseEvent *e) AutoRigController::instance()->handleMarkerClick(m_pActiveWidget, e->pos()); return; } + // Face-rig marker editing (auto-seeded, user-adjusted) — same priority: + // a left-click repositions the selected face marker on the mesh surface. + if (FaceRigController::instance()->markerMode()) + { + FaceRigController::instance()->handleMarkerClick(m_pActiveWidget, e->pos()); + return; + } auto* editCtrl = EditModeController::instance(); From f5a5b852cd212a004b2c310653273e74c6d9371f Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 15 Jul 2026 22:36:28 -0400 Subject: [PATCH 13/38] =?UTF-8?q?feat(#889):=20marker-driven=20RBF=20pre-w?= =?UTF-8?q?arp=20=E2=80=94=20markers=20now=20DRIVE=20the=20fit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Even with all markers set, transferred shapes smeared (jawOpen dragged the whole lower face down instead of hinging the jaw): markers were only SOFT constraints inside NRICP, so on faces far from the template's proportions (cartoon jowls/chins) the un-anchored regions still slid to a wrong low-residual correspondence, and the resample smeared the deltas. Fix: rbfWarpByAnchors() — a thin-plate RBF space warp (φ(r)=r + affine, small dense solve, pure data) driven by the marker pairs. buildFaceRig pre-warps the WHOLE template into the user's face proportions before the fit, so each marked feature (mouth corners, chin, eyes, nose) STARTS exactly on its marked position and the space between interpolates smoothly; NRICP then only refines locally. Applied only for small curated anchor sets (≤32 — the marker path); bulk auto-detected sets are excluded (a garbage detection would fold the template). DeformationTransfer still uses the ORIGINAL template neutral as its source rest — the fit output stays the same correspondence semantics. Unit-tested: anchors land exactly, a pure-translation anchor field translates every vertex (thin-plate is affine-exact), <4 anchors → empty (caller falls back to the unwarped fit). Full FaceRig suite green (19 tests). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/FaceRig/FaceRigger.cpp | 123 +++++++++++++++++++++++++++++++- src/FaceRig/FaceRigger.h | 12 ++++ src/FaceRig/FaceRigger_test.cpp | 37 ++++++++++ 3 files changed, 169 insertions(+), 3 deletions(-) diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index 18b3a3bb9..2a90beaa7 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -124,6 +124,109 @@ double bboxDiag(const std::vector& v) } // namespace +std::vector rbfWarpByAnchors(const std::vector& tmplV, + const std::vector& anchors) +{ + const int nv = int(tmplV.size() / 3); + // Collect valid, de-duplicated centers + displacements (two anchors on the + // same template vertex would make the system singular — first one wins). + std::vector cs; // template vertex per center + std::vector> C, D; // center pos, displacement + for (const auto& a : anchors) { + if (a.tmplVertex < 0 || a.tmplVertex >= nv) continue; + bool dup = false; + for (int c : cs) if (c == a.tmplVertex) { dup = true; break; } + if (dup) continue; + cs.push_back(a.tmplVertex); + std::array c{}, d{}; + for (int k = 0; k < 3; ++k) { + c[size_t(k)] = tmplV[size_t(a.tmplVertex)*3 + k]; + d[size_t(k)] = double(a.target[size_t(k)]) - c[size_t(k)]; + } + C.push_back(c); + D.push_back(d); + } + const int N = int(cs.size()); + if (N < 4) return {}; + + // System: [Φ P; Pᵀ 0] [w; a] = [D; 0], φ(r)=r, P = [1 x y z]. + const int M = N + 4; + std::vector A(size_t(M)*M, 0.0); + for (int i = 0; i < N; ++i) { + for (int j = 0; j < N; ++j) { + const double dx = C[size_t(i)][0]-C[size_t(j)][0]; + const double dy = C[size_t(i)][1]-C[size_t(j)][1]; + const double dz = C[size_t(i)][2]-C[size_t(j)][2]; + A[size_t(i)*M + j] = std::sqrt(dx*dx + dy*dy + dz*dz); + } + A[size_t(i)*M + N + 0] = 1.0; + A[size_t(i)*M + N + 1] = C[size_t(i)][0]; + A[size_t(i)*M + N + 2] = C[size_t(i)][1]; + A[size_t(i)*M + N + 3] = C[size_t(i)][2]; + A[size_t(N+0)*M + i] = 1.0; + A[size_t(N+1)*M + i] = C[size_t(i)][0]; + A[size_t(N+2)*M + i] = C[size_t(i)][1]; + A[size_t(N+3)*M + i] = C[size_t(i)][2]; + } + + // Solve the 3 rhs (one per axis) with one LU-style elimination. Partial + // pivoting; bail on a near-singular pivot (degenerate anchor layout). + std::vector> rhs(size_t(M), {0,0,0}); + for (int i = 0; i < N; ++i) rhs[size_t(i)] = D[size_t(i)]; + for (int col = 0; col < M; ++col) { + int piv = col; + for (int r = col+1; r < M; ++r) + if (std::abs(A[size_t(r)*M+col]) > std::abs(A[size_t(piv)*M+col])) + piv = r; + if (std::abs(A[size_t(piv)*M+col]) < 1e-10) return {}; + if (piv != col) { + for (int c = 0; c < M; ++c) + std::swap(A[size_t(piv)*M+c], A[size_t(col)*M+c]); + std::swap(rhs[size_t(piv)], rhs[size_t(col)]); + } + const double p = A[size_t(col)*M+col]; + for (int r = col+1; r < M; ++r) { + const double f = A[size_t(r)*M+col] / p; + if (f == 0.0) continue; + for (int c = col; c < M; ++c) + A[size_t(r)*M+c] -= f * A[size_t(col)*M+c]; + for (int d = 0; d < 3; ++d) + rhs[size_t(r)][size_t(d)] -= f * rhs[size_t(col)][size_t(d)]; + } + } + std::vector> sol(size_t(M), {0,0,0}); + for (int r = M-1; r >= 0; --r) { + std::array acc = rhs[size_t(r)]; + for (int c = r+1; c < M; ++c) + for (int d = 0; d < 3; ++d) + acc[size_t(d)] -= A[size_t(r)*M+c] * sol[size_t(c)][size_t(d)]; + const double p = A[size_t(r)*M+r]; + for (int d = 0; d < 3; ++d) sol[size_t(r)][size_t(d)] = acc[size_t(d)] / p; + } + + // Warp every template vertex: p' = p + Σ w_i φ(|p-c_i|) + a0 + a1x + a2y + a3z + std::vector out(tmplV.size()); + for (int v = 0; v < nv; ++v) { + const double px = tmplV[size_t(v)*3], py = tmplV[size_t(v)*3+1], + pz = tmplV[size_t(v)*3+2]; + std::array disp = { + sol[size_t(N+0)][0] + sol[size_t(N+1)][0]*px + sol[size_t(N+2)][0]*py + sol[size_t(N+3)][0]*pz, + sol[size_t(N+0)][1] + sol[size_t(N+1)][1]*px + sol[size_t(N+2)][1]*py + sol[size_t(N+3)][1]*pz, + sol[size_t(N+0)][2] + sol[size_t(N+1)][2]*px + sol[size_t(N+2)][2]*py + sol[size_t(N+3)][2]*pz }; + for (int i = 0; i < N; ++i) { + const double dx = px-C[size_t(i)][0], dy = py-C[size_t(i)][1], + dz = pz-C[size_t(i)][2]; + const double r = std::sqrt(dx*dx + dy*dy + dz*dz); + for (int d = 0; d < 3; ++d) + disp[size_t(d)] += sol[size_t(i)][size_t(d)] * r; + } + out[size_t(v)*3] = float(px + disp[0]); + out[size_t(v)*3+1] = float(py + disp[1]); + out[size_t(v)*3+2] = float(pz + disp[2]); + } + return out; +} + FaceRigResult buildFaceRig(const std::vector& userV, const std::vector& userF, const ArkitTemplate& tmpl, @@ -196,10 +299,24 @@ FaceRigResult buildFaceRig(const std::vector& userV, return progress ? progress(done, total, phase) : true; }; - // 1) NRICP: template neutral → user neutral = correspondence X. Report each - // annealing level so the (long) fit phase visibly advances. + // Marker-driven RBF pre-warp: with a small, CURATED anchor set (the + // user-adjusted markers — ≤ ~16), warp the whole template into the user's + // face proportions before the fit, so the mouth/eyes/chin START on the + // marked positions and the space between interpolates smoothly. Soft + // in-fit constraints alone let un-anchored regions slide on faces far from + // the template (cartoon proportions), smearing the transferred shapes. + // Deliberately NOT applied to bulk auto-detected anchor sets (hundreds of + // points): a garbage detection would fold the template. + std::vector fitTmplV = tmpl.neutral(); + if (!landmarks.empty() && landmarks.size() <= 32) { + std::vector warped = rbfWarpByAnchors(tmpl.neutral(), landmarks); + if (!warped.empty()) fitTmplV = std::move(warped); + } + + // 1) NRICP: (pre-warped) template neutral → user neutral = correspondence X. + // Report each annealing level so the (long) fit phase visibly advances. const NricpResult fit = FaceRig::fit( - tmpl.neutral(), tmpl.faces(), fitV, fitF, fitOpts, + fitTmplV, tmpl.faces(), fitV, fitF, fitOpts, [&](int level, int /*levelCount*/) -> bool { if (!tick(level, "Fitting face template…")) { cancelled = true; return false; } return true; diff --git a/src/FaceRig/FaceRigger.h b/src/FaceRig/FaceRigger.h index 7b21b5f43..c5d462052 100644 --- a/src/FaceRig/FaceRigger.h +++ b/src/FaceRig/FaceRigger.h @@ -91,6 +91,18 @@ FaceRigResult buildFaceRig(const std::vector& userV, const std::vector& landmarks = {}, const FaceRigProgressFn& progress = {}); +// RBF (thin-plate, φ(r)=r) space warp of the template driven by the landmark +// anchors: each anchor's template vertex lands EXACTLY on its user target and +// the space between interpolates smoothly. Used to PRE-WARP the template into +// the user's face proportions before NRICP — soft in-fit constraints alone let +// the un-anchored regions slide on faces far from the template (cartoon +// proportions), which smears the transferred shapes. Returns the warped +// positions, or empty when there are too few / degenerate anchors (< 4, or a +// singular system) — the caller then fits the unwarped template as before. +// Pure data + unit-tested. +std::vector rbfWarpByAnchors(const std::vector& tmplV, + const std::vector& anchors); + } // namespace FaceRig #endif // FACERIGGER_H diff --git a/src/FaceRig/FaceRigger_test.cpp b/src/FaceRig/FaceRigger_test.cpp index bc08fb94f..3a663fbed 100644 --- a/src/FaceRig/FaceRigger_test.cpp +++ b/src/FaceRig/FaceRigger_test.cpp @@ -159,6 +159,43 @@ TEST(FaceRigger, ProducesPerUserVertexShapes) EXPECT_LT(movedCentroidY(r.shapes[1]), movedCentroidY(r.shapes[0])); } +// RBF pre-warp: anchors land exactly on their targets and the space between +// interpolates smoothly (a pure translation of all anchors translates the +// whole mesh). +TEST(FaceRigger, RbfWarpInterpolatesAnchors) +{ + const Grid g = makeGrid(8, 2.0f, 0.1f); + const int n = int(g.V.size()/3); + + // pure translation: 5 anchors all displaced by (0.3, -0.2, 0.1) + std::vector anchors; + const int picks[5] = {0, 7, n/2, n-8, n-1}; + for (int p : picks) { + FaceRig::NricpLandmark a; + a.tmplVertex = p; + a.target = {g.V[size_t(p)*3] + 0.3f, g.V[size_t(p)*3+1] - 0.2f, + g.V[size_t(p)*3+2] + 0.1f}; + anchors.push_back(a); + } + const auto warped = FaceRig::rbfWarpByAnchors(g.V, anchors); + ASSERT_EQ(warped.size(), g.V.size()); + // anchors land exactly (affine part reproduces the translation) + for (int p : picks) { + EXPECT_NEAR(warped[size_t(p)*3], g.V[size_t(p)*3] + 0.3f, 1e-3f); + EXPECT_NEAR(warped[size_t(p)*3+1], g.V[size_t(p)*3+1] - 0.2f, 1e-3f); + EXPECT_NEAR(warped[size_t(p)*3+2], g.V[size_t(p)*3+2] + 0.1f, 1e-3f); + } + // a pure-translation anchor set translates EVERY vertex (thin-plate exact + // for affine displacement fields) + for (int v = 0; v < n; ++v) { + EXPECT_NEAR(warped[size_t(v)*3], g.V[size_t(v)*3] + 0.3f, 1e-2f); + EXPECT_NEAR(warped[size_t(v)*3+1], g.V[size_t(v)*3+1] - 0.2f, 1e-2f); + } + // too few anchors → empty (caller falls back to the unwarped template) + anchors.resize(3); + EXPECT_TRUE(FaceRig::rbfWarpByAnchors(g.V, anchors).empty()); +} + TEST(FaceRigger, RejectsNonFaceMesh) { // template = bumpy plane with a shape From 75a881fbcb56a22504ea1d26a6ec3bac60927f63 Mon Sep 17 00:00:00 2001 From: Fernando Date: Wed, 15 Jul 2026 22:58:56 -0400 Subject: [PATCH 14/38] fix(#889): well-conditioned marker warp + re-rig replaces instead of stacking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field report: after the marker rig, NO slider moved anything. Two root causes: 1) Ill-conditioned warp — face markers are nearly COPLANAR (all on the front of the face), so the thin-plate RBF's affine term was unconstrained along the depth axis and could shear the template's unmarked regions (back of the head) into garbage, crumpling the fit. rbfWarpByAnchors is now a SIMILARITY prealign (centroid + RMS-spread scale — robust for coplanar sets) plus a ridge-regularized GAUSSIAN RBF on the residuals, whose influence DECAYS away from the face — far vertices get the similarity only, so the warp physically cannot explode. sigma = mean nearest-neighbour marker spacing x1.5. 2) Re-rig stacked instead of replacing — attaching over an already-rigged mesh ADDED duplicate same-named poses into the existing animations, and VAT_POSE keyframes reference poses BY INDEX, so the old references corrupt — the "sliders do nothing" state. The controller now DELETES existing same-named targets first (inside the same undo macro, atomic undo/redo). Also: marker spheres now actually show their state colours (self-illumination — with lighting disabled Ogre ignores diffuse/ambient, so they rendered plain white), and the rig logs anchors/replaced/attached counts for diagnosis. CLI auto path regression-checked (51 shapes @ 0.144% on Rumba); FaceRig suite green (19 tests) incl. the reworked warp test. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/FaceRig/FaceRigger.cpp | 171 ++++++++++++++++++++++++------------- src/FaceRigController.cpp | 35 +++++++- 2 files changed, 143 insertions(+), 63 deletions(-) diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index 2a90beaa7..47074bdc9 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -128,101 +128,150 @@ std::vector rbfWarpByAnchors(const std::vector& tmplV, const std::vector& anchors) { const int nv = int(tmplV.size() / 3); - // Collect valid, de-duplicated centers + displacements (two anchors on the - // same template vertex would make the system singular — first one wins). - std::vector cs; // template vertex per center - std::vector> C, D; // center pos, displacement + // Collect valid, de-duplicated centers + targets (two anchors on the same + // template vertex would make the system singular - first one wins). + std::vector cs; + std::vector> C, T; // template center, user target for (const auto& a : anchors) { if (a.tmplVertex < 0 || a.tmplVertex >= nv) continue; bool dup = false; for (int c : cs) if (c == a.tmplVertex) { dup = true; break; } if (dup) continue; cs.push_back(a.tmplVertex); - std::array c{}, d{}; - for (int k = 0; k < 3; ++k) { - c[size_t(k)] = tmplV[size_t(a.tmplVertex)*3 + k]; - d[size_t(k)] = double(a.target[size_t(k)]) - c[size_t(k)]; - } - C.push_back(c); - D.push_back(d); + C.push_back({tmplV[size_t(a.tmplVertex)*3], + tmplV[size_t(a.tmplVertex)*3+1], + tmplV[size_t(a.tmplVertex)*3+2]}); + T.push_back({double(a.target[0]), double(a.target[1]), + double(a.target[2])}); } const int N = int(cs.size()); if (N < 4) return {}; - // System: [Φ P; Pᵀ 0] [w; a] = [D; 0], φ(r)=r, P = [1 x y z]. - const int M = N + 4; - std::vector A(size_t(M)*M, 0.0); + // 1) SIMILARITY prealign (centroid + RMS-spread scale, no rotation - both + // faces are upright/front-facing by contract). Face markers are nearly + // COPLANAR, so a full affine/thin-plate warp is ill-conditioned along the + // depth axis and can shear the back of the head into garbage - the + // similarity handles the global part robustly, the Gaussian RBF below only + // carries the local residuals and DECAYS away from the face. + std::array cT{0,0,0}, cU{0,0,0}; + for (int i = 0; i < N; ++i) + for (int d = 0; d < 3; ++d) { + cT[size_t(d)] += C[size_t(i)][size_t(d)] / N; + cU[size_t(d)] += T[size_t(i)][size_t(d)] / N; + } + double sT = 0, sU = 0; + for (int i = 0; i < N; ++i) { + double dt = 0, du = 0; + for (int d = 0; d < 3; ++d) { + const double a = C[size_t(i)][size_t(d)] - cT[size_t(d)]; + const double b = T[size_t(i)][size_t(d)] - cU[size_t(d)]; + dt += a*a; du += b*b; + } + sT += std::sqrt(dt); sU += std::sqrt(du); + } + if (sT < 1e-9) return {}; + const double scale = (sU > 1e-9) ? sU / sT : 1.0; + auto prealign = [&](const std::array& p) { + std::array q; + for (int d = 0; d < 3; ++d) + q[size_t(d)] = (p[size_t(d)] - cT[size_t(d)]) * scale + cU[size_t(d)]; + return q; + }; + + // Prealigned centers + residual displacements the RBF must carry. + std::vector> Cp(size_t(N), {0,0,0}); + std::vector> R(size_t(N), {0,0,0}); for (int i = 0; i < N; ++i) { + Cp[size_t(i)] = prealign(C[size_t(i)]); + for (int d = 0; d < 3; ++d) + R[size_t(i)][size_t(d)] = + T[size_t(i)][size_t(d)] - Cp[size_t(i)][size_t(d)]; + } + + // 2) Gaussian RBF on the residuals, ridge-regularized. sigma = mean + // nearest-neighbour center spacing (x1.5) so influence blobs overlap + // smoothly; far from the face the displacement decays to the similarity. + double sigma = 0; + for (int i = 0; i < N; ++i) { + double best = 1e30; for (int j = 0; j < N; ++j) { - const double dx = C[size_t(i)][0]-C[size_t(j)][0]; - const double dy = C[size_t(i)][1]-C[size_t(j)][1]; - const double dz = C[size_t(i)][2]-C[size_t(j)][2]; - A[size_t(i)*M + j] = std::sqrt(dx*dx + dy*dy + dz*dz); + if (i == j) continue; + double d2 = 0; + for (int d = 0; d < 3; ++d) { + const double dd = Cp[size_t(i)][size_t(d)] - Cp[size_t(j)][size_t(d)]; + d2 += dd*dd; + } + best = std::min(best, d2); } - A[size_t(i)*M + N + 0] = 1.0; - A[size_t(i)*M + N + 1] = C[size_t(i)][0]; - A[size_t(i)*M + N + 2] = C[size_t(i)][1]; - A[size_t(i)*M + N + 3] = C[size_t(i)][2]; - A[size_t(N+0)*M + i] = 1.0; - A[size_t(N+1)*M + i] = C[size_t(i)][0]; - A[size_t(N+2)*M + i] = C[size_t(i)][1]; - A[size_t(N+3)*M + i] = C[size_t(i)][2]; + sigma += std::sqrt(best) / N; } + sigma *= 1.5; + if (sigma < 1e-9) return {}; + const double inv2s2 = 1.0 / (2.0 * sigma * sigma); + const double ridge = 1e-3; - // Solve the 3 rhs (one per axis) with one LU-style elimination. Partial - // pivoting; bail on a near-singular pivot (degenerate anchor layout). - std::vector> rhs(size_t(M), {0,0,0}); - for (int i = 0; i < N; ++i) rhs[size_t(i)] = D[size_t(i)]; - for (int col = 0; col < M; ++col) { + // Solve (A + ridge*I) w = R for the 3 axes with Gaussian elimination. + std::vector A(size_t(N)*N, 0.0); + for (int i = 0; i < N; ++i) + for (int j = 0; j < N; ++j) { + double d2 = 0; + for (int d = 0; d < 3; ++d) { + const double dd = Cp[size_t(i)][size_t(d)] - Cp[size_t(j)][size_t(d)]; + d2 += dd*dd; + } + A[size_t(i)*N + j] = std::exp(-d2 * inv2s2) + (i == j ? ridge : 0.0); + } + std::vector> rhs = R; + for (int col = 0; col < N; ++col) { int piv = col; - for (int r = col+1; r < M; ++r) - if (std::abs(A[size_t(r)*M+col]) > std::abs(A[size_t(piv)*M+col])) + for (int r = col+1; r < N; ++r) + if (std::abs(A[size_t(r)*N+col]) > std::abs(A[size_t(piv)*N+col])) piv = r; - if (std::abs(A[size_t(piv)*M+col]) < 1e-10) return {}; + if (std::abs(A[size_t(piv)*N+col]) < 1e-12) return {}; if (piv != col) { - for (int c = 0; c < M; ++c) - std::swap(A[size_t(piv)*M+c], A[size_t(col)*M+c]); + for (int c = 0; c < N; ++c) + std::swap(A[size_t(piv)*N+c], A[size_t(col)*N+c]); std::swap(rhs[size_t(piv)], rhs[size_t(col)]); } - const double p = A[size_t(col)*M+col]; - for (int r = col+1; r < M; ++r) { - const double f = A[size_t(r)*M+col] / p; + const double p = A[size_t(col)*N+col]; + for (int r = col+1; r < N; ++r) { + const double f = A[size_t(r)*N+col] / p; if (f == 0.0) continue; - for (int c = col; c < M; ++c) - A[size_t(r)*M+c] -= f * A[size_t(col)*M+c]; + for (int c = col; c < N; ++c) + A[size_t(r)*N+c] -= f * A[size_t(col)*N+c]; for (int d = 0; d < 3; ++d) rhs[size_t(r)][size_t(d)] -= f * rhs[size_t(col)][size_t(d)]; } } - std::vector> sol(size_t(M), {0,0,0}); - for (int r = M-1; r >= 0; --r) { + std::vector> w(size_t(N), {0,0,0}); + for (int r = N-1; r >= 0; --r) { std::array acc = rhs[size_t(r)]; - for (int c = r+1; c < M; ++c) + for (int c = r+1; c < N; ++c) for (int d = 0; d < 3; ++d) - acc[size_t(d)] -= A[size_t(r)*M+c] * sol[size_t(c)][size_t(d)]; - const double p = A[size_t(r)*M+r]; - for (int d = 0; d < 3; ++d) sol[size_t(r)][size_t(d)] = acc[size_t(d)] / p; + acc[size_t(d)] -= A[size_t(r)*N+c] * w[size_t(c)][size_t(d)]; + for (int d = 0; d < 3; ++d) + w[size_t(r)][size_t(d)] = acc[size_t(d)] / A[size_t(r)*N+r]; } - // Warp every template vertex: p' = p + Σ w_i φ(|p-c_i|) + a0 + a1x + a2y + a3z + // Warp every template vertex: similarity, then the decaying residual field. std::vector out(tmplV.size()); for (int v = 0; v < nv; ++v) { - const double px = tmplV[size_t(v)*3], py = tmplV[size_t(v)*3+1], - pz = tmplV[size_t(v)*3+2]; - std::array disp = { - sol[size_t(N+0)][0] + sol[size_t(N+1)][0]*px + sol[size_t(N+2)][0]*py + sol[size_t(N+3)][0]*pz, - sol[size_t(N+0)][1] + sol[size_t(N+1)][1]*px + sol[size_t(N+2)][1]*py + sol[size_t(N+3)][1]*pz, - sol[size_t(N+0)][2] + sol[size_t(N+1)][2]*px + sol[size_t(N+2)][2]*py + sol[size_t(N+3)][2]*pz }; + const std::array p = prealign( + {tmplV[size_t(v)*3], tmplV[size_t(v)*3+1], tmplV[size_t(v)*3+2]}); + std::array disp{0,0,0}; for (int i = 0; i < N; ++i) { - const double dx = px-C[size_t(i)][0], dy = py-C[size_t(i)][1], - dz = pz-C[size_t(i)][2]; - const double r = std::sqrt(dx*dx + dy*dy + dz*dz); + double d2 = 0; + for (int d = 0; d < 3; ++d) { + const double dd = p[size_t(d)] - Cp[size_t(i)][size_t(d)]; + d2 += dd*dd; + } + const double phi = std::exp(-d2 * inv2s2); for (int d = 0; d < 3; ++d) - disp[size_t(d)] += sol[size_t(i)][size_t(d)] * r; + disp[size_t(d)] += w[size_t(i)][size_t(d)] * phi; } - out[size_t(v)*3] = float(px + disp[0]); - out[size_t(v)*3+1] = float(py + disp[1]); - out[size_t(v)*3+2] = float(pz + disp[2]); + out[size_t(v)*3] = float(p[0] + disp[0]); + out[size_t(v)*3+1] = float(p[1] + disp[1]); + out[size_t(v)*3+2] = float(p[2] + disp[2]); } return out; } diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 003e7b7f6..7298d44ad 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -22,6 +22,7 @@ #include #include +#include #include FaceRigController* FaceRigController::m_pSingleton = nullptr; @@ -278,11 +279,31 @@ bool FaceRigController::runRigAsync( for (size_t ci = 0; ci + 1 < cmds.size(); ++ci) cmds[ci]->setDeferInit(true); // all but the last defer re-init + // Re-rig = REPLACE, not stack: VAT_POSE keyframes reference poses + // BY INDEX, so attaching a second same-named set on an already- + // rigged mesh both duplicates poses AND corrupts the existing + // keyframe references — the "sliders do nothing" failure. Delete + // the old same-named targets first, inside the same macro so + // undo/redo stays atomic. + std::set existing; + if (auto mesh = entity->getMesh()) + for (const Ogre::Pose* p : mesh->getPoseList()) + if (p) existing.insert(p->getName()); + std::vector dels; + for (const FaceRig::FaceRigShape& shape : result->shapes) + if (existing.count(shape.name.toStdString())) + dels.push_back(new DeleteMorphTargetCommand(entity, shape.name)); + auto* undo = UndoManager::getSingleton(); auto* stack = undo ? undo->stack() : nullptr; if (stack) stack->beginMacro(QStringLiteral("Add ARKit Blendshapes")); + for (auto* del : dels) undo->push(del); for (auto* cmd : cmds) { undo->push(cmd); rep.shapesAttached++; } if (stack) stack->endMacro(); + qWarning("[facerig] rig: replaced %zu existing, attached %d shapes " + "(fit mean %.3f%% max %.3f%%)", + dels.size(), rep.shapesAttached, + rep.fitMeanResidualPct, rep.fitMaxResidualPct); if (enode) entity->setVisible(wasVisible); // Restore the animation states we disabled (refreshAvailable... in @@ -494,6 +515,10 @@ bool FaceRigController::rigFromMarkers(int maxShapes, double maxResidualPct) if (!m_markerMode) { emit error(QStringLiteral("Not in marker mode.")); return false; } auto tmpl = m_markerTmpl; const auto anchors = FaceRig::anchorsFromMarkers(m_markers); + int placedCount = 0; + for (const auto& m : m_markers) placedCount += m.placed ? 1 : 0; + qWarning("[facerig] rigFromMarkers: %d/%zu markers placed -> %zu anchors", + placedCount, m_markers.size(), anchors.size()); // Leave marker mode (clears overlays) before the rig runs. m_markerMode = false; m_selMarker = -1; @@ -537,8 +562,14 @@ void FaceRigController::refreshMarkerOverlays() if (!mm.resourceExists(n)) { auto mat = mm.create(n, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); auto* pass = mat->getTechnique(0)->getPass(0); - pass->setLightingEnabled(false); - pass->setDiffuse(c); pass->setAmbient(c); + // Lighting ON + self-illumination is what actually colours the + // sphere — with lighting disabled Ogre ignores diffuse/ambient and + // the markers render plain white (indistinguishable states). + pass->setLightingEnabled(true); + pass->setSelfIllumination(c); + pass->setDiffuse(Ogre::ColourValue::Black); + pass->setAmbient(Ogre::ColourValue::Black); + pass->setSpecular(Ogre::ColourValue::Black); pass->setDepthCheckEnabled(false); } }; From 5fe5ecfab6f95bb968defaf6ce2e6c3abdd8d097 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 16 Jul 2026 00:20:34 -0400 Subject: [PATCH 15/38] =?UTF-8?q?fix(#889):=20constellation=20gate=20?= =?UTF-8?q?=E2=80=94=20garbage=20landmark=20seeds=20crushed=20the=20deltas?= =?UTF-8?q?=2050x?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of "no slider does anything" after the marker rig, found by reproducing the GUI marker path headlessly (QTMESH_FACERIG_MARKER_SIM): MediaPipe's garbage detection on cartoon faces PASSED the count-based confidence check and pre-placed the markers at scattered-blob positions. Clumped garbage anchors contract the similarity scale + shred the Gaussian warp, the fit crumples, and every transferred delta collapses (measured: jawOpen maxDisp 0.001 = 0.12% of the head — invisible). With sane marker positions the SAME pipeline produces jawOpen maxDisp 0.053 over 931 verts — a clearly visible jaw open, the best result so far. - constellationResidual(): align the template marker layout onto the detected user layout with a similarity and measure the normalised residual — a real face detection agrees (<0.25), a garbage blob doesn't. - seedFaceMarkers: detection seeds are used ONLY when the constellation is consistent; otherwise EVERY marker seeds at its head-box-projected template position, placed=true — the proportional defaults measurably produce a good rig on their own (Rumba: jawOpen 0.053 without touching a marker), and the user refines from there (needed for muzzles like the fox). - buildLandmarkAnchors (auto path): same gate — a garbage anchor set is dropped entirely (unanchored fit beats a poisoned one). - CLI: QTMESH_FACERIG_MARKER_SIM={1,2} diagnostic that exercises the exact GUI marker path headlessly with per-shape delta stats (how this was found). FaceRig suite green (19 tests); CLI auto path regression-checked. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/CLIPipeline.cpp | 96 +++++++++++++++++++++-- src/FaceRig/FaceRigLandmarks.cpp | 127 ++++++++++++++++++++++++++----- 2 files changed, 197 insertions(+), 26 deletions(-) diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 68e7a7d06..733ecacf3 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -42,7 +42,9 @@ #include "SkinWeights.h" #include "SkinEvaluate.h" #include "AutoRig.h" +#include "FaceRig/ArkitTemplate.h" #include "FaceRig/FaceRigAttach.h" +#include "FaceRig/FaceRigLandmarks.h" #include "ImageTo3D/MeshGenPredictor.h" #include "ImageTo3D/TripoSGPredictor.h" #include "ImageTo3D/MeshGenBuilder.h" @@ -9540,11 +9542,95 @@ int CLIPipeline::cmdFaceRig(int argc, char* argv[]) } Ogre::Entity* entity = meshEntities.first(); - const FaceRig::AttachReport rep = - FaceRig::attachFaceRigWithBundledTemplate(entity, opts); - if (!rep.ok) { - err() << "Error: face-rig failed — " << rep.error << Qt::endl; - return 1; + FaceRig::AttachReport rep; + if (qEnvironmentVariableIntValue("QTMESH_FACERIG_MARKER_SIM")) { + // Diagnostic: exercise the GUI's MARKER path headlessly — seed the + // markers, force-place them at their seeded/default positions, and rig + // from those anchors (RBF warp + anchored fit), printing per-shape + // delta stats. Mirrors FaceRigController::rigFromMarkers. + const QString tpath = FaceRig::ArkitTemplate::ensureModelBlocking(); + FaceRig::ArkitTemplate tmpl; + QString terr; + if (tpath.isEmpty() || !tmpl.load(tpath, &terr)) { + err() << "Error: template unavailable: " << terr << Qt::endl; + return 1; + } + FaceRig::FaceRigGeometry geo = FaceRig::extractGeometry(entity); + std::vector headV; std::vector headF; + FaceRig::headSubmesh(geo, headV, headF); + bool confident = false; + auto markers = FaceRig::seedFaceMarkers(entity, headV, headF, tmpl, &confident); + if (qEnvironmentVariableIntValue("QTMESH_FACERIG_MARKER_SIM") == 2) { + // sim mode 2: ignore the detection seeds and place every marker at + // its head-box-projected template position — approximates a user + // placing markers carefully on a proportional face. + float lo[3]={1e30f,1e30f,1e30f}, hi[3]={-1e30f,-1e30f,-1e30f}; + const int unv = int(headV.size()/3); + for (int i = 0; i < unv; ++i) + for (int a = 0; a < 3; ++a) { + lo[a]=std::min(lo[a],headV[size_t(i)*3+a]); + hi[a]=std::max(hi[a],headV[size_t(i)*3+a]); + } + const auto& tn = tmpl.neutral(); + float tlo[3]={1e30f,1e30f,1e30f}, thi[3]={-1e30f,-1e30f,-1e30f}; + for (int i = 0; i < tmpl.vertexCount(); ++i) + for (int a = 0; a < 3; ++a) { + tlo[a]=std::min(tlo[a],tn[size_t(i)*3+a]); + thi[a]=std::max(thi[a],tn[size_t(i)*3+a]); + } + for (auto& m : markers) { + if (m.tmplVertex < 0) continue; + for (int a = 0; a < 3; ++a) { + const float tv = tn[size_t(m.tmplVertex)*3+a]; + const float f = (thi[a]-tlo[a])>1e-6f ? (tv-tlo[a])/(thi[a]-tlo[a]) : 0.5f; + m.userPos[size_t(a)] = lo[a] + f*(hi[a]-lo[a]); + } + } + } + for (auto& m : markers) m.placed = true; // simulate the user placing all + const auto anchors = FaceRig::anchorsFromMarkers(markers); + err() << "[sim] markers=" << markers.size() << " anchors=" << anchors.size() + << " seedConfident=" << confident << Qt::endl; + // warp diagnostics: how big is the warped template vs the user head? + { + std::vector w = FaceRig::rbfWarpByAnchors(tmpl.neutral(), anchors); + auto diagOf = [](const std::vector& v){ + float lo[3]={1e30f,1e30f,1e30f}, hi[3]={-1e30f,-1e30f,-1e30f}; + for (size_t i=0;i+2getParentSceneNode(); diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index 175bc186b..8b94a97c5 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -246,6 +246,11 @@ Ogre::Entity* templateEntity(const std::vector& v, } // namespace +namespace { +double constellationResidual(const std::vector>& C, + const std::vector>& U); +} // namespace + std::vector buildLandmarkAnchors( Ogre::Entity* userEntity, const std::vector& userLocalV, @@ -291,11 +296,29 @@ std::vector buildLandmarkAnchors( if (best < 0) continue; anchors.push_back({best, ulm.points[size_t(i)]}); } + + // Gate on constellation consistency: MediaPipe returns a scattered garbage + // blob on cartoon/stylized faces, and anchoring the fit to garbage is worse + // than fitting unanchored. Compare the template-vs-user landmark layouts + // under a similarity — a real detection agrees, garbage doesn't. + { + std::vector> C, U; + C.reserve(anchors.size()); U.reserve(anchors.size()); + for (const auto& a : anchors) { + C.push_back({tn[size_t(a.tmplVertex)*3], + tn[size_t(a.tmplVertex)*3+1], + tn[size_t(a.tmplVertex)*3+2]}); + U.push_back(a.target); + } + const double resid = constellationResidual(C, U); #ifndef NDEBUG - if (std::getenv("QTMESH_FACERIG_DEBUG")) - std::fprintf(stderr, "[facerig] landmark anchors: tmpl.ok=%d user.ok=%d " - "anchors=%zu\n", tlm.ok, ulm.ok, anchors.size()); + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] landmark anchors: tmpl.ok=%d " + "user.ok=%d anchors=%zu residual=%.3f\n", + tlm.ok, ulm.ok, anchors.size(), resid); #endif + if (resid >= 0.25) anchors.clear(); // garbage → fit unanchored + } return anchors; } @@ -336,6 +359,50 @@ int nearestTemplateVertex(const std::vector& tn, int tvc, } return best; } + +// Does the user point constellation actually LOOK like the template's? Align +// C onto U with a similarity (centroid + RMS scale, no rotation — both faces +// upright/front by contract) and measure the mean residual normalised by U's +// spread. A real face layout agrees (≲0.2); a garbage detection (MediaPipe on +// a cartoon face returns a scattered blob) does not. This is the gate that +// keeps garbage landmarks from silently poisoning the warp/fit. +double constellationResidual(const std::vector>& C, + const std::vector>& U) +{ + const int n = int(std::min(C.size(), U.size())); + if (n < 4) return 1e9; + std::array cC{0,0,0}, cU{0,0,0}; + for (int i = 0; i < n; ++i) + for (int d = 0; d < 3; ++d) { + cC[size_t(d)] += C[size_t(i)][size_t(d)] / n; + cU[size_t(d)] += U[size_t(i)][size_t(d)] / n; + } + double sC = 0, sU = 0; + for (int i = 0; i < n; ++i) { + double dc = 0, du = 0; + for (int d = 0; d < 3; ++d) { + const double a = C[size_t(i)][size_t(d)] - cC[size_t(d)]; + const double b = U[size_t(i)][size_t(d)] - cU[size_t(d)]; + dc += a*a; du += b*b; + } + sC += std::sqrt(dc); sU += std::sqrt(du); + } + if (sC < 1e-12 || sU < 1e-12) return 1e9; + const double scale = sU / sC; + double resid = 0; + for (int i = 0; i < n; ++i) { + double d2 = 0; + for (int d = 0; d < 3; ++d) { + const double m = (C[size_t(i)][size_t(d)] - cC[size_t(d)]) * scale + + cU[size_t(d)]; + const double e = U[size_t(i)][size_t(d)] - m; + d2 += e*e; + } + resid += std::sqrt(d2); + } + resid /= n; + return resid / (sU / n); // normalise by mean spread of U +} } // namespace std::vector seedFaceMarkers( @@ -378,30 +445,51 @@ std::vector seedFaceMarkers( } } - // User detection seeds the editable positions. If it's confident, use its - // points; otherwise leave markers unplaced at a sensible default (projected - // template landmark into the user head box) for the user to drag. + // User detection seeds the editable positions — but ONLY when the detected + // constellation actually looks like a face layout. MediaPipe returns a + // scattered garbage blob on cartoon/stylized faces, and garbage seeds + // silently poison the warp/fit (measured: jawOpen deltas 50x too small). const MeshLandmarks ulm = detectMeshLandmarks(userEntity, userLocalV, userLocalF); - // Confidence heuristic: enough of the catalog markers hit the surface AND - // the landmark cloud isn't a degenerate blob. We approximate "trustworthy" - // by how many catalog markers are valid. int seeded = 0; + std::vector> detC, detU; // template/user pairs if (ulm.ok) { + for (auto& m : markers) { + const int i = m.mediapipeIndex; + if (i >= 0 && i < int(ulm.points.size()) && ulm.valid[size_t(i)] + && m.tmplVertex >= 0) { + ++seeded; + detC.push_back({tn[size_t(m.tmplVertex)*3], + tn[size_t(m.tmplVertex)*3+1], + tn[size_t(m.tmplVertex)*3+2]}); + detU.push_back(ulm.points[size_t(i)]); + } + } + } + const double resid = constellationResidual(detC, detU); + const bool confident = ulm.ok && seeded >= int(markers.size()) * 3 / 4 + && resid < 0.25; +#ifndef NDEBUG + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] seed: %d/%zu detected, constellation " + "residual %.3f -> %s\n", seeded, markers.size(), resid, + confident ? "trusted" : "using proportional defaults"); +#endif + + if (confident) { for (auto& m : markers) { const int i = m.mediapipeIndex; if (i >= 0 && i < int(ulm.points.size()) && ulm.valid[size_t(i)]) { m.userPos = ulm.points[size_t(i)]; m.placed = true; - ++seeded; } } - } - // If auto-seed was weak, place the rest at the head-box-projected template - // landmark so every marker starts SOMEWHERE on the face for the user to - // nudge (rather than at the origin). - if (seeded < int(markers.size())) { - // user head AABB + } else { + // Garbage / weak detection: seed EVERY marker at the head-box-projected + // template position instead. Those proportional defaults measurably + // produce a good rig on their own (the cartoon-face path), and the user + // refines from there. placed=true so they act as anchors even if the + // user rigs without touching them. std::array lo{1e30f,1e30f,1e30f}, hi{-1e30f,-1e30f,-1e30f}; const int unv = int(userLocalV.size()/3); for (int i = 0; i < unv; ++i) @@ -409,7 +497,6 @@ std::vector seedFaceMarkers( lo[a] = std::min(lo[a], userLocalV[size_t(i)*3+a]); hi[a] = std::max(hi[a], userLocalV[size_t(i)*3+a]); } - // template head AABB (its neutral) std::array tlo{1e30f,1e30f,1e30f}, thi{-1e30f,-1e30f,-1e30f}; for (int i = 0; i < tvc; ++i) for (int a = 0; a < 3; ++a) { @@ -417,19 +504,17 @@ std::vector seedFaceMarkers( thi[a] = std::max(thi[a], tn[size_t(i)*3+a]); } for (auto& m : markers) { - if (m.placed || m.tmplVertex < 0) continue; - // map the template vertex position into the user head box. + if (m.tmplVertex < 0) continue; for (int a = 0; a < 3; ++a) { const float tv = tn[size_t(m.tmplVertex)*3+a]; const float f = (thi[a]-tlo[a]) > 1e-6f ? (tv - tlo[a]) / (thi[a]-tlo[a]) : 0.5f; m.userPos[size_t(a)] = lo[a] + f * (hi[a]-lo[a]); } - // left unplaced: it's a default guess the user should confirm/drag. + m.placed = true; } } - const bool confident = ulm.ok && seeded >= int(markers.size()) * 3 / 4; if (outConfident) *outConfident = confident; return markers; } From cd89b1b228d867a2b630faf6e4edc517773d20d2 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 16 Jul 2026 09:01:34 -0400 Subject: [PATCH 16/38] fix(#889): strict detection gates + amplitude safety net (invisible-shapes fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field report: GUI marker rig attached shapes with amplitudes ~0.0005 (0.06% of the head — invisible) while the identical headless sim produced 0.053. Cause: the GUI render differs from headless (skybox/lighting), so MediaPipe's garbage constellation VARIES run-to-run, and the user's run scored under the 0.25 gate and got trusted — poisoning the warp/fit again. - seedFaceMarkers gate tightened to 0.12: detection must look UNAMBIGUOUSLY like a face layout to be trusted; anything else seeds the proportional defaults (measured good: jawOpen 0.053/931 verts on Rumba untouched). - buildLandmarkAnchors (auto path) gate tightened to 0.15. - Amplitude safety net in buildFaceRig: if an ANCHORED run yields max shape amplitude < 0.5% of the head diag (invisible), retry once unanchored — a plain head-isolated fit always beats an invisible one. Makes "attached but invisible" impossible regardless of where garbage sneaks in. - UI now reports the amplitudes in the success status (jawOpen amp / max amp) so a crushed rig is visible at a glance instead of masquerading as success. Sim-verified (jawOpen 0.053) + FaceRig suite green. Co-Authored-By: Claude Opus 4.8 (1M context) --- qml/PropertiesPanel.qml | 4 +++- src/FaceRig/FaceRigLandmarks.cpp | 10 ++++++++-- src/FaceRig/FaceRigger.cpp | 19 +++++++++++++++++++ src/FaceRigController.cpp | 14 ++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 50d7bbb00..25403fb21 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -9617,7 +9617,9 @@ Rectangle { function onCompleted(report) { arkitStatus.text = "✓ Attached " + report.shapesAttached + " ARKit shapes (fit " - + report.fitMeanResidualPct.toFixed(2) + "% mean)." + + report.fitMeanResidualPct.toFixed(2) + "% mean, jawOpen amp " + + report.jawOpenDisp.toFixed(4) + ", max amp " + + report.maxShapeDisp.toFixed(4) + ")." arkitStatus.color = PropertiesPanelController.textColor morphCol.targets = MorphAnimationManager.morphTargetsForSelection() || [] } diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index 8b94a97c5..e85de3e9b 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -317,7 +317,7 @@ std::vector buildLandmarkAnchors( "user.ok=%d anchors=%zu residual=%.3f\n", tlm.ok, ulm.ok, anchors.size(), resid); #endif - if (resid >= 0.25) anchors.clear(); // garbage → fit unanchored + if (resid >= 0.15) anchors.clear(); // garbage → fit unanchored } return anchors; } @@ -466,9 +466,15 @@ std::vector seedFaceMarkers( } } } + // STRICT gate (0.12): the GUI render differs from headless (skybox, + // lighting), so MediaPipe's garbage varies run-to-run and looser gates let + // some of it through (field-reproduced: a "trusted" garbage constellation + // crushed every shape to 0.06% amplitude). Detection must look UNAMBIGUOUSLY + // like a face layout to be trusted; otherwise the proportional defaults are + // measurably good and the user refines from there. const double resid = constellationResidual(detC, detU); const bool confident = ulm.ok && seeded >= int(markers.size()) * 3 / 4 - && resid < 0.25; + && resid < 0.12; #ifndef NDEBUG if (std::getenv("QTMESH_FACERIG_DEBUG")) std::fprintf(stderr, "[facerig] seed: %d/%zu detected, constellation " diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index 47074bdc9..d50f44a84 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -462,6 +462,25 @@ FaceRigResult buildFaceRig(const std::vector& userV, r.shapes.push_back(std::move(out)); } + // Amplitude safety net: a poisoned anchor set (garbage landmarks that + // slipped every gate) crushes the fit so the transferred shapes come out + // technically-attached but INVISIBLE (~0.05% of the head, vs ~5% for a + // healthy jawOpen). If the anchored run produced nothing visible, retry + // once WITHOUT anchors — a plain head-isolated fit always beats an + // invisible one. (Field-reproduced failure mode; do not remove.) + if (!landmarks.empty()) { + double maxAmp = 0; + for (const auto& sh : r.shapes) + maxAmp = std::max(maxAmp, double(sh.maxDisp)); + if (maxAmp < 0.005 * diag) { + std::fprintf(stderr, "[facerig] anchored fit produced invisible " + "shapes (max %.5f on diag %.3f) — retrying " + "unanchored\n", maxAmp, diag); + return buildFaceRig(userV, userF, tmpl, opts, headMask, {}, + progress); + } + } + r.ok = true; return r; } diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 7298d44ad..2eadff761 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -333,6 +333,20 @@ bool FaceRigController::runRigAsync( map["userVertexCount"] = rep.userVertexCount; map["fitMeanResidualPct"] = rep.fitMeanResidualPct; map["fitMaxResidualPct"] = rep.fitMaxResidualPct; + // Amplitude diagnostics: without these, "shapes attached but + // invisible" (deltas 50x too small) looks identical to success in + // the UI. jawDisp specifically because jawOpen is the shape users + // test first. + double maxDisp = 0, jawDisp = 0; + for (const auto& sh : result->shapes) { + maxDisp = std::max(maxDisp, double(sh.maxDisp)); + if (sh.name == QLatin1String("jawOpen")) + jawDisp = double(sh.maxDisp); + } + map["maxShapeDisp"] = maxDisp; + map["jawOpenDisp"] = jawDisp; + qWarning("[facerig] attached=%d jawOpenDisp=%.5f maxDisp=%.5f", + rep.shapesAttached, jawDisp, maxDisp); emit self->completed(map); }, Qt::QueuedConnection); }).detach(); From b982a093bf997de718994b3f73b82400f0159d72 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 16 Jul 2026 09:44:44 -0400 Subject: [PATCH 17/38] fix(#889): correct marker side semantics + symmetrize template anchors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The user placed "Left mouth corner" on the CHARACTER's left — correctly — but the anchor paired it with the template's RIGHT corner, a mirrored constraint that folds the template through itself and crushes the fit. Measured on the template: MediaPipe's "left"-named indices (33/133/61/105) sit at +X = character-RIGHT (MP names sides in IMAGE space, mirrored for a camera-facing subject). Also measured: MediaPipe drifts on the untextured template render (nose tip detected 4 units off the midline), skewing every template anchor. - faceMarkerCatalog relabelled to CHARACTER-space sides (what a user placing markers on a model naturally means). - seedFaceMarkers symmetrizes the detected template landmarks before resolving vertices: the ICT template is x-symmetric with midline x=0, so midline features snap to x=0 and side pairs get mirrored positions (pair-mean height/depth, mean |x|) — detection supplies only what it gets right. Verified: nose/chin/lips/forehead at x=0, eye pair at ±6.75. - anchorsFromMarkers(markers, tmpl) auto-corrects a MIRRORED placement: scores the as-placed vs left/right-swapped pairings against the template constellation and keeps the better one — either side convention works. - QML hint states the convention explicitly. Sim: default-marker rig on Rumba now uses the CORRECT template anchors (jawOpen 0.0177 over 819 verts — clearly visible). FaceRig suite green. Co-Authored-By: Claude Opus 4.8 (1M context) --- qml/PropertiesPanel.qml | 3 +- src/CLIPipeline.cpp | 10 ++- src/FaceRig/FaceRigLandmarks.cpp | 150 ++++++++++++++++++++++++++----- src/FaceRig/FaceRigLandmarks.h | 8 +- src/FaceRigController.cpp | 3 +- 5 files changed, 146 insertions(+), 28 deletions(-) diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 25403fb21..06d69e765 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -9474,7 +9474,8 @@ Rectangle { text: (FaceRigController.markersSeededFromDetection ? "Auto-detected. " : "Auto-detect was weak — ") + "Click a marker below, then click on the face to move " - + "it. Cyan = selected." + + "it. Cyan = selected. Left/Right = the CHARACTER's " + + "side (a mirrored placement is auto-corrected)." } // Marker chips — click to select which one the next mesh // click will move. diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 733ecacf3..cce03d124 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -9588,9 +9588,17 @@ int CLIPipeline::cmdFaceRig(int argc, char* argv[]) } } for (auto& m : markers) m.placed = true; // simulate the user placing all - const auto anchors = FaceRig::anchorsFromMarkers(markers); + const auto anchors = FaceRig::anchorsFromMarkers(markers, tmpl); err() << "[sim] markers=" << markers.size() << " anchors=" << anchors.size() << " seedConfident=" << confident << Qt::endl; + // side probe: character faces -Z, up +Y => character-LEFT = -X. + for (const auto& m : markers) + if (m.tmplVertex >= 0) + err() << "[sim] marker '" << m.label << "' tmplX=" + << tmpl.neutral()[size_t(m.tmplVertex)*3] + << (tmpl.neutral()[size_t(m.tmplVertex)*3] < 0 + ? " (character-LEFT side)" : " (character-RIGHT side)") + << Qt::endl; // warp diagnostics: how big is the warped template vs the user head? { std::vector w = FaceRig::rbfWarpByAnchors(tmpl.neutral(), anchors); diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index e85de3e9b..1b8a12773 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -323,28 +323,55 @@ std::vector buildLandmarkAnchors( } // Canonical MediaPipe FaceMesh indices for the few anatomical anchors the fit -// needs. These indices are stable in the FaceMesh topology (documented in the -// MediaPipe canonical face model). +// needs. IMPORTANT side semantics: MediaPipe names sides in IMAGE space, which +// is MIRRORED for a camera-facing subject — MP "left" indices (33/133/61/105) +// are the CHARACTER'S RIGHT (measured on the template: they sit at +X = +// character-right). Labels here are in CHARACTER space (what a user placing +// markers on a model naturally means by left/right). const std::vector>& faceMarkerCatalog() { static const std::vector> kCatalog = { - {QStringLiteral("Nose tip"), 1}, - {QStringLiteral("Chin"), 152}, - {QStringLiteral("Left eye outer"), 33}, - {QStringLiteral("Right eye outer"), 263}, - {QStringLiteral("Left eye inner"), 133}, - {QStringLiteral("Right eye inner"), 362}, - {QStringLiteral("Left mouth corner"), 61}, - {QStringLiteral("Right mouth corner"),291}, - {QStringLiteral("Upper lip"), 13}, - {QStringLiteral("Lower lip"), 14}, - {QStringLiteral("Left brow"), 105}, - {QStringLiteral("Right brow"), 334}, - {QStringLiteral("Forehead"), 10}, + {QStringLiteral("Nose tip"), 1}, + {QStringLiteral("Chin"), 152}, + {QStringLiteral("Right eye outer"), 33}, + {QStringLiteral("Left eye outer"), 263}, + {QStringLiteral("Right eye inner"), 133}, + {QStringLiteral("Left eye inner"), 362}, + {QStringLiteral("Right mouth corner"), 61}, + {QStringLiteral("Left mouth corner"), 291}, + {QStringLiteral("Upper lip"), 13}, + {QStringLiteral("Lower lip"), 14}, + {QStringLiteral("Right brow"), 105}, + {QStringLiteral("Left brow"), 334}, + {QStringLiteral("Forehead"), 10}, }; return kCatalog; } +namespace { +// Side-pair / midline structure of the catalog (MediaPipe indices). Used to +// SYMMETRIZE the template-side anchors (the template is x-symmetric with the +// midline at x=0, but MediaPipe drifts on the untextured template render — +// measured: "nose tip" detected 4 units off the midline) and to auto-correct +// a user who placed markers with the opposite left/right convention. +constexpr int kMidlineIdx[] = {1, 152, 13, 14, 10}; +constexpr int kPairIdx[][2] = {{33, 263}, {133, 362}, {61, 291}, {105, 334}}; + +bool isMidline(int mpIdx) +{ + for (int m : kMidlineIdx) if (m == mpIdx) return true; + return false; +} +int pairOf(int mpIdx) +{ + for (const auto& p : kPairIdx) { + if (p[0] == mpIdx) return p[1]; + if (p[1] == mpIdx) return p[0]; + } + return -1; +} +} // namespace + namespace { int nearestTemplateVertex(const std::vector& tn, int tvc, const std::array& p) @@ -438,10 +465,41 @@ std::vector seedFaceMarkers( const int tvc = tmpl.vertexCount(); const auto& tn = tmpl.neutral(); if (tlm.ok) { + // SYMMETRIZE the detected template landmarks before resolving vertices: + // the ICT template is x-symmetric (midline at x=0), but MediaPipe + // drifts on the untextured template render (measured: nose tip 4 units + // off-midline), which mis-anchors EVERYTHING downstream. Midline + // features snap to x=0; side pairs get mirrored positions with the + // pair-mean height/depth and mean |x|. Detection still supplies the + // vertical/depth placement it gets roughly right. + auto detected = [&](int mpIdx, std::array& out) -> bool { + if (mpIdx < 0 || mpIdx >= int(tlm.points.size()) + || !tlm.valid[size_t(mpIdx)]) return false; + out = tlm.points[size_t(mpIdx)]; + return true; + }; for (auto& m : markers) { const int i = m.mediapipeIndex; - if (i >= 0 && i < int(tlm.points.size()) && tlm.valid[size_t(i)]) - m.tmplVertex = nearestTemplateVertex(tn, tvc, tlm.points[size_t(i)]); + std::array p; + if (!detected(i, p)) continue; + if (isMidline(i)) { + p[0] = 0.0f; + } else if (const int j = pairOf(i); j >= 0) { + std::array q; + if (detected(j, q)) { + const float xm = 0.5f * (std::fabs(p[0]) + std::fabs(q[0])); + const float ym = 0.5f * (p[1] + q[1]); + const float zm = 0.5f * (p[2] + q[2]); + // keep this marker on the side detection put it (ties → + // MP-left-named index goes to +X = character-right, + // the measured convention on this template). + float sign = p[0] > q[0] ? 1.0f : (p[0] < q[0] ? -1.0f + : ((i == 33 || i == 133 || i == 61 || i == 105) ? 1.0f + : -1.0f)); + p = { sign * xm, ym, zm }; + } // single-sided detection: use as-is + } + m.tmplVertex = nearestTemplateVertex(tn, tvc, p); } } @@ -525,13 +583,59 @@ std::vector seedFaceMarkers( return markers; } -std::vector anchorsFromMarkers(const std::vector& markers) +std::vector anchorsFromMarkers(const std::vector& markers, + const ArkitTemplate& tmpl) { - std::vector anchors; - for (const auto& m : markers) - if (m.placed && m.tmplVertex >= 0) - anchors.push_back({m.tmplVertex, m.userPos}); - return anchors; + // Two candidate pairings: as placed, and with the left/right PAIR targets + // swapped — a user may reasonably use either the character's or the + // screen's left/right. Score both against the template constellation and + // keep the one that agrees; a mirrored anchor set would ask the warp to + // fold the template through itself and crush every shape. + auto build = [&](bool swapped) { + std::vector anchors; + for (const auto& m : markers) { + if (!m.placed || m.tmplVertex < 0) continue; + std::array target = m.userPos; + if (swapped) { + const int j = pairOf(m.mediapipeIndex); + if (j >= 0) { + // use the PAIRED marker's position instead + for (const auto& o : markers) + if (o.mediapipeIndex == j && o.placed) { + target = o.userPos; + break; + } + } + } + anchors.push_back({m.tmplVertex, target}); + } + return anchors; + }; + auto residualOf = [&](const std::vector& anchors) { + std::vector> C, U; + const auto& tn = tmpl.neutral(); + const int tvc = tmpl.vertexCount(); + for (const auto& a : anchors) { + if (a.tmplVertex < 0 || a.tmplVertex >= tvc) continue; + C.push_back({tn[size_t(a.tmplVertex)*3], + tn[size_t(a.tmplVertex)*3+1], + tn[size_t(a.tmplVertex)*3+2]}); + U.push_back(a.target); + } + return constellationResidual(C, U); + }; + + std::vector normal = build(false); + if (!tmpl.valid()) return normal; + std::vector swapped = build(true); + const double rn = residualOf(normal); + const double rs = residualOf(swapped); + if (rs < rn) { + std::fprintf(stderr, "[facerig] markers look MIRRORED (residual %.3f " + "vs %.3f) — auto-swapping left/right pairs\n", rn, rs); + return swapped; + } + return normal; } } // namespace FaceRig diff --git a/src/FaceRig/FaceRigLandmarks.h b/src/FaceRig/FaceRigLandmarks.h index af88463db..5b923f9e6 100644 --- a/src/FaceRig/FaceRigLandmarks.h +++ b/src/FaceRig/FaceRigLandmarks.h @@ -86,8 +86,12 @@ std::vector seedFaceMarkers( bool* outConfident = nullptr); // Build NRICP anchors from the (possibly user-edited) markers — one per placed -// marker with a resolved template vertex. -std::vector anchorsFromMarkers(const std::vector& markers); +// marker with a resolved template vertex. Auto-corrects a MIRRORED placement: +// if swapping the left/right pair targets matches the template constellation +// better (the user assumed the opposite left/right convention), the swapped +// pairing is used — so either convention works. +std::vector anchorsFromMarkers(const std::vector& markers, + const ArkitTemplate& tmpl); } // namespace FaceRig diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 2eadff761..5ff948465 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -528,7 +528,8 @@ bool FaceRigController::rigFromMarkers(int maxShapes, double maxResidualPct) { if (!m_markerMode) { emit error(QStringLiteral("Not in marker mode.")); return false; } auto tmpl = m_markerTmpl; - const auto anchors = FaceRig::anchorsFromMarkers(m_markers); + if (!tmpl) { emit error(QStringLiteral("Template not loaded.")); return false; } + const auto anchors = FaceRig::anchorsFromMarkers(m_markers, *tmpl); int placedCount = 0; for (const auto& m : m_markers) placedCount += m.placed ? 1 : 0; qWarning("[facerig] rigFromMarkers: %d/%zu markers placed -> %zu anchors", From 5769441843ace47b614fa9b64c60163370e539fd Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 16 Jul 2026 15:58:50 -0400 Subject: [PATCH 18/38] feat(#889): marker UX polish + firmer anchors + Strength control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field feedback on the (now working) marker rig: - OFFSET: the deformation landed slightly BELOW the marked lips — the anchors loosened at the fine anneal levels (weight 10 x alpha drops under the data terms once alpha < 1) and the fitted lip line drifted. landmarkWeight 10→30 pins the marked features through the whole anneal; measured jawOpen amplitude nearly doubles (0.018 → 0.033) with the same default markers. - Markers now auto-advance IN CATALOG ORDER after each placement (the old "next unplaced" logic stuck on one marker since defaults mark all placed), so the user can walk the whole set click-by-click. - Chip highlight fixed: markerPlaced() is an invokable, so its QML binding never re-evaluated (stale placed state); it now depends on the notifying selectedMarker property. Selected chip gets a bold white outline. - "Strength" slider (0.5–3.0x, default 1.5): amplitude multiplier applied to every transferred delta (FaceRigOptions.amplitude) — the exaggeration knob the user asked for, legitimate now that the fit itself is correct. Co-Authored-By: Claude Opus 4.8 (1M context) --- qml/PropertiesPanel.qml | 41 ++++++++++++++++++++++++++++++++++---- src/FaceRig/FaceRigger.cpp | 6 ++++-- src/FaceRig/FaceRigger.h | 5 +++++ src/FaceRig/NonRigidICP.h | 6 +++++- src/FaceRigController.cpp | 24 +++++++++++----------- src/FaceRigController.h | 8 +++++--- 6 files changed, 68 insertions(+), 22 deletions(-) diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 06d69e765..672d3d7e4 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -9478,7 +9478,8 @@ Rectangle { + "side (a mirrored placement is auto-corrected)." } // Marker chips — click to select which one the next mesh - // click will move. + // click will move. After placing, selection auto-advances + // to the NEXT chip in this order. Flow { width: parent.width spacing: 3 @@ -9489,17 +9490,27 @@ Rectangle { width: chipText.implicitWidth + 12 radius: 3 property bool sel: index === FaceRigController.selectedMarker - property bool placed: FaceRigController.markerPlaced(index) + // NB: markerPlaced() is an invokable, not a + // property — reference selectedMarker in the + // binding so it re-evaluates on markersChanged + // (otherwise the chip state goes stale). + property bool placed: { + var _dep = FaceRigController.selectedMarker + return FaceRigController.markerPlaced(index) + } color: sel ? "#2ae6ff" : placed ? PropertiesPanelController.highlightColor : PropertiesPanelController.controlBgColor - border.color: PropertiesPanelController.borderColor + border.color: sel ? "#ffffff" + : PropertiesPanelController.borderColor + border.width: sel ? 2 : 1 opacity: placed || sel ? 1.0 : 0.6 Text { id: chipText anchors.centerIn: parent text: modelData font.pixelSize: 8 + font.bold: sel color: sel ? "#003" : PropertiesPanelController.textColor } MouseArea { @@ -9510,6 +9521,28 @@ Rectangle { } } } + // Strength (amplitude) — exaggeration multiplier for the + // transferred shapes; stylized faces often want >1. + RowLayout { + width: parent.width + spacing: 6 + Text { + text: "Strength" + font.pixelSize: 9 + color: PropertiesPanelController.textColor + } + Slider { + id: ampSlider + Layout.fillWidth: true + from: 0.5; to: 3.0; value: 1.5 + stepSize: 0.1 + } + Text { + text: "×" + ampSlider.value.toFixed(1) + font.pixelSize: 9 + color: PropertiesPanelController.textColor + } + } RowLayout { width: parent.width spacing: 4 @@ -9532,7 +9565,7 @@ Rectangle { anchors.fill: parent hoverEnabled: true cursorShape: Qt.PointingHandCursor - onClicked: FaceRigController.rigFromMarkers(0, 8.0) + onClicked: FaceRigController.rigFromMarkers(0, 8.0, ampSlider.value) } } Rectangle { diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index d50f44a84..d36878cab 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -443,11 +443,13 @@ FaceRigResult buildFaceRig(const std::vector& userV, // Deltas are always full-mesh sized; head isolation writes only the // head vertices (via subToFull), leaving the body at zero. out.userDeltas.assign(size_t(nuFull) * 3, 0.0f); + const float amp = float(std::clamp(opts.amplitude, 0.1, 5.0)); for (int i = 0; i < nu; ++i) { const int t = userToTmpl[size_t(i)]; if (t < 0) continue; - float dvec[3] = {tmplDelta[size_t(t)*3], tmplDelta[size_t(t)*3+1], - tmplDelta[size_t(t)*3+2]}; + float dvec[3] = {amp * tmplDelta[size_t(t)*3], + amp * tmplDelta[size_t(t)*3+1], + amp * tmplDelta[size_t(t)*3+2]}; const double mag = std::sqrt(double(dvec[0])*dvec[0] + double(dvec[1])*dvec[1] + double(dvec[2])*dvec[2]); diff --git a/src/FaceRig/FaceRigger.h b/src/FaceRig/FaceRigger.h index c5d462052..1a5566868 100644 --- a/src/FaceRig/FaceRigger.h +++ b/src/FaceRig/FaceRigger.h @@ -62,6 +62,11 @@ struct FaceRigOptions { // Cap the shapes generated (0 = all in the template). Names are matched // against the template's own shape order. int maxShapes = 0; + // Amplitude multiplier applied to every transferred delta (an + // "exaggeration" control, like Mixamo's arm-space). 1 = as transferred; + // >1 amplifies subtle results on stylized faces whose transfer comes out + // conservative. Clamped by callers to a sane range. + double amplitude = 1.0; }; // Progress callback: (done, total, phase). `phase` is a short static label diff --git a/src/FaceRig/NonRigidICP.h b/src/FaceRig/NonRigidICP.h index a5cec93af..7d63a4beb 100644 --- a/src/FaceRig/NonRigidICP.h +++ b/src/FaceRig/NonRigidICP.h @@ -57,7 +57,11 @@ struct NricpOptions { // high-weight data rows; the weight is strongest at the rigid (high-alpha) // levels so orientation/scale lock first, then relaxes as the fit refines. std::vector landmarks; - double landmarkWeight = 10.0; // base weight (× alpha at each level) + // Base anchor weight (× alpha at each level). High enough that the marked + // features stay PINNED through the fine anneal levels — at 10 the anchors + // loosened once alpha dropped below 1 and the fitted lip line drifted a few + // mm below the marked lips (field-observed offset). + double landmarkWeight = 30.0; }; // Progress callback for the annealing loop: (level, levelCount). Return false diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 5ff948465..8e003ced3 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -71,7 +71,8 @@ bool FaceRigController::hasMeshSelection() const return first && first->getMesh(); } -bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidualPct) +bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidualPct, + double amplitude) { if (m_busy) { emit error(QStringLiteral("A face-rig is already running.")); @@ -130,12 +131,12 @@ bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidu FaceRig::buildLandmarkAnchors(entity, headV, headF, *tmpl); m_geo = geo; - return runRigAsync(tmpl, maxShapes, maxResidualPct, anchors); + return runRigAsync(tmpl, maxShapes, maxResidualPct, amplitude, anchors); } bool FaceRigController::runRigAsync( const std::shared_ptr& tmpl, - int maxShapes, double maxResidualPct, + int maxShapes, double maxResidualPct, double amplitude, const std::vector& anchorsIn) { auto geo = m_geo; @@ -152,6 +153,7 @@ bool FaceRigController::runRigAsync( FaceRig::FaceRigOptions opts; opts.maxShapes = maxShapes; opts.maxFitResidualPct = maxResidualPct; + opts.amplitude = amplitude; auto anchors = std::make_shared>(anchorsIn); @@ -511,20 +513,18 @@ bool FaceRigController::handleMarkerClick(OgreWidget* widget, const QPoint& scre auto& m = m_markers[size_t(m_selMarker)]; m.userPos = { hitLocal.x, hitLocal.y, hitLocal.z }; m.placed = true; - // auto-advance to the next unplaced marker for a smooth flow. - int next = -1; - for (int i = 1; i <= int(m_markers.size()); ++i) { - const int idx = (m_selMarker + i) % int(m_markers.size()); - if (!m_markers[size_t(idx)].placed) { next = idx; break; } - } - m_selMarker = next >= 0 ? next : m_selMarker; + // auto-advance in CATALOG ORDER so the user can walk the whole set + // click-by-click (defaults mark everything "placed", so advancing to + // the next *unplaced* one would just stick on the same marker). + m_selMarker = (m_selMarker + 1) % int(m_markers.size()); } refreshMarkerOverlays(); emit markersChanged(); return true; } -bool FaceRigController::rigFromMarkers(int maxShapes, double maxResidualPct) +bool FaceRigController::rigFromMarkers(int maxShapes, double maxResidualPct, + double amplitude) { if (!m_markerMode) { emit error(QStringLiteral("Not in marker mode.")); return false; } auto tmpl = m_markerTmpl; @@ -541,7 +541,7 @@ bool FaceRigController::rigFromMarkers(int maxShapes, double maxResidualPct) emit markerModeChanged(); emit markersChanged(); if (!tmpl) { emit error(QStringLiteral("Template not loaded.")); return false; } - return runRigAsync(tmpl, maxShapes, maxResidualPct, anchors); + return runRigAsync(tmpl, maxShapes, maxResidualPct, amplitude, anchors); } void FaceRigController::clearMarkerOverlays() diff --git a/src/FaceRigController.h b/src/FaceRigController.h index 2284d005b..6fd2ba890 100644 --- a/src/FaceRigController.h +++ b/src/FaceRigController.h @@ -68,7 +68,8 @@ class FaceRigController : public QObject /// (invalid selection / already busy — `error` is emitted); the real /// outcome arrives via completed(report) or error(msg). Q_INVOKABLE bool addArkitBlendshapesAsync(int maxShapes = 0, - double maxResidualPct = 8.0); + double maxResidualPct = 8.0, + double amplitude = 1.0); /// Request cancellation of an in-flight fit (no-op otherwise). The worker /// stops at the next progress step; `error("cancelled")` follows. @@ -97,7 +98,8 @@ class FaceRigController : public QObject /// Commit: build NRICP anchors from the (edited) markers and run the rig /// (worker thread), same flow as addArkitBlendshapesAsync but anchored to /// the user-corrected markers. Returns false if it couldn't start. - Q_INVOKABLE bool rigFromMarkers(int maxShapes = 0, double maxResidualPct = 8.0); + Q_INVOKABLE bool rigFromMarkers(int maxShapes = 0, double maxResidualPct = 8.0, + double amplitude = 1.0); /// Called by TransformOperator on a viewport left-click while markerMode is /// active: ray-casts to the mesh surface and moves the SELECTED marker @@ -125,7 +127,7 @@ class FaceRigController : public QObject // main thread; `anchors` are the landmark constraints (auto or marker-based, // possibly empty). Runs the fit on a worker, attaches on the main thread. bool runRigAsync(const std::shared_ptr& tmpl, - int maxShapes, double maxResidualPct, + int maxShapes, double maxResidualPct, double amplitude, const std::vector& anchors); void clearMarkerOverlays(); From 463ebb6ff8ea9378f72420b676fda09ed26297bb Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 16 Jul 2026 20:02:32 -0400 Subject: [PATCH 19/38] fix(#889): include eye/teeth submeshes in the face rig (geometric head mask) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-reported: "this mesh has the eyes as a separated submesh, couldn't it animate the eyes as well?" — measured and confirmed: Rumba's eye/teeth submeshes are skinned to non-body-region bones (eye bones) that the rig-prior labeler can't classify, so they were SILENTLY EXCLUDED from the head mask (414-vert submesh with 0 masked verts) and could never receive blendshape deltas — eyeBlink moved 7 stray verts. Fix: after the label-based mask, geometrically expand it — any vertex inside the labeled head's slightly-padded AABB joins the face rig regardless of its skinning. Measured on Rumba: eyeBlinkLeft 7→205 verts (amplitude 0.0005→0.026, 50x), jawOpen 0.033→0.041, fit max residual 10%→2.9% (the fuller head helps the whole fit). Also adds per-submesh mask-coverage stats to the marker-sim diagnostic. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/CLIPipeline.cpp | 10 ++++++++++ src/FaceRig/FaceRigAttach.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index cce03d124..fdafa6585 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -9556,6 +9556,16 @@ int CLIPipeline::cmdFaceRig(int argc, char* argv[]) return 1; } FaceRig::FaceRigGeometry geo = FaceRig::extractGeometry(entity); + // per-submesh head-mask coverage (eye/teeth submeshes skinned to + // non-body-region bones can be silently excluded from the mask). + for (const auto& o : geo.owners) { + int inMask = 0; + for (int i = 0; i < o.count; ++i) + if (int(geo.headMask.size()) > int(o.base) + i + && geo.headMask[size_t(o.base) + size_t(i)]) ++inMask; + err() << "[sim] submesh handle=" << o.handle << " verts=" << o.count + << " inHeadMask=" << inMask << Qt::endl; + } std::vector headV; std::vector headF; FaceRig::headSubmesh(geo, headV, headF); bool confident = false; diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp index c6b223b57..dc2d99513 100644 --- a/src/FaceRig/FaceRigAttach.cpp +++ b/src/FaceRig/FaceRigAttach.cpp @@ -140,6 +140,35 @@ FaceRigGeometry extractGeometry(Ogre::Entity* entity) geo.headMask.assign(size_t(nv), 0); for (int v = 0; v < nv; ++v) if (labels[size_t(v)] == headPart) { geo.headMask[size_t(v)] = 1; ++headCount; } + + // GEOMETRIC expansion: eyes / teeth / tongue / lashes are often + // separate submeshes skinned to non-body-region bones (eye bones), + // which the rig-prior can't label — they'd be silently EXCLUDED from + // the face rig and never blink or look around (field-reported: 414-vert + // eye submesh with 0 masked verts). Anything inside the labeled head's + // slightly-expanded AABB belongs to the face. + if (headCount >= 50) { + float lo[3] = {1e30f, 1e30f, 1e30f}, hi[3] = {-1e30f, -1e30f, -1e30f}; + for (int v = 0; v < nv; ++v) { + if (!geo.headMask[size_t(v)]) continue; + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], geo.userV[size_t(v)*3 + a]); + hi[a] = std::max(hi[a], geo.userV[size_t(v)*3 + a]); + } + } + float pad[3]; + for (int a = 0; a < 3; ++a) pad[a] = 0.05f * (hi[a] - lo[a]); + for (int v = 0; v < nv; ++v) { + if (geo.headMask[size_t(v)]) continue; + bool inside = true; + for (int a = 0; a < 3; ++a) { + const float p = geo.userV[size_t(v)*3 + a]; + if (p < lo[a] - pad[a] || p > hi[a] + pad[a]) { inside = false; break; } + } + if (inside) { geo.headMask[size_t(v)] = 1; ++headCount; } + } + } + // Only isolate when the head is a real, minority region of the mesh — // i.e. this looks like a full body, not a bare face. A head that IS // most of the mesh means it's already a face crop; fit it whole. From 7864cc3747f0d9423f7dc744e4d614c8d842ccf6 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 16 Jul 2026 22:32:01 -0400 Subject: [PATCH 20/38] =?UTF-8?q?chore(#889):=20QTMESH=5FMORPH=5FPLAYTEST?= =?UTF-8?q?=20probe=20=E2=80=94=20headless=20morph=20playback=20verificati?= =?UTF-8?q?on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enables a named/first target at weight 1 on the imported mesh, runs the software animation update, and prints the measured vertex displacement — distinguishes 'targets dead' from 'targets subtle' without a GUI. Used to verify that re-imported glb morphs play at FULL amplitude (jawOpen 1.186 on the ARKit reference), i.e. export/import/playback are all healthy. Co-Authored-By: Claude Fable 5 --- src/CLIPipeline.cpp | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index fdafa6585..93577ce81 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -10230,6 +10230,64 @@ int CLIPipeline::cmdMorph(int argc, char* argv[]) } } + // Diagnostic (env-gated): verify a target actually PLAYS after import — + // enable it at weight 1 and measure the software-animated displacement. + // Distinguishes "targets listed but dead" (importer clip/pose bug) from a + // GUI-side playback issue. + if (!targets.isEmpty() + && qEnvironmentVariableIsSet("QTMESH_MORPH_PLAYTEST")) { + const QByteArray want = qgetenv("QTMESH_MORPH_PLAYTEST"); + QString name = QString::fromUtf8(want); + if (name == "1" || name.isEmpty()) name = targets.first(); + for (Ogre::Entity* entity : entities) { + auto* states = entity->getAllAnimationStates(); + const bool hasState = states + && states->hasAnimationState(name.toStdString()); + err() << "[playtest] entity=" << QString::fromStdString(entity->getName()) + << " target=" << name + << " hasAnimState=" << hasState + << " meshHasVertexAnim=" << entity->getMesh()->hasVertexAnimation() + << Qt::endl; + if (!hasState) continue; + auto* st = states->getAnimationState(name.toStdString()); + st->setEnabled(true); + st->setWeight(1.0f); + st->setTimePosition(0.0f); + entity->addSoftwareAnimationRequest(false); + entity->_updateAnimation(); + // measure displacement on each subentity's software-animated data + double maxDisp = 0; + for (unsigned int si = 0; si < entity->getNumSubEntities(); ++si) { + Ogre::SubEntity* se = entity->getSubEntity(si); + Ogre::VertexData* animVd = se->_getSoftwareVertexAnimVertexData(); + Ogre::VertexData* baseVd = se->getSubMesh()->vertexData; + if (!animVd || !baseVd) continue; + auto readPos = [](Ogre::VertexData* vd, std::vector& out) { + const auto* pe = vd->vertexDeclaration->findElementBySemantic(Ogre::VES_POSITION); + if (!pe) return; + auto vb = vd->vertexBufferBinding->getBuffer(pe->getSource()); + const size_t stride = vb->getVertexSize(); + auto* base = static_cast(vb->lock(Ogre::HardwareBuffer::HBL_READ_ONLY)); + out.resize(vd->vertexCount * 3); + for (size_t i = 0; i < vd->vertexCount; ++i) { + float* fp = nullptr; + pe->baseVertexPointerToElement(base + i*stride, &fp); + out[i*3] = fp[0]; out[i*3+1] = fp[1]; out[i*3+2] = fp[2]; + } + vb->unlock(); + }; + std::vector a, b; + readPos(animVd, a); readPos(baseVd, b); + for (size_t i = 0; i + 2 < std::min(a.size(), b.size()); i += 3) { + const double dx = a[i]-b[i], dy = a[i+1]-b[i+1], dz = a[i+2]-b[i+2]; + maxDisp = std::max(maxDisp, std::sqrt(dx*dx+dy*dy+dz*dz)); + } + } + err() << "[playtest] maxDisp after weight=1: " << maxDisp << Qt::endl; + entity->removeSoftwareAnimationRequest(false); + } + } + if (jsonOutput) { QJsonArray arr; for (const QString& n : targets) arr.append(n); From 0b20554148478a8f4f206732ec7d735ea577f236 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 16 Jul 2026 23:07:16 -0400 Subject: [PATCH 21/38] =?UTF-8?q?fix(#889):=20component-aware=20template?= =?UTF-8?q?=20fit=20=E2=80=94=20eyeballs/teeth=20survive=20the=20rig?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ICT template is 191 connected components (main face shell 14,016 verts; eyeballs, corneas, teeth, mouth interior ~half the vertex count). The NRICP surface fit dragged every interior component onto the outer skin, so eye shapes (eyeLook*/eyeBlink*) transferred onto collapsed geometry and never moved the user's eyes. Now: union-find the template faces into components, NRICP-fit only the main component (landmarks remapped onto it), then place each satellite component by a local least-squares affine estimated from its K=60 nearest fitted main verts. Falls back to the warped template rest on singular/insufficient neighborhoods. Reference self-rig deltas: eyeBlinkLeft 153→395 verts, eyeLookUpLeft 0→278, jawOpen 3,930→5,467 (teeth/mouth interior now follow the jaw), fit max residual 0.50%→0.34%. Co-Authored-By: Claude Fable 5 --- src/FaceRig/FaceRigger.cpp | 195 ++++++++++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 4 deletions(-) diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index d36878cab..f42d69d26 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -362,10 +362,89 @@ FaceRigResult buildFaceRig(const std::vector& userV, if (!warped.empty()) fitTmplV = std::move(warped); } - // 1) NRICP: (pre-warped) template neutral → user neutral = correspondence X. + // ── TEMPLATE COMPONENT SPLIT (the eyes/teeth fix) ─────────────────────── + // The ICT template is ~191 connected components: the outer face surface + // (14k verts) plus eyeballs, corneas, teeth, mouth interior, lashes… The + // surface fit drags INTERIOR component verts onto the OUTER skin (closest- + // point has no better answer), destroying their correspondence — measured: + // eyeLook/eyeBlink deltas landed nowhere and the eyes never moved, on the + // TEMPLATE ITSELF. Fit ONLY the main component; place each satellite by the + // local AFFINE its surrounding main-surface region underwent, preserving + // the eyeball/teeth structure inside the fitted head. + const int tvc = tmpl.vertexCount(); + std::vector comp(size_t(tvc), 0); + int compCount = 1; + { + std::vector par(size_t(tvc), 0); + for (int i = 0; i < tvc; ++i) par[size_t(i)] = i; + std::function findRoot = [&](int x) { + while (par[size_t(x)] != x) { + par[size_t(x)] = par[size_t(par[size_t(x)])]; + x = par[size_t(x)]; + } + return x; + }; + const auto& tf = tmpl.faces(); + for (size_t f = 0; f + 2 < tf.size(); f += 3) { + int a = findRoot(tf[f]), b = findRoot(tf[f+1]); + if (a != b) par[size_t(a)] = b; + a = findRoot(tf[f+1]); b = findRoot(tf[f+2]); + if (a != b) par[size_t(a)] = b; + } + std::unordered_map remap; + compCount = 0; + for (int i = 0; i < tvc; ++i) { + const int root = findRoot(i); + auto it = remap.find(root); + if (it == remap.end()) { remap.emplace(root, compCount); comp[size_t(i)] = compCount++; } + else comp[size_t(i)] = it->second; + } + } + int mainComp = 0; + { + std::vector cnt(size_t(compCount), 0); + for (int i = 0; i < tvc; ++i) cnt[size_t(comp[size_t(i)])]++; + for (int c = 1; c < compCount; ++c) + if (cnt[size_t(c)] > cnt[size_t(mainComp)]) mainComp = c; + } + + // Extract the main-component sub-template (from the possibly-warped verts) + // and remap anchors onto it (markers sit on the outer surface; any anchor + // that resolved onto a satellite is dropped). + std::vector mainV; std::vector mainF; std::vector mainToFull; + std::vector fullToMainIdx(size_t(tvc), -1); + if (compCount > 1) { + for (int i = 0; i < tvc; ++i) { + if (comp[size_t(i)] != mainComp) continue; + fullToMainIdx[size_t(i)] = int(mainToFull.size()); + mainToFull.push_back(i); + mainV.insert(mainV.end(), {fitTmplV[size_t(i)*3], + fitTmplV[size_t(i)*3+1], + fitTmplV[size_t(i)*3+2]}); + } + const auto& tf = tmpl.faces(); + for (size_t f = 0; f + 2 < tf.size(); f += 3) { + const int a = fullToMainIdx[size_t(tf[f])], + b = fullToMainIdx[size_t(tf[f+1])], + c = fullToMainIdx[size_t(tf[f+2])]; + if (a >= 0 && b >= 0 && c >= 0) mainF.insert(mainF.end(), {a, b, c}); + } + std::vector mainAnchors; + for (auto lm : fitOpts.landmarks) { + if (lm.tmplVertex < 0 || lm.tmplVertex >= tvc) continue; + const int mi = fullToMainIdx[size_t(lm.tmplVertex)]; + if (mi >= 0) { lm.tmplVertex = mi; mainAnchors.push_back(lm); } + } + fitOpts.landmarks = std::move(mainAnchors); + } + const bool splitTmpl = compCount > 1 && mainV.size() >= 9 && mainF.size() >= 3; + const std::vector& fitTV = splitTmpl ? mainV : fitTmplV; + const std::vector& fitTF = splitTmpl ? mainF : tmpl.faces(); + + // 1) NRICP: (pre-warped) template MAIN SURFACE → user neutral. // Report each annealing level so the (long) fit phase visibly advances. const NricpResult fit = FaceRig::fit( - fitTmplV, tmpl.faces(), fitV, fitF, fitOpts, + fitTV, fitTF, fitV, fitF, fitOpts, [&](int level, int /*levelCount*/) -> bool { if (!tick(level, "Fitting face template…")) { cancelled = true; return false; } return true; @@ -395,12 +474,120 @@ FaceRigResult buildFaceRig(const std::vector& userV, return r; } + // Assemble the FULL fitted correspondence: main verts from the fit; + // satellites by the local affine their neighbouring main region underwent + // (least-squares over the K nearest FINITE main verts). + const std::vector& tn = tmpl.neutral(); + std::vector fitted; + if (!splitTmpl) { + fitted = fit.fitted; + } else { + fitted.assign(size_t(tvc) * 3, 0.0f); + for (size_t m = 0; m < mainToFull.size(); ++m) + for (int d = 0; d < 3; ++d) + fitted[size_t(mainToFull[m])*3 + d] = fit.fitted[m*3 + d]; + + // group satellite verts per component + std::unordered_map> sats; + for (int i = 0; i < tvc; ++i) + if (comp[size_t(i)] != mainComp) sats[comp[size_t(i)]].push_back(i); + + for (auto& [cid, verts] : sats) { + // centroid in the (warped) template space + std::array ctr{0,0,0}; + for (int v : verts) + for (int d = 0; d < 3; ++d) + ctr[size_t(d)] += fitTmplV[size_t(v)*3+d] / double(verts.size()); + // K nearest FINITE main verts to the centroid + constexpr int K = 60; + std::vector> near; // (dist², main idx) + near.reserve(mainToFull.size()); + for (size_t m = 0; m < mainToFull.size(); ++m) { + bool finite = true; + for (int d = 0; d < 3; ++d) + if (!std::isfinite(fit.fitted[m*3+d])) { finite = false; break; } + if (!finite) continue; + const int fv = mainToFull[m]; + float d2 = 0; + for (int d = 0; d < 3; ++d) { + const float dd = fitTmplV[size_t(fv)*3+d] - float(ctr[size_t(d)]); + d2 += dd*dd; + } + near.push_back({d2, int(m)}); + } + const int k = std::min(K, int(near.size())); + if (k < 4) { + // no usable neighbours — leave the satellite at the template + // rest (it just won't deform meaningfully). + for (int v : verts) + for (int d = 0; d < 3; ++d) + fitted[size_t(v)*3+d] = tn[size_t(v)*3+d]; + continue; + } + std::partial_sort(near.begin(), near.begin()+k, near.end()); + // least-squares affine: (warped rest) → (fitted), normal equations + // per output dim: (SᵀS) w = Sᵀ t, S rows = [x y z 1]. + double StS[4][4] = {{0}}, Stt[3][4] = {{0}}; + for (int n = 0; n < k; ++n) { + const int m = near[size_t(n)].second; + const int fv = mainToFull[size_t(m)]; + const double s[4] = {fitTmplV[size_t(fv)*3], fitTmplV[size_t(fv)*3+1], + fitTmplV[size_t(fv)*3+2], 1.0}; + for (int a = 0; a < 4; ++a) + for (int b = 0; b < 4; ++b) + StS[a][b] += s[a]*s[b]; + for (int d = 0; d < 3; ++d) + for (int a = 0; a < 4; ++a) + Stt[d][a] += double(fit.fitted[size_t(m)*3+d]) * s[a]; + } + // solve 4x4 (Gaussian, shared factorisation for the 3 rhs) + double A[4][7]; + for (int a = 0; a < 4; ++a) { + for (int b = 0; b < 4; ++b) A[a][b] = StS[a][b]; + for (int d = 0; d < 3; ++d) A[a][4+d] = Stt[d][a]; + } + bool singular = false; + for (int col = 0; col < 4 && !singular; ++col) { + int piv = col; + for (int rr = col+1; rr < 4; ++rr) + if (std::abs(A[rr][col]) > std::abs(A[piv][col])) piv = rr; + if (std::abs(A[piv][col]) < 1e-12) { singular = true; break; } + if (piv != col) for (int cc = 0; cc < 7; ++cc) std::swap(A[piv][cc], A[col][cc]); + for (int rr = col+1; rr < 4; ++rr) { + const double f2 = A[rr][col] / A[col][col]; + for (int cc = col; cc < 7; ++cc) A[rr][cc] -= f2 * A[col][cc]; + } + } + double W[3][4]; // affine rows per output dim + if (!singular) { + for (int d = 0; d < 3; ++d) + for (int rr = 3; rr >= 0; --rr) { + double acc = A[rr][4+d]; + for (int cc = rr+1; cc < 4; ++cc) acc -= A[rr][cc] * W[d][cc]; + W[d][rr] = acc / A[rr][rr]; + } + } + for (int v : verts) { + if (singular) { + for (int d = 0; d < 3; ++d) + fitted[size_t(v)*3+d] = tn[size_t(v)*3+d]; + continue; + } + const double p[4] = {fitTmplV[size_t(v)*3], fitTmplV[size_t(v)*3+1], + fitTmplV[size_t(v)*3+2], 1.0}; + for (int d = 0; d < 3; ++d) { + double o = 0; + for (int a = 0; a < 4; ++a) o += W[d][a] * p[a]; + fitted[size_t(v)*3+d] = float(o); + } + } + } + } + // sanitize the correspondence: a handful of template verts may have // diverged (NaN/inf) even in an accepted fit (< 5% by the NRICP gate). // Fall those back to the template neutral so they don't poison the // deformation-transfer solve — they simply won't deform meaningfully. - std::vector fitted = fit.fitted; - const std::vector& tn = tmpl.neutral(); for (size_t i = 0; i < fitted.size() && i < tn.size(); ++i) if (!std::isfinite(fitted[i])) fitted[i] = tn[i]; From e7abe0159dbf254d979f94272188981c2dd517d2 Mon Sep 17 00:00:00 2001 From: Fernando Date: Thu, 16 Jul 2026 23:07:16 -0400 Subject: [PATCH 22/38] fix(#889): ARKit blendshape names survive the glb round-trip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assimp's glTF2 exporter drops mesh.extras.targetNames, so a rigged glb re-imported into the editor degraded to Shape_N morph names — losing the ARKit vocabulary face capture matches on. Import: Importer::loadModel reads the .arkit.json sidecar (schema qtmesh-arkit-blendshapes-v1) and passes the ordered names to MeshProcessor as name hints, applied only when the scene has exactly one morphed mesh (the unambiguous case) and the aiMesh carries no names of its own. Export: MeshImporterExporter::exporter writes the same sidecar next to any exported mesh that has poses (GUI parity with the CLI/MCP face-rig paths). Verified: qtmesh morph arkit_reference2.glb --list shows all 51 ARKit names (zero Shape_N); playtest jawOpen plays at the exported amplitude (0.751). Co-Authored-By: Claude Fable 5 --- src/Assimp/Importer.cpp | 24 ++++++++++++++++++++++++ src/Assimp/MeshProcessor.cpp | 19 +++++++++++++++++-- src/Assimp/MeshProcessor.h | 9 +++++++++ src/MeshImporterExporter.cpp | 22 ++++++++++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/Assimp/Importer.cpp b/src/Assimp/Importer.cpp index 84768f68f..af6e9eb4f 100644 --- a/src/Assimp/Importer.cpp +++ b/src/Assimp/Importer.cpp @@ -37,6 +37,10 @@ THE SOFTWARE. #include #include +#include +#include +#include +#include namespace { @@ -235,6 +239,26 @@ Ogre::MeshPtr AssimpToOgreImporter::loadModel(const std::string& path, bool conv // Process the root node recursively (meshes) MeshProcessor meshProcessor(skeleton, isZup); + + // ARKit blendshape name sidecar (`.arkit.json`, schema + // qtmesh-arkit-blendshapes-v1): Assimp's glTF2 exporter drops + // `targetNames`, so shapes in a re-imported glb arrive nameless and would + // degrade to "Shape_N". Restore the authored ARKit names from the sidecar + // the face-rig exporters write next to the mesh. + { + QFile sidecar(QString::fromStdString(path) + ".arkit.json"); + if (sidecar.exists() && sidecar.open(QIODevice::ReadOnly)) { + const QJsonObject root = QJsonDocument::fromJson(sidecar.readAll()).object(); + if (root.value("schema").toString().startsWith("qtmesh-arkit-blendshapes")) { + std::vector names; + for (const auto& v : root.value("names").toArray()) + names.push_back(v.toString().toStdString()); + if (!names.empty()) + meshProcessor.setMorphNameHints(std::move(names)); + } + } + } + meshProcessor.processNode(scene->mRootNode, scene); Ogre::MeshPtr ogreMesh = meshProcessor.createMesh(modelName, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME, materialProcessor); diff --git a/src/Assimp/MeshProcessor.cpp b/src/Assimp/MeshProcessor.cpp index a3427750c..f38d705f0 100644 --- a/src/Assimp/MeshProcessor.cpp +++ b/src/Assimp/MeshProcessor.cpp @@ -135,12 +135,27 @@ SubMeshData* MeshProcessor::processMesh(aiMesh* mesh, const aiScene* scene) { // time, where we already have the base positions in hand for the // Ogre::Pose constructor. Apply the same Z-up axis bake the base // vertex pass uses so the shape and base agree on coordinate frame. + // Sidecar name hints only apply when the scene has exactly ONE morphed + // mesh — the unambiguous case (a flat ordered name list can't be split + // across submeshes safely). + bool useNameHints = false; + if (!m_nameHints.empty()) { + unsigned morphedMeshes = 0; + for (auto mi = 0u; mi < scene->mNumMeshes; mi++) + if (scene->mMeshes[mi] && scene->mMeshes[mi]->mNumAnimMeshes > 0) + morphedMeshes++; + useNameHints = (morphedMeshes == 1); + } for(auto am = 0u; am < mesh->mNumAnimMeshes; am++) { const aiAnimMesh* anim = mesh->mAnimMeshes[am]; if (!anim || !anim->mVertices || anim->mNumVertices != mesh->mNumVertices) continue; MorphTargetData target; - target.name = anim->mName.length > 0 ? anim->mName.C_Str() - : (std::string("Shape_") + std::to_string(am)); + if (anim->mName.length > 0) + target.name = anim->mName.C_Str(); + else if (useNameHints && am < m_nameHints.size() && !m_nameHints[am].empty()) + target.name = m_nameHints[am]; + else + target.name = std::string("Shape_") + std::to_string(am); target.positions.reserve(anim->mNumVertices); for (auto i = 0u; i < anim->mNumVertices; i++) { Ogre::Vector3 v(anim->mVertices[i].x, anim->mVertices[i].y, anim->mVertices[i].z); diff --git a/src/Assimp/MeshProcessor.h b/src/Assimp/MeshProcessor.h index ef4c112d6..a2504804c 100644 --- a/src/Assimp/MeshProcessor.h +++ b/src/Assimp/MeshProcessor.h @@ -33,6 +33,14 @@ class MeshProcessor { void processNode(aiNode* node, const aiScene* scene); Ogre::MeshPtr createMesh(const Ogre::String& name, const Ogre::String& group, MaterialProcessor &materialProcessor); + // Morph-target name hints from a `.arkit.json` sidecar (ordered + // names). Assimp's glTF2 EXPORTER drops `targetNames`, so a re-imported + // rigged glb otherwise degrades to generated "Shape_N" names — losing the + // ARKit vocabulary face capture matches on. Applied only when the aiMesh + // itself carries no names and the scene has a single morphed mesh (the + // unambiguous case — e.g. the ARKit reference head). + void setMorphNameHints(std::vector names) { m_nameHints = std::move(names); } + protected: // Protected for testing purposes SubMeshData* processMesh(aiMesh* mesh, const aiScene* scene); @@ -42,4 +50,5 @@ class MeshProcessor { std::vector boneAssignments; Ogre::SkeletonPtr skeleton; bool m_isZup; + std::vector m_nameHints; }; diff --git a/src/MeshImporterExporter.cpp b/src/MeshImporterExporter.cpp index 431a6ac70..22ee0fadc 100755 --- a/src/MeshImporterExporter.cpp +++ b/src/MeshImporterExporter.cpp @@ -72,6 +72,7 @@ THE SOFTWARE. #include "SceneLightsIO.h" #include "SelectionSet.h" #include "SentryReporter.h" +#include "FaceRig/FaceRigAttach.h" #include "ExportOptimizer.h" #include "RTShaderHelper.h" @@ -3964,6 +3965,27 @@ int MeshImporterExporter::exporter(const Ogre::SceneNode *_sn, const QString &_u if (formatId == "gltf2" || formatId == "glb2") injectMorphWeightAnimations(file.filePath(), e, /*isBinary=*/formatId == "glb2"); + + // Morph-target NAME sidecar: Assimp's glTF2 exporter also + // drops `targetNames`, so a rigged mesh re-imported from this + // export would degrade to "Shape_N" names. Persist the + // ordered pose names next to the file (the same + // `.arkit.json` the CLI/MCP face-rig paths write); the + // importer restores them on load. + if (e->getMesh() && e->getMesh()->getPoseCount() > 0) + { + std::vector poseNames; + QSet seen; + for (const auto* pose : e->getMesh()->getPoseList()) + { + const QString n = QString::fromStdString(pose->getName()); + if (n.isEmpty() || seen.contains(n)) continue; + seen.insert(n); + poseNames.push_back(n); + } + if (!poseNames.empty()) + FaceRig::writeArkitSidecar(file.filePath(), poseNames); + } } delete scene; From 76bcb14372c1d11aeda1f7543d04603523dc4150 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 00:08:48 -0400 Subject: [PATCH 23/38] =?UTF-8?q?fix(#889):=20template=20had=20HALF=20its?= =?UTF-8?q?=20faces=20=E2=80=94=20repack=20+=20orientation-robust=20landma?= =?UTF-8?q?rk=20detection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of 'auto-detect finds nothing' and the checkerboard renders: export-arkit-template.py parsed only the first 3 indices of each face line, but ICT-FaceKit meshes are QUADS — the shipped template (and everything rendered or fit from it) was missing every quad's second triangle. Fan- triangulate in load_obj (26,384 → 52,220 tris) and repack. Detection fixes stacked on top, each verified end-to-end: - Multi-view: render front/back/left/right (+ a fog depth-map fallback pass that is immune to broken normals/winding) and keep the most confident view. Nothing guarantees an asset faces the renderer's front — glTF assets commonly face +Z while front() puts the camera on -Z, and the LH-flip asymmetry between import and glTF export flips facing on every round-trip. - Tight-crop search: MediaPipe FaceMesh is trained on detector-cropped faces (measured presence logit: -27 full-frame, -8.6 head-crop, +19.8 face-crop). detect() now scans silhouette-guided face-square candidates (subject-on-black renders) and refines around the winning landmark bbox. - The template's throwaway render entity now gets smooth vertex normals (it was POSITION-only — undefined normals turned the shaded render into per-triangle noise). - The flat depth material is now double-sided so inconsistently-wound assets don't render as culled-speckle. - MeshDepthRenderer: save/restore the REAL showBoundingBox state (the unconditional restore-to-true leaked debug boxes into later captures), hide other nodes' boxes during capture, and renderDepthMapView gains the same focusAabb head-framing as the shaded path. - Detection debug prints are now runtime-gated only (QTMESH_FACERIG_DEBUG worked only in Debug builds). With the repacked template the full auto-detect chain passes for the first time: template + user views detect at conf 1.00, 13/13 markers seeded, constellation residual 0.061 (trusted), anchored fit max residual 0.23%. Co-Authored-By: Claude Fable 5 --- scripts/export-arkit-template.py | 8 +- src/FaceRig/FaceLandmarkDetector.cpp | 100 +++++++++++++++++++++-- src/FaceRig/FaceLandmarkDetector.h | 8 ++ src/FaceRig/FaceRigLandmarks.cpp | 116 ++++++++++++++++++++++----- src/MeshDepthRenderer.cpp | 48 ++++++++--- src/MeshDepthRenderer.h | 6 +- 6 files changed, 247 insertions(+), 39 deletions(-) diff --git a/scripts/export-arkit-template.py b/scripts/export-arkit-template.py index 7a46a53c4..1275f11ee 100644 --- a/scripts/export-arkit-template.py +++ b/scripts/export-arkit-template.py @@ -84,7 +84,13 @@ def load_obj(path): if ln.startswith("v "): V.append([float(x) for x in ln.split()[1:4]]) elif ln.startswith("f "): - F.append([int(t.split("/")[0]) - 1 for t in ln.split()[1:4]]) + # Fan-triangulate: ICT-FaceKit meshes are QUADS. Taking only + # the first 3 indices dropped every quad's second triangle, + # shipping a template with half its faces missing (holed + # renders, fragmented connectivity). + idx = [int(t.split("/")[0]) - 1 for t in ln.split()[1:]] + for k in range(1, len(idx) - 1): + F.append([idx[0], idx[k], idx[k + 1]]) return np.asarray(V, np.float64), np.asarray(F, np.int32) diff --git a/src/FaceRig/FaceLandmarkDetector.cpp b/src/FaceRig/FaceLandmarkDetector.cpp index ed5072d1f..55fb7f153 100644 --- a/src/FaceRig/FaceLandmarkDetector.cpp +++ b/src/FaceRig/FaceLandmarkDetector.cpp @@ -181,12 +181,102 @@ LandmarkResult FaceLandmarkDetector::detect(const QImage& image) LandmarkResult r; if (!isAvailable() || image.isNull()) return r; - // Centre-square crop of the render, then resize to 256 — the head fills the - // frame (we rendered it), so the upstream face detector is unnecessary. We - // track the crop rect to map landmark px back into the ORIGINAL image space. const int W = image.width(), H = image.height(); - const int side = std::min(W, H); - const int ox = (W - side) / 2, oy = (H - side) / 2; + const int side0 = std::min(W, H); + + // The landmark graph is trained on tight, detector-cropped FACES. Measured + // on our own renders: presence logit -27 full-frame, -8.6 with the head + // filling the frame, +19.8 on a chin-to-forehead crop — so candidate crops + // must reach face-tightness before the model reports a face at all. + // + // Our renders put the subject on a black background, so a silhouette scan + // finds the head deterministically: bbox of non-black pixels, head width + // measured over the TOP part of the blob (shoulders are wider), then a few + // face-square candidates around it. A non-black background degrades to the + // centre-square fallback candidate. + struct Cand { int ox, oy, side; }; + std::vector cands; + { + const QImage gray = image.convertToFormat(QImage::Format_Grayscale8); + int top = H, bot = -1; + std::vector rowMin(size_t(H), W), rowMax(size_t(H), -1); + for (int y = 0; y < H; ++y) { + const uchar* ln = gray.constScanLine(y); + for (int x = 0; x < W; ++x) { + if (ln[x] > 12) { + top = std::min(top, y); bot = std::max(bot, y); + rowMin[size_t(y)] = std::min(rowMin[size_t(y)], x); + rowMax[size_t(y)] = std::max(rowMax[size_t(y)], x); + } + } + } + if (bot > top + 16) { + // head width = median row span over the top 35% of the blob + const int headRows = std::max(8, (bot - top) * 35 / 100); + int wMin = W, wMax = -1; + long cxSum = 0; int cxN = 0; + for (int y = top + headRows / 4; y < top + headRows; ++y) { + if (rowMax[size_t(y)] < 0) continue; + wMin = std::min(wMin, rowMin[size_t(y)]); + wMax = std::max(wMax, rowMax[size_t(y)]); + cxSum += (rowMin[size_t(y)] + rowMax[size_t(y)]) / 2; ++cxN; + } + if (wMax > wMin + 16 && cxN > 0) { + const int headW = wMax - wMin; + const int cx = int(cxSum / cxN); + // face square candidates: the face sits in the lower-middle of + // the head — try a few sizes/centres around it. + for (float s : {0.9f, 1.15f, 1.45f}) { + int side = std::min(int(headW * s), std::min(W, H)); + if (side < 32) continue; + const int cyFace = top + int(side * 0.62f); + cands.push_back({ + std::clamp(cx - side / 2, 0, W - side), + std::clamp(cyFace - side / 2, 0, H - side), + side }); + } + } + } + } + cands.push_back({ (W - side0) / 2, (H - side0) / 2, side0 }); // fallback + + LandmarkResult best; + for (const Cand& c : cands) { + LandmarkResult pr = runPass(image, c.ox, c.oy, c.side); + if (pr.ok && !pr.points.empty() && pr.confidence > best.confidence) { + best = std::move(pr); + if (best.confidence >= 0.98f) break; + } + } + if (!best.ok || best.points.empty()) return best; + + // Refine: tight re-crop around the winning landmark bbox (×1.6, the + // expansion MediaPipe's own face detector applies). + float mnx = best.points[0][0], mxx = mnx; + float mny = best.points[0][1], mxy = mny; + for (const auto& p : best.points) { + mnx = std::min(mnx, p[0]); mxx = std::max(mxx, p[0]); + mny = std::min(mny, p[1]); mxy = std::max(mxy, p[1]); + } + const float cx = (mnx + mxx) * 0.5f, cy = (mny + mxy) * 0.5f; + int side1 = int(std::max(mxx - mnx, mxy - mny) * 1.6f); + if (side1 >= 32) { + side1 = std::min(side1, std::min(W, H)); + const int ox1 = std::clamp(int(cx - side1 * 0.5f), 0, W - side1); + const int oy1 = std::clamp(int(cy - side1 * 0.5f), 0, H - side1); + LandmarkResult pass2 = runPass(image, ox1, oy1, side1); + if (pass2.ok && !pass2.points.empty() + && pass2.confidence >= best.confidence) + return pass2; + } + return best; +} + +LandmarkResult FaceLandmarkDetector::runPass(const QImage& image, + int ox, int oy, int side) +{ + LandmarkResult r; + if (!isAvailable() || image.isNull() || side <= 0) return r; QImage crop = image.copy(ox, oy, side, side) .convertToFormat(QImage::Format_RGB888) .scaled(kInputSize, kInputSize, Qt::IgnoreAspectRatio, diff --git a/src/FaceRig/FaceLandmarkDetector.h b/src/FaceRig/FaceLandmarkDetector.h index 20023b4cd..0e802823a 100644 --- a/src/FaceRig/FaceLandmarkDetector.h +++ b/src/FaceRig/FaceLandmarkDetector.h @@ -56,9 +56,17 @@ class FaceLandmarkDetector { // Detect landmarks on `image` (any format; the head should fill the frame, // centred and front-facing). Points come back in `image`'s pixel space. + // Runs TWO passes: a loose full-frame pass to locate the face, then a + // tight re-crop around the pass-1 landmark bbox — FaceMesh is trained on + // detector-cropped faces, so a loose frame depresses both the presence + // score and landmark accuracy. LandmarkResult detect(const QImage& image); private: + // One inference on the square crop (ox, oy, side) of `image`; landmark + // x/y mapped back to `image` pixel space. + LandmarkResult runPass(const QImage& image, int ox, int oy, int side); + struct Impl; std::unique_ptr d; QString m_error; diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index 1b8a12773..c3babf0be 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -76,21 +76,57 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, localV[size_t(i)*3+1], localV[size_t(i)*3+2])); } - // 1) render the head front-on (materials intact, lit) for the detector. - QString err; - const MeshDepthRenderer::RenderResult rr = - MeshDepthRenderer::renderShadedView( - entity, kRenderSize, MeshDepthRenderer::front(), &err, - focus.isNull() ? nullptr : &focus); - if (rr.depth.isNull()) return out; -#ifndef NDEBUG - if (const char* dp = std::getenv("QTMESH_FACERIG_DUMP_RENDER")) - rr.depth.save(QString::fromUtf8(dp)); -#endif - - // 2) detect landmarks (image-pixel space). - const LandmarkResult lr = det.detect(rr.depth); - if (!lr.ok || lr.points.empty()) return out; + // 1+2) render the head + detect landmarks — MULTI-VIEW. Nothing guarantees + // the mesh faces the renderer's "front" (glTF assets commonly face +Z + // while MeshDepthRenderer::front() places the camera on -Z; the LH-flip + // asymmetry between import and glTF export also flips facing on + // round-trips). Render the four horizontal views and keep the one + // MediaPipe is most confident about; a true face scores high (~0.9+), the + // back of a head scores low or fails outright. Early-out on a confident + // hit so the common facing stays one render. + const MeshDepthRenderer::View views[] = { + MeshDepthRenderer::front(), MeshDepthRenderer::back(), + MeshDepthRenderer::left(), MeshDepthRenderer::right(), + }; + MeshDepthRenderer::RenderResult rr; + LandmarkResult lr; + float bestConf = -1.0f; + // Two render styles per view: the shaded render (materials intact — + // carries texture contrast MediaPipe likes) and the fog depth-map render + // (pure geometry statue — immune to broken normals / inconsistent winding + // / missing textures, which turn the shaded render into unusable noise). + for (int depthMode = 0; depthMode <= 1 && bestConf < 0.85f; ++depthMode) { + for (const auto& view : views) { + QString err; + MeshDepthRenderer::RenderResult vrr = depthMode + ? MeshDepthRenderer::renderDepthMapView( + entity, kRenderSize, view, &err, + focus.isNull() ? nullptr : &focus) + : MeshDepthRenderer::renderShadedView( + entity, kRenderSize, view, &err, + focus.isNull() ? nullptr : &focus); + if (vrr.depth.isNull()) continue; + LandmarkResult vlr = det.detect(vrr.depth); + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, + "[facerig] detect view=%s mode=%s ok=%d conf=%.2f\n", + view.name, depthMode ? "depth" : "shaded", + vlr.ok, vlr.confidence); + if (const char* dp = std::getenv("QTMESH_FACERIG_DUMP_RENDER")) + vrr.depth.save(QString::fromUtf8(dp) + "." + + QString::fromStdString(entity->getName()) + "." + + (depthMode ? "depth." : "shaded.") + + view.name + ".png"); + if (!vlr.ok || vlr.points.empty()) continue; + if (vlr.confidence > bestConf) { + bestConf = vlr.confidence; + rr = std::move(vrr); + lr = std::move(vlr); + } + if (bestConf >= 0.85f) break; // confident face — stop rendering + } + } + if (rr.depth.isNull() || !lr.ok || lr.points.empty()) return out; out.confidence = lr.confidence; #ifndef NDEBUG if (const char* dp = std::getenv("QTMESH_FACERIG_DUMP_LANDMARKS")) { @@ -201,9 +237,48 @@ Ogre::Entity* templateEntity(const std::vector& v, sub->vertexData->vertexCount = size_t(vc); auto* decl = sub->vertexData->vertexDeclaration; decl->addElement(0, 0, Ogre::VET_FLOAT3, Ogre::VES_POSITION); + decl->addElement(0, sizeof(float) * 3, Ogre::VET_FLOAT3, Ogre::VES_NORMAL); + + // Smooth vertex normals (area-weighted face-normal accumulation). Without + // a NORMAL attribute the render pipeline samples undefined per-vertex data + // and the shaded render degrades to per-triangle noise MediaPipe can't + // detect a face in. + std::vector normals(v.size(), 0.0f); + for (size_t i = 0; i + 2 < f.size(); i += 3) { + const int a = f[i], b = f[i+1], c = f[i+2]; + if (a < 0 || b < 0 || c < 0 || a >= vc || b >= vc || c >= vc) continue; + const Ogre::Vector3 pa(v[size_t(a)*3], v[size_t(a)*3+1], v[size_t(a)*3+2]); + const Ogre::Vector3 pb(v[size_t(b)*3], v[size_t(b)*3+1], v[size_t(b)*3+2]); + const Ogre::Vector3 pc(v[size_t(c)*3], v[size_t(c)*3+1], v[size_t(c)*3+2]); + const Ogre::Vector3 n = (pb - pa).crossProduct(pc - pa); // area-weighted + for (int k : {a, b, c}) { + normals[size_t(k)*3+0] += n.x; + normals[size_t(k)*3+1] += n.y; + normals[size_t(k)*3+2] += n.z; + } + } + for (int i = 0; i < vc; ++i) { + Ogre::Vector3 n(normals[size_t(i)*3], normals[size_t(i)*3+1], + normals[size_t(i)*3+2]); + if (n.isZeroLength()) n = Ogre::Vector3::UNIT_Z; + n.normalise(); + normals[size_t(i)*3+0] = n.x; + normals[size_t(i)*3+1] = n.y; + normals[size_t(i)*3+2] = n.z; + } + + std::vector interleaved(size_t(vc) * 6); + for (int i = 0; i < vc; ++i) { + interleaved[size_t(i)*6+0] = v[size_t(i)*3+0]; + interleaved[size_t(i)*6+1] = v[size_t(i)*3+1]; + interleaved[size_t(i)*6+2] = v[size_t(i)*3+2]; + interleaved[size_t(i)*6+3] = normals[size_t(i)*3+0]; + interleaved[size_t(i)*6+4] = normals[size_t(i)*3+1]; + interleaved[size_t(i)*6+5] = normals[size_t(i)*3+2]; + } auto vbuf = Ogre::HardwareBufferManager::getSingleton().createVertexBuffer( decl->getVertexSize(0), vc, Ogre::HardwareBuffer::HBU_STATIC_WRITE_ONLY); - vbuf->writeData(0, v.size() * sizeof(float), v.data()); + vbuf->writeData(0, interleaved.size() * sizeof(float), interleaved.data()); sub->vertexData->vertexBufferBinding->setBinding(0, vbuf); const bool use32 = vc > 65535; @@ -311,12 +386,10 @@ std::vector buildLandmarkAnchors( U.push_back(a.target); } const double resid = constellationResidual(C, U); -#ifndef NDEBUG if (std::getenv("QTMESH_FACERIG_DEBUG")) std::fprintf(stderr, "[facerig] landmark anchors: tmpl.ok=%d " "user.ok=%d anchors=%zu residual=%.3f\n", tlm.ok, ulm.ok, anchors.size(), resid); -#endif if (resid >= 0.15) anchors.clear(); // garbage → fit unanchored } return anchors; @@ -533,12 +606,11 @@ std::vector seedFaceMarkers( const double resid = constellationResidual(detC, detU); const bool confident = ulm.ok && seeded >= int(markers.size()) * 3 / 4 && resid < 0.12; -#ifndef NDEBUG if (std::getenv("QTMESH_FACERIG_DEBUG")) - std::fprintf(stderr, "[facerig] seed: %d/%zu detected, constellation " - "residual %.3f -> %s\n", seeded, markers.size(), resid, + std::fprintf(stderr, "[facerig] seed: ulm.ok=%d conf=%.2f %d/%zu " + "detected, constellation residual %.3f -> %s\n", + ulm.ok, ulm.confidence, seeded, markers.size(), resid, confident ? "trusted" : "using proportional defaults"); -#endif if (confident) { for (auto& m : markers) { diff --git a/src/MeshDepthRenderer.cpp b/src/MeshDepthRenderer.cpp index ec7058864..d4c1e4f1f 100644 --- a/src/MeshDepthRenderer.cpp +++ b/src/MeshDepthRenderer.cpp @@ -64,6 +64,12 @@ Ogre::MaterialPtr ensureDepthMaterial() pass->setAmbient(0, 0, 0); // Fog must affect this pass. pass->setFog(false); // false => inherit scene fog + // Double-sided: assets with inconsistent triangle winding (LH round-trips, + // raw scans) would otherwise render half their faces as culled holes and + // the depth map degrades to speckle noise. The depth buffer still keeps + // the nearest surface, so a correctly-wound mesh is unaffected. + pass->setCullingMode(Ogre::CULL_NONE); + pass->setManualCullingMode(Ogre::MANUAL_CULL_NONE); return mat; } @@ -138,7 +144,8 @@ QImage MeshDepthRenderer::renderDepthMap(Ogre::Entity* entity, int size, } MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( - Ogre::Entity* entity, int size, const View& view, QString* errorOut) + Ogre::Entity* entity, int size, const View& view, QString* errorOut, + const Ogre::AxisAlignedBox* focusAabb) { RenderResult result; if (!entity) { @@ -154,8 +161,11 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( // Frame the camera on the entity's bounding sphere so the whole // mesh fills the view (matches how the generated texture is - // projection-baked back). - const Ogre::AxisAlignedBox aabb = entity->getWorldBoundingBox(true); + // projection-baked back). A focus box (e.g. the head of a full-body + // character) overrides the framing when given. + const Ogre::AxisAlignedBox aabb = + (focusAabb && !focusAabb->isNull()) ? *focusAabb + : entity->getWorldBoundingBox(true); const Ogre::Vector3 center = aabb.getCenter(); const Ogre::Real radius = aabb.getHalfSize().length(); if (radius <= 0.0f) { @@ -221,20 +231,31 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( } // Turn off the target entity's bounding box for the capture - // (it's on by default when selected). Remember to restore. + // (it's on when selected). Save the PRIOR state — restoring an + // unconditional `true` used to leave stray debug boxes on (and they + // draw into later captures: the box render ignores visibility). Ogre::SceneNode* targetNode = entity->getParentSceneNode(); + const bool targetBoxWasShown = targetNode && targetNode->getShowBoundingBox(); if (targetNode) targetNode->showBoundingBox(false); // Hide other entities entirely, remembering each node's prior // visibility so we restore exactly what we changed (a node that - // was already hidden must stay hidden on restore). + // was already hidden must stay hidden on restore). Their bounding + // boxes must be turned off too — showBoundingBox draws via the + // scene manager's debug pass even when the node's objects are hidden. std::vector> hiddenNodes; + std::vector hiddenBoxes; if (Manager::getSingletonPtr()) { for (Ogre::Entity* other : Manager::getSingleton()->getEntities()) { if (!other || other == entity) continue; if (other->getMovableType() != "Entity") continue; Ogre::SceneNode* n = other->getParentSceneNode(); - if (n && n->getAttachedObject(0) && n->getAttachedObject(0)->getVisible()) { + if (!n) continue; + if (n->getShowBoundingBox()) { + hiddenBoxes.push_back(n); + n->showBoundingBox(false); + } + if (n->getAttachedObject(0) && n->getAttachedObject(0)->getVisible()) { hiddenNodes.emplace_back(n, true); n->setVisible(false); } @@ -261,7 +282,8 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( sm->setFog(savedFogMode, savedFogColour, 0.0f, savedFogStart, savedFogEnd); sm->setAmbientLight(savedAmbient); if (gridNode) gridNode->setVisible(gridWasVisible); - if (targetNode) targetNode->showBoundingBox(true); + if (targetNode) targetNode->showBoundingBox(targetBoxWasShown); + for (auto* n : hiddenBoxes) n->showBoundingBox(true); for (auto& [n, wasVisible] : hiddenNodes) n->setVisible(wasVisible); }; struct Restorer { @@ -363,14 +385,21 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderShadedView( } } Ogre::SceneNode* targetNode = entity->getParentSceneNode(); + const bool targetBoxWasShown = targetNode && targetNode->getShowBoundingBox(); if (targetNode) targetNode->showBoundingBox(false); std::vector> hiddenNodes; + std::vector hiddenBoxes; if (Manager::getSingletonPtr()) { for (Ogre::Entity* other : Manager::getSingleton()->getEntities()) { if (!other || other == entity) continue; if (other->getMovableType() != "Entity") continue; Ogre::SceneNode* n = other->getParentSceneNode(); - if (n && n->getAttachedObject(0) && n->getAttachedObject(0)->getVisible()) { + if (!n) continue; + if (n->getShowBoundingBox()) { + hiddenBoxes.push_back(n); + n->showBoundingBox(false); + } + if (n->getAttachedObject(0) && n->getAttachedObject(0)->getVisible()) { hiddenNodes.emplace_back(n, true); n->setVisible(false); } @@ -384,7 +413,8 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderShadedView( sm->getRootSceneNode()->removeAndDestroyChild(lightNode); } if (light) sm->destroyLight(light); if (gridNode) gridNode->setVisible(gridWasVisible); - if (targetNode) targetNode->showBoundingBox(true); + if (targetNode) targetNode->showBoundingBox(targetBoxWasShown); + for (auto* n : hiddenBoxes) n->showBoundingBox(true); for (auto& [n, wasVisible] : hiddenNodes) n->setVisible(wasVisible); }; struct Restorer { std::function fn; ~Restorer() { fn(); } } restorer{restore}; diff --git a/src/MeshDepthRenderer.h b/src/MeshDepthRenderer.h index 4b26b6f64..eb9600e89 100644 --- a/src/MeshDepthRenderer.h +++ b/src/MeshDepthRenderer.h @@ -72,11 +72,13 @@ class MeshDepthRenderer { // Render `entity`'s depth map at `size` x `size` from `view`. Returns a // RenderResult whose `.depth` is null on failure (errorOut populated). // Must be called on the main/render thread — it touches the Ogre scene - // manager. + // manager. `focusAabb` (optional, WORLD space) frames the camera on that + // box instead of the whole entity (same semantics as renderShadedView). static RenderResult renderDepthMapView(Ogre::Entity* entity, int size, const View& view, - QString* errorOut = nullptr); + QString* errorOut = nullptr, + const Ogre::AxisAlignedBox* focusAabb = nullptr); // Back-compat convenience: front-view depth image only (issue #403). static QImage renderDepthMap(Ogre::Entity* entity, From d9cf384c89384f5f3d5090476e5444e0f102c747 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 02:43:25 -0400 Subject: [PATCH 24/38] =?UTF-8?q?fix(#889):=20GUI-context=20landmark=20det?= =?UTF-8?q?ection=20=E2=80=94=20frozen=20RTT,=20blown-out=20lighting,=20so?= =?UTF-8?q?ft=20anchors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three GUI-only failures made in-editor auto-detect useless while the headless CLI path passed (reproduced by driving the live editor via MCP): 1. Frozen RTT: MeshDepthRenderer restores the editor render target after every capture, but GL3+ kept the offscreen viewport cached as active — the next update() skipped the FBO re-bind and rendered into the editor window. All 16 multi-view captures came back bit-identical (the first frame). restoreEditorRenderTarget() now clears the cached viewport (rs->_setViewport(nullptr)) so each capture re-binds. 2. Blown-out capture: scene/user lights stacked on the capture's ambient 0.75 + headlight and saturated the render to a pure-white silhouette. MediaPipe false-positives (presence >= 0.85) on the featureless blob, and because template and user renders degrade to the SAME canonical garbage layout, the constellation gate passed it — garbage anchors, fit max residual 3.65%, and the crumpled eyelid/mouth deltas the user saw. renderShadedView now disables all existing scene lights for the capture (deterministic ambient 0.35 + 0.65 headlight, restored after), and detectMeshLandmarks skips renders whose subject has near-zero intensity variance (a silhouette carries no facial features). 3. Hard anchors: landmark weight only rode alpha, so at the finest anneal level a slightly-off user mark still outweighed the surface term and dragged its neighbourhood off the mesh (spiky eyeBlink/mouthClose deltas). The weight now ALSO decays linearly to zero by the last level — markers steer the coarse alignment, pure surface snapping finishes. Markers act as an initialization, not a constraint. Live-editor rig of the ARKit reference: fit max residual 3.65% -> 0.37%, detection confident (conf 1.00 on the correct view, early-out after 2 renders), simulated imperfect markers stay smooth (max 0.35%). Co-Authored-By: Claude Fable 5 --- src/FaceRig/FaceRigLandmarks.cpp | 24 ++++++++++++++++++++++++ src/FaceRig/NonRigidICP.cpp | 14 ++++++++++++-- src/MeshDepthRenderer.cpp | 28 +++++++++++++++++++++++----- src/OgreRenderTargetUtil.h | 9 ++++++++- 4 files changed, 67 insertions(+), 8 deletions(-) diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index c3babf0be..e092cf4c6 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -106,6 +106,30 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, entity, kRenderSize, view, &err, focus.isNull() ? nullptr : &focus); if (vrr.depth.isNull()) continue; + // Flatness sanity: a blown-out / silhouette render (near-zero + // intensity variance inside the subject) carries no facial + // features — MediaPipe false-positives on such blobs with high + // presence, and the garbage landmarks CORRELATE between the + // template and user renders, slipping through the constellation + // gate. A genuinely shaded face has stddev well above this. + { + const QImage g = vrr.depth.convertToFormat(QImage::Format_Grayscale8); + double sum = 0, sum2 = 0; long n = 0; + for (int y = 0; y < g.height(); ++y) { + const uchar* ln = g.constScanLine(y); + for (int x = 0; x < g.width(); ++x) { + if (ln[x] > 12) { sum += ln[x]; sum2 += double(ln[x]) * ln[x]; ++n; } + } + } + const double var = n > 0 ? (sum2 / n - (sum / n) * (sum / n)) : 0.0; + if (n < 64 || var < 36.0) { // stddev < 6 → featureless + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] detect view=%s mode=%s " + "SKIPPED (flat render, var=%.1f)\n", + view.name, depthMode ? "depth" : "shaded", var); + continue; + } + } LandmarkResult vlr = det.detect(vrr.depth); if (std::getenv("QTMESH_FACERIG_DEBUG")) std::fprintf(stderr, diff --git a/src/FaceRig/NonRigidICP.cpp b/src/FaceRig/NonRigidICP.cpp index 2bad23508..b8651434e 100644 --- a/src/FaceRig/NonRigidICP.cpp +++ b/src/FaceRig/NonRigidICP.cpp @@ -272,14 +272,24 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, // Landmark anchors that reference a valid template vertex. Weight // rides alpha so it dominates while the fit is still rigid (locking - // orientation/scale), then relaxes as alpha anneals down. + // orientation/scale), then relaxes as alpha anneals down — AND + // decays linearly to ZERO by the finest level, so anchors act as + // an INITIALIZATION, not a hard constraint. User-placed markers + // are never pixel-perfect; without the decay a slightly-off mark + // outweighs the surface term at the end of the anneal and drags + // its neighbourhood off the mesh (spiky eyelid/lip deltas). The + // final levels are pure surface snapping from the marker-steered + // coarse alignment. std::vector lms; lms.reserve(opts.landmarks.size()); for (const auto& lm : opts.landmarks) if (lm.tmplVertex >= 0 && lm.tmplVertex < Nt) lms.push_back(&lm); const int L = int(lms.size()); - const double lw = opts.landmarkWeight * alpha; + const double anneal = levelCount > 1 + ? double(levelCount - 1 - levelIdx) / double(levelCount - 1) + : 1.0; + const double lw = opts.landmarkWeight * alpha * anneal; // assemble A (rows = Nt data + E stiffness + L landmark) once; // rhs differs per axis diff --git a/src/MeshDepthRenderer.cpp b/src/MeshDepthRenderer.cpp index d4c1e4f1f..709c34685 100644 --- a/src/MeshDepthRenderer.cpp +++ b/src/MeshDepthRenderer.cpp @@ -350,23 +350,40 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderShadedView( st.cameraNode->lookAt(center, Ogre::Node::TS_WORLD, Ogre::Vector3::NEGATIVE_UNIT_Z); - // Neutral flat lighting so MediaPipe sees an evenly-lit, photo-like face - // (no fog, materials intact). Bright ambient + a head-on light fills in - // feature shading without harsh shadows that would confuse the detector. + // Neutral, DETERMINISTIC lighting so MediaPipe sees an evenly-lit, + // photo-like face (no fog, materials intact): moderate ambient + one + // head-on directional. All EXISTING scene lights are disabled for the + // capture — in the live editor the user/default lights stack on top and + // saturate the render to a pure-white silhouette, which the detector + // false-positives on (and the resulting garbage landmarks correlate + // between template and user render, slipping through the constellation + // gate). Headless and GUI captures must produce the same image. const Ogre::ColourValue savedAmbient = sm->getAmbientLight(); const Ogre::FogMode savedFogMode = sm->getFogMode(); const Ogre::ColourValue savedFogColour = sm->getFogColour(); const Ogre::Real savedFogStart = sm->getFogStart(); const Ogre::Real savedFogEnd = sm->getFogEnd(); sm->setFog(Ogre::FOG_NONE); - sm->setAmbientLight(Ogre::ColourValue(0.75f, 0.75f, 0.75f)); + sm->setAmbientLight(Ogre::ColourValue(0.35f, 0.35f, 0.35f)); + + std::vector disabledLights; + { + auto it = sm->getMovableObjectIterator("Light"); + while (it.hasMoreElements()) { + auto* l = static_cast(it.getNext()); + if (l && l->getVisible()) { + disabledLights.push_back(l); + l->setVisible(false); + } + } + } Ogre::Light* light = nullptr; Ogre::SceneNode* lightNode = nullptr; try { light = sm->createLight("QtMeshFaceRigLight"); light->setType(Ogre::Light::LT_DIRECTIONAL); - light->setDiffuseColour(Ogre::ColourValue(0.5f, 0.5f, 0.5f)); + light->setDiffuseColour(Ogre::ColourValue(0.65f, 0.65f, 0.65f)); light->setSpecularColour(Ogre::ColourValue::Black); lightNode = sm->getRootSceneNode()->createChildSceneNode(); lightNode->attachObject(light); @@ -412,6 +429,7 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderShadedView( if (lightNode) { lightNode->detachAllObjects(); sm->getRootSceneNode()->removeAndDestroyChild(lightNode); } if (light) sm->destroyLight(light); + for (auto* l : disabledLights) l->setVisible(true); if (gridNode) gridNode->setVisible(gridWasVisible); if (targetNode) targetNode->showBoundingBox(targetBoxWasShown); for (auto* n : hiddenBoxes) n->showBoundingBox(true); diff --git a/src/OgreRenderTargetUtil.h b/src/OgreRenderTargetUtil.h index d0702f922..789aae42d 100644 --- a/src/OgreRenderTargetUtil.h +++ b/src/OgreRenderTargetUtil.h @@ -45,8 +45,15 @@ inline void restoreEditorRenderTarget() } } } - if (editorWindow) + if (editorWindow) { rs->_setRenderTarget(editorWindow); + // Clear the render system's cached active viewport. Without this the + // NEXT offscreen RTT update sees its viewport still marked active, + // skips the FBO re-bind, and renders into the editor window instead — + // the RTT then returns the same frozen frame for every later capture + // (multi-view landmark detection got 16 bit-identical images). + rs->_setViewport(nullptr); + } } } // namespace OgreRenderTargetUtil From da39bd82ad5d1e7c44ec96d0dd2125e7ea9602a2 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 12:06:14 -0400 Subject: [PATCH 25/38] fix(#889): reset face-rig controls when the selection switches models An active marker session (chips, viewport overlays, Rig-from-markers state) belongs to the entity it was started on; switching to another model kept it alive and the stale status line read as if it applied to the new selection. On selectionChanged: cancel the marker session when the selected entity differs from the session's entity (or nothing is selected), and clear the status line when idle. Co-Authored-By: Claude Fable 5 --- src/FaceRigController.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp index 8e003ced3..8d6874f23 100644 --- a/src/FaceRigController.cpp +++ b/src/FaceRigController.cpp @@ -51,7 +51,23 @@ void FaceRigController::kill() FaceRigController::FaceRigController() : QObject(nullptr) { connect(SelectionSet::getSingleton(), &SelectionSet::selectionChanged, - this, &FaceRigController::selectionChanged); + this, [this]() { + // Selecting a DIFFERENT model must reset the face-rig controls: an + // active marker session (chips, overlays, Rig-from-markers state) + // belongs to the entity it was started on, and a stale status line + // from the previous rig reads as if it applied to the new selection. + if (m_markerMode) { + auto* sel = SelectionSet::getSingleton(); + const auto entities = sel ? sel->getResolvedEntities() + : QList{}; + Ogre::Entity* first = entities.isEmpty() ? nullptr : entities.first(); + if (!first || first->getName() != m_markerEntityName) + cancelFaceMarkers(); + } else if (!m_busy) { + setStatus(QString()); + } + emit selectionChanged(); + }); } void FaceRigController::setStatus(const QString& s) From 641b2a2f293a2ac028f38fe043c8cbde73ef4891 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 12:17:04 -0400 Subject: [PATCH 26/38] =?UTF-8?q?fix(#889):=20rank=20detection=20views=20b?= =?UTF-8?q?y=20RAW=20presence=20logit=20=E2=80=94=20no=20more=20back-of-he?= =?UTF-8?q?ad=20lock-on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MediaPipe's presence sigmoid saturates near 1.0 for both a true face (logit ~+20) and a convincing false positive — measured: the smooth back/ side of a head scores up to 0.87 sigmoid but only ~+2 logit. The old first-hit-above-0.85 early-out could therefore lock onto the BACK view of a backwards-facing import (glb round-trips flip facing) and anchor the whole rig there. detectMeshLandmarks now evaluates ALL four views per render mode and keeps the highest RAW logit — effectively detecting the mesh orientation before the fit, via the detector itself. The crop-candidate and refine passes inside detect() rank by logit too. The depth-mode fallback still only runs when no shaded view reaches a strong-face logit (>= 6). Reference verification (backwards-facing glb): views score front -3.2 / back +19.8 / left -3.2 / right +1.9 — the true face wins by an order of magnitude; live-editor rig fit max residual 0.299%. Co-Authored-By: Claude Fable 5 --- src/FaceRig/FaceLandmarkDetector.cpp | 7 ++++--- src/FaceRig/FaceLandmarkDetector.h | 5 +++++ src/FaceRig/FaceRigLandmarks.cpp | 23 ++++++++++++++++------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/FaceRig/FaceLandmarkDetector.cpp b/src/FaceRig/FaceLandmarkDetector.cpp index 55fb7f153..62448c4a7 100644 --- a/src/FaceRig/FaceLandmarkDetector.cpp +++ b/src/FaceRig/FaceLandmarkDetector.cpp @@ -243,9 +243,9 @@ LandmarkResult FaceLandmarkDetector::detect(const QImage& image) LandmarkResult best; for (const Cand& c : cands) { LandmarkResult pr = runPass(image, c.ox, c.oy, c.side); - if (pr.ok && !pr.points.empty() && pr.confidence > best.confidence) { + if (pr.ok && !pr.points.empty() + && pr.presenceLogit > best.presenceLogit) { best = std::move(pr); - if (best.confidence >= 0.98f) break; } } if (!best.ok || best.points.empty()) return best; @@ -266,7 +266,7 @@ LandmarkResult FaceLandmarkDetector::detect(const QImage& image) const int oy1 = std::clamp(int(cy - side1 * 0.5f), 0, H - side1); LandmarkResult pass2 = runPass(image, ox1, oy1, side1); if (pass2.ok && !pass2.points.empty() - && pass2.confidence >= best.confidence) + && pass2.presenceLogit >= best.presenceLogit) return pass2; } return best; @@ -327,6 +327,7 @@ LandmarkResult FaceLandmarkDetector::runPass(const QImage& image, r.confidence = presenceFound ? 1.f / (1.f + std::exp(-std::clamp(presenceLogit, -50.f, 50.f))) : 1.f; + r.presenceLogit = presenceFound ? presenceLogit : 0.f; const int n = int(landmarkFloats / 3); r.points.reserve(size_t(n)); diff --git a/src/FaceRig/FaceLandmarkDetector.h b/src/FaceRig/FaceLandmarkDetector.h index 0e802823a..6f69f0804 100644 --- a/src/FaceRig/FaceLandmarkDetector.h +++ b/src/FaceRig/FaceLandmarkDetector.h @@ -29,6 +29,11 @@ struct LandmarkResult { // z a relative depth (unused for back-projection). Empty when no face. std::vector> points; float confidence = 0.0f; // presence sigmoid, 0 = no face + // RAW presence logit. The sigmoid saturates (~1.0) for both a true face + // (logit ~+20) and a convincing false positive like the smooth back of a + // head (low positive logit) — rank candidate views/crops by THIS, never + // by `confidence`. + float presenceLogit = -1e9f; bool ok = false; }; diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index e092cf4c6..cea7d3e90 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -90,12 +90,21 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, }; MeshDepthRenderer::RenderResult rr; LandmarkResult lr; - float bestConf = -1.0f; + float bestLogit = -1e9f; // Two render styles per view: the shaded render (materials intact — // carries texture contrast MediaPipe likes) and the fog depth-map render // (pure geometry statue — immune to broken normals / inconsistent winding // / missing textures, which turn the shaded render into unusable noise). - for (int depthMode = 0; depthMode <= 1 && bestConf < 0.85f; ++depthMode) { + // + // EVERY view in a mode is evaluated — no first-hit early-out. The winner + // is the highest RAW presence logit: a true face scores ~+20 while a + // false positive (the smooth back of a head) scores far lower, but both + // saturate the sigmoid, so an early-out on `confidence` locked onto the + // back of backwards-facing imports (glb round-trips flip facing). Ranking + // all four views by logit IS the orientation detection. + constexpr float kStrongFaceLogit = 6.0f; // sigmoid ≈ 0.998 + for (int depthMode = 0; + depthMode <= 1 && bestLogit < kStrongFaceLogit; ++depthMode) { for (const auto& view : views) { QString err; MeshDepthRenderer::RenderResult vrr = depthMode @@ -133,21 +142,21 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, LandmarkResult vlr = det.detect(vrr.depth); if (std::getenv("QTMESH_FACERIG_DEBUG")) std::fprintf(stderr, - "[facerig] detect view=%s mode=%s ok=%d conf=%.2f\n", + "[facerig] detect view=%s mode=%s ok=%d conf=%.2f " + "logit=%.1f\n", view.name, depthMode ? "depth" : "shaded", - vlr.ok, vlr.confidence); + vlr.ok, vlr.confidence, vlr.presenceLogit); if (const char* dp = std::getenv("QTMESH_FACERIG_DUMP_RENDER")) vrr.depth.save(QString::fromUtf8(dp) + "." + QString::fromStdString(entity->getName()) + "." + (depthMode ? "depth." : "shaded.") + view.name + ".png"); if (!vlr.ok || vlr.points.empty()) continue; - if (vlr.confidence > bestConf) { - bestConf = vlr.confidence; + if (vlr.presenceLogit > bestLogit) { + bestLogit = vlr.presenceLogit; rr = std::move(vrr); lr = std::move(vlr); } - if (bestConf >= 0.85f) break; // confident face — stop rendering } } if (rr.depth.isNull() || !lr.ok || lr.points.empty()) return out; From d8e2dbd14a72cb8bf76fe585aaf09213fd1ed096 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 12:37:56 -0400 Subject: [PATCH 27/38] =?UTF-8?q?fix(#889):=20facing-aware=20default=20mar?= =?UTF-8?q?kers=20=E2=80=94=20face=20side,=20not=20the=20back=20of=20the?= =?UTF-8?q?=20head?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When detection is too weak to trust (stylized/covered faces), the seeded markers fall back to head-box-projected template positions — but the box mapping assumed the mesh faces the template's +Z, so a backwards-facing import got every default on the BACK of the head. detectMeshLandmarks now exposes the mesh's facing (MESH-LOCAL) resolved by a priority ladder: 1. presence-logit winner view — when any view shows positive face evidence; 2. FEET direction — full-body characters only (body > 2.5x head height): horizontal centroid of the feet slab (lowest 8%) vs the ankle slab above it; toes extend forward. Verified on the masked-face bandit where every MediaPipe logit is negative and depth-detail mis-ranks (braids + hat out-detail a covered face); 3. depth-map Laplacian detail winner — busts/heads with no body. seedFaceMarkers yaw-rotates the template coordinates to the resolved cardinal facing before the head-box mapping, so proportional defaults land on the face whichever way the model faces. The trusted-detection path and the ICT reference are unchanged (13/13, residual 0.034, fit max 0.29%). Co-Authored-By: Claude Fable 5 --- src/FaceRig/FaceRigLandmarks.cpp | 164 ++++++++++++++++++++++++++++++- src/FaceRig/FaceRigLandmarks.h | 8 ++ 2 files changed, 167 insertions(+), 5 deletions(-) diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index cea7d3e90..90fe2f49e 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -2,6 +2,7 @@ #include "ArkitTemplate.h" #include "FaceLandmarkDetector.h" +#include "FaceRigAttach.h" #include "../Manager.h" #include "../MeshDepthRenderer.h" @@ -91,6 +92,28 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, MeshDepthRenderer::RenderResult rr; LandmarkResult lr; float bestLogit = -1e9f; + // Geometric facing fallback: mean |Laplacian| of the fog depth render + // over subject pixels. The face side of a head (nose, brows, lips, chin) + // carries far more depth detail than the smooth occiput — used to decide + // facing when NO view shows positive face evidence (stylized faces + // MediaPipe can't read return only noise logits). + double bestDetail = -1.0; + Ogre::Vector3 bestDetailCamDir = Ogre::Vector3::ZERO; + auto depthDetail = [](const QImage& img) -> double { + const QImage g = img.convertToFormat(QImage::Format_Grayscale8); + double acc = 0; long n = 0; + for (int y = 1; y + 1 < g.height(); ++y) { + const uchar* lm = g.constScanLine(y - 1); + const uchar* lc = g.constScanLine(y); + const uchar* lp = g.constScanLine(y + 1); + for (int x = 1; x + 1 < g.width(); ++x) { + if (lc[x] <= 12) continue; // background + acc += std::fabs(4.0 * lc[x] - lc[x-1] - lc[x+1] - lm[x] - lp[x]); + ++n; + } + } + return n > 0 ? acc / double(n) : -1.0; + }; // Two render styles per view: the shaded render (materials intact — // carries texture contrast MediaPipe likes) and the fog depth-map render // (pure geometry statue — immune to broken normals / inconsistent winding @@ -115,6 +138,16 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, entity, kRenderSize, view, &err, focus.isNull() ? nullptr : &focus); if (vrr.depth.isNull()) continue; + if (depthMode) { + const double det = depthDetail(vrr.depth); + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] detail view=%s score=%.2f\n", + view.name, det); + if (det > bestDetail) { + bestDetail = det; + bestDetailCamDir = vrr.camDirection; + } + } // Flatness sanity: a blown-out / silhouette render (near-zero // intensity variance inside the subject) carries no facial // features — MediaPipe false-positives on such blobs with high @@ -159,6 +192,95 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, } } } + // Facing signal: the face points TOWARD the winning view's camera + // (= against its look direction). When NO view produced positive face + // evidence (all logits negative — MediaPipe can't read stylized faces), + // the logit "winner" is noise; fall back to the depth-DETAIL winner + // instead (the face side out-details the smooth back of the head). + // Exposed in MESH-LOCAL space even when the landmarks themselves are too + // weak to use — the proportional-default marker placement needs only the + // facing. + { + Ogre::Vector3 camDir = rr.depth.isNull() ? Ogre::Vector3::ZERO + : rr.camDirection; + if (bestLogit < 0.f) { + // No positive face evidence anywhere (stylized / covered faces) — + // the logit "winner" is noise. For a FULL-BODY character the feet + // are the strongest facing cue: toes extend forward of the ankle. + // Compare the horizontal centroid of the feet slab (lowest 8% of + // the body) against the ankle slab above it. + bool feetResolved = false; + const FaceRigGeometry full = extractGeometry(entity); + if (full.valid()) { + const int n = int(full.userV.size() / 3); + float bLoY = 1e30f, bHiY = -1e30f; + for (int i = 0; i < n; ++i) { + bLoY = std::min(bLoY, full.userV[size_t(i)*3+1]); + bHiY = std::max(bHiY, full.userV[size_t(i)*3+1]); + } + // head height from the head verts we render (localV) + float hLoY = 1e30f, hHiY = -1e30f; + for (int i = 0; i < int(localV.size()/3); ++i) { + hLoY = std::min(hLoY, localV[size_t(i)*3+1]); + hHiY = std::max(hHiY, localV[size_t(i)*3+1]); + } + const float bodyH = bHiY - bLoY, headH = hHiY - hLoY; + if (bodyH > 2.5f * headH && bodyH > 1e-6f) { + const float feetTop = bLoY + 0.08f * bodyH; + const float ankleTop = bLoY + 0.16f * bodyH; + double fx = 0, fz = 0, ax = 0, az = 0; + long nf = 0, na = 0; + for (int i = 0; i < n; ++i) { + const float y = full.userV[size_t(i)*3+1]; + if (y < feetTop) { + fx += full.userV[size_t(i)*3]; fz += full.userV[size_t(i)*3+2]; ++nf; + } else if (y < ankleTop) { + ax += full.userV[size_t(i)*3]; az += full.userV[size_t(i)*3+2]; ++na; + } + } + if (nf > 8 && na > 8) { + const double dx = fx/nf - ax/na, dz = fz/nf - az/na; + const double len = std::sqrt(dx*dx + dz*dz); + if (len > 0.01 * bodyH) { + // LOCAL-space facing; convert to a WORLD camDir + // (the block below converts back) by mapping + // through the entity transform. + const Ogre::Vector3 o = world * Ogre::Vector3::ZERO; + Ogre::Vector3 faceW = + (world * Ogre::Vector3(float(dx), 0, float(dz))) - o; + if (!faceW.isZeroLength()) { + camDir = -faceW; // camera looks against facing + feetResolved = true; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, + "[facerig] facing from FEET dir " + "local=(%.2f,0,%.2f)\n", dx/len, dz/len); + } + } + } + } + } + if (!feetResolved && bestDetail >= 0.0 + && !bestDetailCamDir.isZeroLength()) { + camDir = bestDetailCamDir; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] facing from depth detail " + "(bestLogit=%.1f, detail=%.2f)\n", + bestLogit, bestDetail); + } + } + if (!camDir.isZeroLength()) { + const Ogre::Matrix4 wInv = world.inverse(); + const Ogre::Vector3 faceW = -camDir; + const Ogre::Vector3 o = wInv * Ogre::Vector3::ZERO; + Ogre::Vector3 faceL = (wInv * faceW) - o; + if (!faceL.isZeroLength()) { + faceL.normalise(); + out.faceDirLocal = {faceL.x, faceL.y, faceL.z}; + out.faceDirValid = true; + } + } + } if (rr.depth.isNull() || !lr.ok || lr.points.empty()) return out; out.confidence = lr.confidence; #ifndef NDEBUG @@ -659,6 +781,32 @@ std::vector seedFaceMarkers( // produce a good rig on their own (the cartoon-face path), and the user // refines from there. placed=true so they act as anchors even if the // user rigs without touching them. + // + // FACING-AWARE: the straight box mapping assumes the user's face + // points the template's way (+Z); a backwards-facing import (glb + // round-trips flip facing) landed every default on the BACK of the + // head. Even a too-weak-to-trust detection still tells us which + // render view scored best — yaw the template coordinates to that + // cardinal facing before the box mapping. + int yaw = 0; // 0:+Z (template) 1:-Z 2:+X 3:-X + if (ulm.faceDirValid) { + const float fx = ulm.faceDirLocal[0], fz = ulm.faceDirLocal[2]; + yaw = (std::fabs(fz) >= std::fabs(fx)) ? (fz >= 0.f ? 0 : 1) + : (fx >= 0.f ? 2 : 3); + } + auto yawRot = [yaw](std::array p) -> std::array { + switch (yaw) { + case 1: return {-p[0], p[1], -p[2]}; // 180° + case 2: return { p[2], p[1], -p[0]}; // +Z → +X + case 3: return {-p[2], p[1], p[0]}; // +Z → -X + default: return p; + } + }; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] defaults: faceDirValid=%d " + "dir=(%.2f,%.2f,%.2f) yaw=%d\n", + ulm.faceDirValid ? 1 : 0, ulm.faceDirLocal[0], + ulm.faceDirLocal[1], ulm.faceDirLocal[2], yaw); std::array lo{1e30f,1e30f,1e30f}, hi{-1e30f,-1e30f,-1e30f}; const int unv = int(userLocalV.size()/3); for (int i = 0; i < unv; ++i) @@ -667,17 +815,23 @@ std::vector seedFaceMarkers( hi[a] = std::max(hi[a], userLocalV[size_t(i)*3+a]); } std::array tlo{1e30f,1e30f,1e30f}, thi{-1e30f,-1e30f,-1e30f}; - for (int i = 0; i < tvc; ++i) + for (int i = 0; i < tvc; ++i) { + const std::array tv = yawRot({tn[size_t(i)*3], + tn[size_t(i)*3+1], + tn[size_t(i)*3+2]}); for (int a = 0; a < 3; ++a) { - tlo[a] = std::min(tlo[a], tn[size_t(i)*3+a]); - thi[a] = std::max(thi[a], tn[size_t(i)*3+a]); + tlo[a] = std::min(tlo[a], tv[size_t(a)]); + thi[a] = std::max(thi[a], tv[size_t(a)]); } + } for (auto& m : markers) { if (m.tmplVertex < 0) continue; + const std::array tv = yawRot({tn[size_t(m.tmplVertex)*3], + tn[size_t(m.tmplVertex)*3+1], + tn[size_t(m.tmplVertex)*3+2]}); for (int a = 0; a < 3; ++a) { - const float tv = tn[size_t(m.tmplVertex)*3+a]; const float f = (thi[a]-tlo[a]) > 1e-6f - ? (tv - tlo[a]) / (thi[a]-tlo[a]) : 0.5f; + ? (tv[size_t(a)] - tlo[a]) / (thi[a]-tlo[a]) : 0.5f; m.userPos[size_t(a)] = lo[a] + f * (hi[a]-lo[a]); } m.placed = true; diff --git a/src/FaceRig/FaceRigLandmarks.h b/src/FaceRig/FaceRigLandmarks.h index 5b923f9e6..677e60626 100644 --- a/src/FaceRig/FaceRigLandmarks.h +++ b/src/FaceRig/FaceRigLandmarks.h @@ -47,6 +47,14 @@ struct MeshLandmarks { std::vector> points; std::vector valid; float confidence = 0.0f; + // Direction the FACE points, in MESH-LOCAL space — derived from the view + // whose render won the presence-logit ranking (the face points toward that + // camera). Valid even when `ok` is false: a weak detection is still a + // usable FACING signal, and the proportional-default marker placement + // needs it so defaults land on the face instead of the back of the head + // for meshes that don't face the template's +Z. + std::array faceDirLocal{0, 0, 1}; + bool faceDirValid = false; bool ok = false; }; From a3e96925df08b24afc524a8277f436a5eec5fbec Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 16:19:58 -0400 Subject: [PATCH 28/38] feat(#889): active-viewport camera as the primary facing hint for marker seeding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per user request: when rigging, the user has orbited the viewport to LOOK AT the face — that camera direction is a stronger facing prior than any geometric heuristic. detectMeshLandmarks now uses the active viewport camera's direction as the facing whenever detection isn't conclusive (presence logit below the strong-face bar), ahead of the feet-direction and depth-detail fallbacks. Headless CLI has no active widget (new non-creating TransformOperator::getSingletonPtr()), so it keeps the feet/detail ladder unchanged. Co-Authored-By: Claude Fable 5 --- src/FaceRig/FaceRigLandmarks.cpp | 27 ++++++++++++++++++++++++++- src/TransformOperator.h | 3 +++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index 90fe2f49e..ed39748e2 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -6,6 +6,8 @@ #include "../Manager.h" #include "../MeshDepthRenderer.h" +#include "../TransformOperator.h" +#include "../OgreWidget.h" #include #include @@ -203,7 +205,30 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, { Ogre::Vector3 camDir = rr.depth.isNull() ? Ogre::Vector3::ZERO : rr.camDirection; - if (bestLogit < 0.f) { + // The ACTIVE VIEWPORT camera is the strongest user-intent hint: the + // user orbits to LOOK AT the face before rigging, so the face points + // toward that camera. Trust it over every geometric fallback whenever + // detection itself isn't conclusive (logit below the strong-face bar). + bool vpResolved = false; + if (bestLogit < kStrongFaceLogit) { + if (auto* to = TransformOperator::getSingletonPtr()) { + if (auto* w = to->getActiveWidget()) { + if (w->getViewport() && w->getViewport()->getCamera()) { + const Ogre::Vector3 d = + w->getViewport()->getCamera()->getDerivedDirection(); + if (!d.isZeroLength()) { + camDir = d; // camera looks toward the face + vpResolved = true; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, + "[facerig] facing from VIEWPORT camera " + "(bestLogit=%.1f)\n", bestLogit); + } + } + } + } + } + if (!vpResolved && bestLogit < 0.f) { // No positive face evidence anywhere (stylized / covered faces) — // the logit "winner" is noise. For a FULL-BODY character the feet // are the strongest facing cue: toes extend forward of the ankle. diff --git a/src/TransformOperator.h b/src/TransformOperator.h index 94a4b8a31..026bec423 100755 --- a/src/TransformOperator.h +++ b/src/TransformOperator.h @@ -33,6 +33,9 @@ class TransformOperator : public QObject, public QtMouseListener public: static TransformOperator* getSingleton(); + /// Non-creating accessor: null until getSingleton() first runs. Use from + /// paths that must not construct the gizmo machinery (headless CLI). + static TransformOperator* getSingletonPtr() { return m_pSingleton; } static void kill(); enum TransformState From 38c4d97432684413dde6c73a69b739b98bb8389f Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 16:49:54 -0400 Subject: [PATCH 29/38] =?UTF-8?q?fix(#889):=20depth-mode=20detections=20ca?= =?UTF-8?q?n=20no=20longer=20decide=20facing=20=E2=80=94=20Rumba=20back-of?= =?UTF-8?q?-head=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fog depth statue has no texture and MediaPipe false-positives on smooth domes: Rumba Dancing's occiput DEPTH render scored presence logit +8.0 (above the strong-face bar) while its true stylized face scored -3.5 — so the facing locked onto the back of the head and every default marker followed. Verified against the GUI capture dumps: the true face is on the opposite view. Facing ladder is now: (1) strong SHADED-mode detection only (textured render — false positives measured well below the bar); (2) active viewport camera; (3) feet direction — magnitude gate lowered to 0.5% of body height (the toe-forward offset on the dance-pose Rumba rig is 1.9cm on a 2m body, just under the old 1% gate, but its direction is reliable); (4) depth- detail winner. Depth-mode detections still feed landmark positions (the constellation gate protects those), never facing. Verified: Rumba -> feet, yaw 180 (correct); bandit -> feet, +Z (correct); ARKit reference -> strong shaded, 13/13 trusted (unchanged). Co-Authored-By: Claude Fable 5 --- src/FaceRig/FaceRigLandmarks.cpp | 49 ++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index ed39748e2..2151071fc 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -94,6 +94,14 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, MeshDepthRenderer::RenderResult rr; LandmarkResult lr; float bestLogit = -1e9f; + // FACING may only be claimed by SHADED-mode detections: the fog depth + // statue has no texture and MediaPipe false-positives on smooth domes + // (measured: Rumba's occiput depth render scored logit +8 while its true + // stylized face scored -3.5 — the facing flipped to the back of the + // head). Depth-mode detections still feed LANDMARKS (the constellation + // gate protects those), just never the facing decision. + float bestShadedLogit = -1e9f; + Ogre::Vector3 bestShadedCamDir = Ogre::Vector3::ZERO; // Geometric facing fallback: mean |Laplacian| of the fog depth render // over subject pixels. The face side of a head (nose, brows, lips, chin) // carries far more depth detail than the smooth occiput — used to decide @@ -187,6 +195,10 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, + (depthMode ? "depth." : "shaded.") + view.name + ".png"); if (!vlr.ok || vlr.points.empty()) continue; + if (!depthMode && vlr.presenceLogit > bestShadedLogit) { + bestShadedLogit = vlr.presenceLogit; + bestShadedCamDir = vrr.camDirection; + } if (vlr.presenceLogit > bestLogit) { bestLogit = vlr.presenceLogit; rr = std::move(vrr); @@ -203,14 +215,24 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, // weak to use — the proportional-default marker placement needs only the // facing. { - Ogre::Vector3 camDir = rr.depth.isNull() ? Ogre::Vector3::ZERO - : rr.camDirection; + // Facing ladder: (1) a strong SHADED-mode detection; (2) the active + // viewport camera; (3) feet direction; (4) depth-detail winner; (5) + // the overall logit winner as a last resort. + Ogre::Vector3 camDir = Ogre::Vector3::ZERO; + const bool strongShaded = bestShadedLogit >= kStrongFaceLogit + && !bestShadedCamDir.isZeroLength(); + if (strongShaded) { + camDir = bestShadedCamDir; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] facing from SHADED detection " + "(logit=%.1f)\n", bestShadedLogit); + } // The ACTIVE VIEWPORT camera is the strongest user-intent hint: the // user orbits to LOOK AT the face before rigging, so the face points // toward that camera. Trust it over every geometric fallback whenever - // detection itself isn't conclusive (logit below the strong-face bar). + // detection itself isn't conclusive. bool vpResolved = false; - if (bestLogit < kStrongFaceLogit) { + if (!strongShaded) { if (auto* to = TransformOperator::getSingletonPtr()) { if (auto* w = to->getActiveWidget()) { if (w->getViewport() && w->getViewport()->getCamera()) { @@ -228,7 +250,7 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, } } } - if (!vpResolved && bestLogit < 0.f) { + if (!strongShaded && !vpResolved) { // No positive face evidence anywhere (stylized / covered faces) — // the logit "winner" is noise. For a FULL-BODY character the feet // are the strongest facing cue: toes extend forward of the ankle. @@ -250,6 +272,10 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, hHiY = std::max(hHiY, localV[size_t(i)*3+1]); } const float bodyH = bHiY - bLoY, headH = hHiY - hLoY; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] feet gate: bodyH=%.2f " + "headH=%.2f ratio=%.2f\n", + bodyH, headH, headH > 0 ? bodyH/headH : 0.f); if (bodyH > 2.5f * headH && bodyH > 1e-6f) { const float feetTop = bLoY + 0.08f * bodyH; const float ankleTop = bLoY + 0.16f * bodyH; @@ -263,10 +289,21 @@ MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, ax += full.userV[size_t(i)*3]; az += full.userV[size_t(i)*3+2]; ++na; } } + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] feet slabs: nf=%ld " + "na=%ld\n", nf, na); if (nf > 8 && na > 8) { const double dx = fx/nf - ax/na, dz = fz/nf - az/na; const double len = std::sqrt(dx*dx + dz*dz); - if (len > 0.01 * bodyH) { + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] feet dir: " + "d=(%.3f,%.3f) len=%.3f min=%.3f\n", + dx, dz, len, 0.005 * bodyH); + // 0.5% of body height: the toe-forward offset is small + // on dance-pose rigs (Rumba: 1.9cm on a 2m body) but + // its DIRECTION is reliable; only reject a truly + // degenerate (near-zero) offset. + if (len > 0.005 * bodyH) { // LOCAL-space facing; convert to a WORLD camDir // (the block below converts back) by mapping // through the entity transform. From 0b8cead9d8c7d414354918b81021aa6b8c26b4b7 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 18:10:14 -0400 Subject: [PATCH 30/38] =?UTF-8?q?fix(#889):=20snap=20default=20markers=20o?= =?UTF-8?q?nto=20the=20head=20surface=20=E2=80=94=20cigar/protrusion=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proportional defaults took their DEPTH from the head bounding box, so any protrusion along the facing axis (Rumba's cigar, long noses, hair) inflated the box and floated every default off the face by the protrusion length. Each marker is now ray-cast from outside the box along the facing direction and takes the first surface hit as its position — defaults land ON the face. The marker-sim gains a per-marker surface-distance probe; Rumba's defaults improve from cigar-length offsets to <= 0.02 units (max ~3% of head height, most under 1%). A protrusion directly in front of a feature (the cigar over the lips) can still catch that feature's ray — those markers land on the protrusion and need a manual nudge; everything lateral/above is unaffected. Co-Authored-By: Claude Fable 5 --- src/CLIPipeline.cpp | 15 +++++++++++++ src/FaceRig/FaceRigLandmarks.cpp | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 93577ce81..aff1fbb49 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -9601,6 +9601,21 @@ int CLIPipeline::cmdFaceRig(int argc, char* argv[]) const auto anchors = FaceRig::anchorsFromMarkers(markers, tmpl); err() << "[sim] markers=" << markers.size() << " anchors=" << anchors.size() << " seedConfident=" << confident << Qt::endl; + // surface-distance probe: how far does each seeded marker float off + // the head mesh? (box-mapped defaults used to hover off the face when + // protrusions inflate the head box) + for (const auto& m : markers) { + if (!m.placed) continue; + float best = 1e30f; + for (size_t i = 0; i + 2 < headV.size(); i += 3) { + const float dx = headV[i] - m.userPos[0]; + const float dy = headV[i+1] - m.userPos[1]; + const float dz = headV[i+2] - m.userPos[2]; + best = std::min(best, dx*dx + dy*dy + dz*dz); + } + err() << "[sim] surfdist '" << m.label << "' = " + << std::sqrt(best) << Qt::endl; + } // side probe: character faces -Z, up +Y => character-LEFT = -X. for (const auto& m : markers) if (m.tmplVertex >= 0) diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index 2151071fc..de93430e2 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -886,6 +886,19 @@ std::vector seedFaceMarkers( thi[a] = std::max(thi[a], tv[size_t(a)]); } } + // Depth axis + ray setup for surface snapping: the box mapping puts + // markers at the box's proportional DEPTH, but protrusions (a cigar, + // a long nose, hair) inflate the head box along the facing axis and + // every default then floats off the face. Ray-cast each marker from + // outside the box along the facing direction and take the first + // surface hit as its depth instead. + const int depthAxis = (yaw <= 1) ? 2 : 0; + const float depthSign = (yaw == 0 || yaw == 2) ? 1.0f : -1.0f; + const float margin = 0.25f * (hi[size_t(depthAxis)] - lo[size_t(depthAxis)]); + Ogre::Vector3 rayDir = Ogre::Vector3::ZERO; + rayDir[depthAxis] = -depthSign; // from the face side into the head + const int unvTot = int(userLocalV.size() / 3); + for (auto& m : markers) { if (m.tmplVertex < 0) continue; const std::array tv = yawRot({tn[size_t(m.tmplVertex)*3], @@ -896,6 +909,31 @@ std::vector seedFaceMarkers( ? (tv[size_t(a)] - tlo[a]) / (thi[a]-tlo[a]) : 0.5f; m.userPos[size_t(a)] = lo[a] + f * (hi[a]-lo[a]); } + // Snap to the head surface along the facing axis. + Ogre::Vector3 o(m.userPos[0], m.userPos[1], m.userPos[2]); + o[depthAxis] = depthSign > 0 + ? hi[size_t(depthAxis)] + margin + : lo[size_t(depthAxis)] - margin; + float bestT = std::numeric_limits::max(); + bool hitAny = false; + Ogre::Vector3 hit; + for (size_t fI = 0; fI + 2 < userLocalF.size(); fI += 3) { + const int ia = userLocalF[fI], ib = userLocalF[fI+1], + ic = userLocalF[fI+2]; + if (ia < 0 || ib < 0 || ic < 0 + || ia >= unvTot || ib >= unvTot || ic >= unvTot) continue; + auto vAt = [&](int k) { + return Ogre::Vector3(userLocalV[size_t(k)*3], + userLocalV[size_t(k)*3+1], + userLocalV[size_t(k)*3+2]); + }; + float t; + if (rayTri(o, rayDir, vAt(ia), vAt(ib), vAt(ic), t) && t < bestT) { + bestT = t; hit = o + rayDir * t; hitAny = true; + } + } + if (hitAny) + m.userPos = {hit.x, hit.y, hit.z}; m.placed = true; } } From ce76a304fc60befffb8068321acea8431fe63c1b Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 21:50:50 -0400 Subject: [PATCH 31/38] feat(#889): consensus outlier correction for trusted marker seeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Individual MediaPipe detections on low-contrast renders drift most at the eye/mouth corners (measured on the ARKit reference, where ground truth is the template itself: eye-outer seed error up to 0.59 units vs ~0.07 for the nose). The trusted-seed path now fits the same similarity model the constellation gate uses (centroid + scale) and replaces any detection deviating from its constellation prediction by more than 2x the median deviation with that prediction, snapped to the head surface. Reference seed errors: Right/Left eye outer 0.59/0.39 -> 0, Left mouth corner 0.23 -> 0; worst remaining marker 0.16 (upper lip — lip-line landmarks are genuinely ambiguous on untextured renders). Non-outlier detections are kept verbatim so real anatomical variation isn't pulled toward the rigid template layout. Marker-sim gains a seed-error probe (exact, reference-only). A head-on->raking capture-light experiment was measured net-neutral and reverted. Co-Authored-By: Claude Fable 5 --- src/CLIPipeline.cpp | 14 ++++++ src/FaceRig/FaceRigLandmarks.cpp | 80 +++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index aff1fbb49..5b35b76f5 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -9601,6 +9601,20 @@ int CLIPipeline::cmdFaceRig(int argc, char* argv[]) const auto anchors = FaceRig::anchorsFromMarkers(markers, tmpl); err() << "[sim] markers=" << markers.size() << " anchors=" << anchors.size() << " seedConfident=" << confident << Qt::endl; + // seed-accuracy probe: when the rig target IS the template geometry + // (the ARKit reference), each marker's true position is its template + // vertex — print the seeding error per marker. + if (qEnvironmentVariableIntValue("QTMESH_FACERIG_MARKER_SIM") == 1) { + for (const auto& m : markers) { + if (!m.placed || m.tmplVertex < 0) continue; + const auto& tv = tmpl.neutral(); + const float dx = tv[size_t(m.tmplVertex)*3] - m.userPos[0]; + const float dy = tv[size_t(m.tmplVertex)*3+1] - m.userPos[1]; + const float dz = tv[size_t(m.tmplVertex)*3+2] - m.userPos[2]; + err() << "[sim] seederr '" << m.label << "' = " + << std::sqrt(dx*dx + dy*dy + dz*dz) << Qt::endl; + } + } // surface-distance probe: how far does each seeded marker float off // the head mesh? (box-mapped defaults used to hover off the face when // protrusions inflate the head box) diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index de93430e2..8cf3d651b 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -830,12 +830,88 @@ std::vector seedFaceMarkers( confident ? "trusted" : "using proportional defaults"); if (confident) { + // Consensus outlier correction: fit the similarity model the + // constellation gate already uses (centroid + scale, no rotation) + // and predict each marker from the TEMPLATE layout. Individual + // detections on low-contrast renders drift most at the eye/mouth + // corners (measured up to 0.59 units on the reference vs ~0.07 + // for the nose); a detection that deviates from the consensus + // prediction by more than 2x the median is detector noise — replace + // it with the prediction, snapped to the head surface. + std::array cC{0,0,0}, cU{0,0,0}; + const int np = int(detC.size()); + for (int i = 0; i < np; ++i) + for (int d = 0; d < 3; ++d) { + cC[size_t(d)] += detC[size_t(i)][size_t(d)] / np; + cU[size_t(d)] += detU[size_t(i)][size_t(d)] / np; + } + double sC = 0, sU = 0; + for (int i = 0; i < np; ++i) { + double dc = 0, du = 0; + for (int d = 0; d < 3; ++d) { + const double a = detC[size_t(i)][size_t(d)] - cC[size_t(d)]; + const double b = detU[size_t(i)][size_t(d)] - cU[size_t(d)]; + dc += a*a; du += b*b; + } + sC += std::sqrt(dc); sU += std::sqrt(du); + } + const double scale = (sC > 1e-12) ? sU / sC : 1.0; + + auto predictOf = [&](int tmplVertex) -> std::array { + std::array p; + for (int d = 0; d < 3; ++d) + p[size_t(d)] = float((double(tn[size_t(tmplVertex)*3 + d]) + - cC[size_t(d)]) * scale + cU[size_t(d)]); + return p; + }; + std::vector devs; for (auto& m : markers) { const int i = m.mediapipeIndex; - if (i >= 0 && i < int(ulm.points.size()) && ulm.valid[size_t(i)]) { + if (i < 0 || i >= int(ulm.points.size()) || !ulm.valid[size_t(i)] + || m.tmplVertex < 0) continue; + const auto pred = predictOf(m.tmplVertex); + const auto& det = ulm.points[size_t(i)]; + const double dx = det[0]-pred[0], dy = det[1]-pred[1], + dz = det[2]-pred[2]; + devs.push_back(std::sqrt(dx*dx + dy*dy + dz*dz)); + } + std::vector sorted = devs; + std::sort(sorted.begin(), sorted.end()); + const double median = sorted.empty() ? 0.0 + : sorted[sorted.size() / 2]; + const double outlierAt = std::max(2.0 * median, 1e-9); + + size_t di = 0; + for (auto& m : markers) { + const int i = m.mediapipeIndex; + if (i < 0 || i >= int(ulm.points.size()) || !ulm.valid[size_t(i)]) + continue; + if (m.tmplVertex >= 0 && di < devs.size() + && devs[di] > outlierAt) { + // consensus prediction, snapped onto the head surface + std::array p = predictOf(m.tmplVertex); + float best = 1e30f; + std::array snap = p; + for (size_t v = 0; v + 2 < userLocalV.size(); v += 3) { + const float dx = userLocalV[v] - p[0]; + const float dy = userLocalV[v+1] - p[1]; + const float dz = userLocalV[v+2] - p[2]; + const float d2 = dx*dx + dy*dy + dz*dz; + if (d2 < best) { + best = d2; + snap = {userLocalV[v], userLocalV[v+1], userLocalV[v+2]}; + } + } + m.userPos = snap; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] seed outlier '%s' " + "dev=%.3f (median %.3f) -> consensus\n", + m.label.toUtf8().constData(), devs[di], median); + } else { m.userPos = ulm.points[size_t(i)]; - m.placed = true; } + if (m.tmplVertex >= 0) ++di; + m.placed = true; } } else { // Garbage / weak detection: seed EVERY marker at the head-box-projected From da3d183770abb758e1a97f911826e55bb2b56254 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 22:56:52 -0400 Subject: [PATCH 32/38] feat(#889): canonical template marker vertices + detector-bias calibration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The user's markers were systematically one anatomical step too HIGH on the lower face (lower lip at the upper lip, chin at the lower lip). Root cause: the template-side marker vertices were resolved by DETECTING on the template's own untextured render — so the detector's statue bias defined the 'ground truth' user detections were matched against, hiding the bias from every measurement while showing it plainly in the GUI. Two changes: - canonicalTemplateVertex(): the 13 marker vertices are now CONSTANTS for the ICT topology, derived offline from the template's own blendshape deltas (mouthSmile peak = mouth corner, mouthLowerDown peak = lower lip, eyeBlink lid extremes = eye corners, gnathion = lowest front midline vertex). Falls back to detection for non-ICT templates. - Detector-bias calibration: the template is still rendered + detected every run — as a CALIBRATION TARGET. Each marker's bias (symmetrized template detection - canonical vertex) is scaled by the template->user constellation size ratio and subtracted from the user detection; the systematic statue bias cancels, leaving actual anatomy. Reference seed errors (vs true anatomical positions): chin 0.043, lower lip 0.091, mouth corners 0.13-0.16, eye outers 0 (consensus) — the lower-face cluster shift is gone; constellation gate trusts again (residual 0.027). Rumba/bandit default paths unchanged. Co-Authored-By: Claude Fable 5 --- src/FaceRig/FaceRigLandmarks.cpp | 168 +++++++++++++++++++++++++------ 1 file changed, 136 insertions(+), 32 deletions(-) diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp index 8cf3d651b..effefcecc 100644 --- a/src/FaceRig/FaceRigLandmarks.cpp +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -618,6 +618,30 @@ std::vector buildLandmarkAnchors( // are the CHARACTER'S RIGHT (measured on the template: they sit at +X = // character-right). Labels here are in CHARACTER space (what a user placing // markers on a model naturally means by left/right). +int canonicalTemplateVertex(int mpIndex) +{ + // ICT-FaceKit topology (26,719 verts). Derived offline from the packed + // template's own blendshape deltas + midline geometry; side assignment + // matches the catalog's measured detector convention (anchorsFromMarkers' + // mirror check absorbs a global L/R flip regardless). + switch (mpIndex) { + case 1: return 4841; // nose tip (front-most midline) + case 152: return 961; // chin / gnathion (lowest front midline) + case 13: return 5829; // upper lip (mouthUpperUp peak, midline) + case 14: return 5945; // lower lip (mouthLowerDown peak, midline) + case 10: return 2138; // forehead (midline above brows) + case 33: return 2798; // eye outer corner (+X lid extreme) + case 133: return 3585; // eye inner corner (+X) + case 61: return 6156; // mouth corner (+X, mouthSmile peak) + case 105: return 2590; // brow (+X, browOuterUp peak) + case 263: return 557; // eye outer corner (-X) + case 362: return 1370; // eye inner corner (-X) + case 291: return 5651; // mouth corner (-X) + case 334: return 349; // brow (-X) + default: return -1; + } +} + const std::vector>& faceMarkerCatalog() { static const std::vector> kCatalog = { @@ -743,17 +767,40 @@ std::vector seedFaceMarkers( if (!FaceLandmarkDetector::backendAvailable()) return markers; { FaceLandmarkDetector probe; if (!probe.load()) return markers; } - // Template detection resolves each marker's TEMPLATE vertex (reliable — the - // ICT template is a real human face). - Ogre::Entity* tent = templateEntity(tmpl.neutral(), tmpl.faces()); - if (!tent) return markers; - if (auto* tnode = tent->getParentSceneNode()) tnode->setVisible(true); - const MeshLandmarks tlm = detectMeshLandmarks(tent, tmpl.neutral(), - std::vector(tmpl.faces())); - if (auto* tnode = tent->getParentSceneNode()) tnode->setVisible(false); - const int tvc = tmpl.vertexCount(); const auto& tn = tmpl.neutral(); + + // Template-side marker vertices: CANONICAL constants for the ICT topology, + // derived offline from the template's own blendshape deltas (the + // mouthSmile peak IS the mouth corner, the eyeBlink-moved lid's lateral + // extremes ARE the eye corners, gnathion = lowest front midline vertex). + // Detection on the template's untextured render carries a systematic + // detector bias (lower-face landmarks drift UP one anatomical step), and + // that bias used to define the "ground truth" every user marker was + // matched against. Falls back to template detection for a non-ICT + // template (vertex count mismatch). + const bool canonicalOk = (tvc == 26719); + if (canonicalOk) { + for (auto& m : markers) + m.tmplVertex = canonicalTemplateVertex(m.mediapipeIndex); + } + // Template detection still runs even with canonical vertices: the + // template is rendered + detected EXACTLY like the user mesh, so the + // difference between where the detector puts a marker on the template and + // its canonical vertex measures the detector's systematic bias on this + // render style (lower-face landmarks drift up one anatomical step on + // untextured statues) — which we then subtract from the user detections. + MeshLandmarks tlm; + { + Ogre::Entity* tent = templateEntity(tmpl.neutral(), tmpl.faces()); + if (!tent) return markers; + if (auto* tnode = tent->getParentSceneNode()) tnode->setVisible(true); + tlm = detectMeshLandmarks(tent, tmpl.neutral(), + std::vector(tmpl.faces())); + if (auto* tnode = tent->getParentSceneNode()) tnode->setVisible(false); + } + std::vector> tlmSym(markers.size(), {0,0,0}); + std::vector tlmSymOk(markers.size(), 0); if (tlm.ok) { // SYMMETRIZE the detected template landmarks before resolving vertices: // the ICT template is x-symmetric (midline at x=0), but MediaPipe @@ -768,7 +815,8 @@ std::vector seedFaceMarkers( out = tlm.points[size_t(mpIdx)]; return true; }; - for (auto& m : markers) { + for (size_t k = 0; k < markers.size(); ++k) { + auto& m = markers[k]; const int i = m.mediapipeIndex; std::array p; if (!detected(i, p)) continue; @@ -789,7 +837,10 @@ std::vector seedFaceMarkers( p = { sign * xm, ym, zm }; } // single-sided detection: use as-is } - m.tmplVertex = nearestTemplateVertex(tn, tvc, p); + tlmSym[k] = p; + tlmSymOk[k] = 1; + if (!canonicalOk) + m.tmplVertex = nearestTemplateVertex(tn, tvc, p); } } @@ -799,20 +850,75 @@ std::vector seedFaceMarkers( // silently poison the warp/fit (measured: jawOpen deltas 50x too small). const MeshLandmarks ulm = detectMeshLandmarks(userEntity, userLocalV, userLocalF); - int seeded = 0; - std::vector> detC, detU; // template/user pairs + // Detector-bias correction: transfer each marker's measured template-side + // bias (symmetrized detection − canonical vertex) into the user's frame + // (scaled by the template→user constellation size ratio) and subtract it + // from the user detection. Template and user go through the same render + + // detector, so the systematic statue bias cancels; what remains is the + // user's actual anatomy. + std::vector> uCorr(markers.size(), {0,0,0}); + std::vector uOk(markers.size(), 0); if (ulm.ok) { - for (auto& m : markers) { - const int i = m.mediapipeIndex; - if (i >= 0 && i < int(ulm.points.size()) && ulm.valid[size_t(i)] - && m.tmplVertex >= 0) { - ++seeded; - detC.push_back({tn[size_t(m.tmplVertex)*3], - tn[size_t(m.tmplVertex)*3+1], - tn[size_t(m.tmplVertex)*3+2]}); - detU.push_back(ulm.points[size_t(i)]); + // constellation size ratio from raw detected pairs (bias-consistent + // on both sides, so the ratio is unaffected by the bias itself) + double sT = 0, sUsr = 0; + { + std::array cT{0,0,0}, cUsr{0,0,0}; + int n = 0; + for (size_t k = 0; k < markers.size(); ++k) { + const int i = markers[k].mediapipeIndex; + if (!tlmSymOk[k] || i < 0 || i >= int(ulm.points.size()) + || !ulm.valid[size_t(i)]) continue; + for (int d = 0; d < 3; ++d) { + cT[size_t(d)] += tlmSym[k][size_t(d)]; + cUsr[size_t(d)] += ulm.points[size_t(i)][size_t(d)]; + } + ++n; + } + if (n >= 4) { + for (int d = 0; d < 3; ++d) { cT[size_t(d)] /= n; cUsr[size_t(d)] /= n; } + for (size_t k = 0; k < markers.size(); ++k) { + const int i = markers[k].mediapipeIndex; + if (!tlmSymOk[k] || i < 0 || i >= int(ulm.points.size()) + || !ulm.valid[size_t(i)]) continue; + double dt = 0, du = 0; + for (int d = 0; d < 3; ++d) { + const double a = tlmSym[k][size_t(d)] - cT[size_t(d)]; + const double b = ulm.points[size_t(i)][size_t(d)] - cUsr[size_t(d)]; + dt += a*a; du += b*b; + } + sT += std::sqrt(dt); sUsr += std::sqrt(du); + } } } + const double ratio = (canonicalOk && tlm.ok && sT > 1e-9) + ? sUsr / sT : 0.0; + for (size_t k = 0; k < markers.size(); ++k) { + const int i = markers[k].mediapipeIndex; + if (i < 0 || i >= int(ulm.points.size()) || !ulm.valid[size_t(i)]) + continue; + uCorr[k] = ulm.points[size_t(i)]; + uOk[k] = 1; + const int cv = markers[k].tmplVertex; + if (ratio > 0.0 && tlmSymOk[k] && cv >= 0) { + for (int d = 0; d < 3; ++d) { + const float bias = tlmSym[k][size_t(d)] + - tn[size_t(cv)*3 + size_t(d)]; + uCorr[k][size_t(d)] -= float(ratio * bias); + } + } + } + } + + int seeded = 0; + std::vector> detC, detU; // template/user pairs + for (size_t k = 0; k < markers.size(); ++k) { + if (!uOk[k] || markers[k].tmplVertex < 0) continue; + ++seeded; + const int cv = markers[k].tmplVertex; + detC.push_back({tn[size_t(cv)*3], tn[size_t(cv)*3+1], + tn[size_t(cv)*3+2]}); + detU.push_back(uCorr[k]); } // STRICT gate (0.12): the GUI render differs from headless (skybox, // lighting), so MediaPipe's garbage varies run-to-run and looser gates let @@ -865,12 +971,11 @@ std::vector seedFaceMarkers( return p; }; std::vector devs; - for (auto& m : markers) { - const int i = m.mediapipeIndex; - if (i < 0 || i >= int(ulm.points.size()) || !ulm.valid[size_t(i)] - || m.tmplVertex < 0) continue; + for (size_t k = 0; k < markers.size(); ++k) { + const auto& m = markers[k]; + if (!uOk[k] || m.tmplVertex < 0) continue; const auto pred = predictOf(m.tmplVertex); - const auto& det = ulm.points[size_t(i)]; + const auto& det = uCorr[k]; const double dx = det[0]-pred[0], dy = det[1]-pred[1], dz = det[2]-pred[2]; devs.push_back(std::sqrt(dx*dx + dy*dy + dz*dz)); @@ -882,10 +987,9 @@ std::vector seedFaceMarkers( const double outlierAt = std::max(2.0 * median, 1e-9); size_t di = 0; - for (auto& m : markers) { - const int i = m.mediapipeIndex; - if (i < 0 || i >= int(ulm.points.size()) || !ulm.valid[size_t(i)]) - continue; + for (size_t k = 0; k < markers.size(); ++k) { + auto& m = markers[k]; + if (!uOk[k]) continue; if (m.tmplVertex >= 0 && di < devs.size() && devs[di] > outlierAt) { // consensus prediction, snapped onto the head surface @@ -908,7 +1012,7 @@ std::vector seedFaceMarkers( "dev=%.3f (median %.3f) -> consensus\n", m.label.toUtf8().constData(), devs[di], median); } else { - m.userPos = ulm.points[size_t(i)]; + m.userPos = uCorr[k]; } if (m.tmplVertex >= 0) ++di; m.placed = true; From 2bc4c1f64fe6102bd9282a6475948700061de177 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 23:29:30 -0400 Subject: [PATCH 33/38] fix(#889): address code-review findings across the face-rig slices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - NonRigidICP: correspondence search now takes the K=4 nearest triangle CENTROIDS and picks by exact point-triangle distance — a single centroid winner mis-corresponds next to large/sliver triangles (PR #899). - DeformationTransfer: reject malformed buffers (trailing floats, indices outside [0,N)) before dereferencing; anchor the translation gauge of EVERY connected component, not just vertex 0 — the ICT template is dozens of islands (eyeballs, corneas, teeth) and each needs its own anchor row + rhs (PR #900). - FaceRigger: '--max-residual' now gates the MAX fit residual directly (it silently allowed 6x the supplied value); mean gated at a quarter of it. Healthy fits (max <= ~4%) pass the default 8% unchanged (PR #901). - FaceRigAttach::extractGeometry: skip a sharedVertexData pool no submesh references — orphan vertices joined the fit with no triangles (PR #901). - ArkitTemplate::ensureModelBlocking: a synchronous startDownload failure no longer blocks for the full 5-minute timeout ('done' guard, the LLM CLI pattern); Sentry breadcrumbs on download start/ok/fail (PR #898). - export-arkit-template.py: document that tongueOut is deliberately absent — ICT-FaceKit has no tongue expression, 51 real shapes (PR #898). - docs/FACE_RIG_SPIKE.md: escape |Δ| pipes that broke the results table (PR #897). Already addressed by earlier commits (noted for the record): the .arkit.json sidecar is now consumed on import (PR #903, commit e7abe015) and re-rigging replaces existing same-named targets instead of stacking (PR #902). FaceRig sources build into UnitTests via src/CMakeLists.txt — the 34 FaceRig/NRICP/DT tests run green. Verified: reference rig max residual 0.35% under the stricter gate; anchored Rumba sim 3.27%. Co-Authored-By: Claude Fable 5 --- docs/FACE_RIG_SPIKE.md | 2 +- scripts/export-arkit-template.py | 6 ++- src/FaceRig/ArkitTemplate.cpp | 23 +++++++--- src/FaceRig/DeformationTransfer.cpp | 70 ++++++++++++++++++++++------- src/FaceRig/DeformationTransfer.h | 1 + src/FaceRig/FaceRigAttach.cpp | 11 ++++- src/FaceRig/FaceRigger.cpp | 11 +++-- src/FaceRig/NonRigidICP.cpp | 65 ++++++++++++++++++++++++--- 8 files changed, 155 insertions(+), 34 deletions(-) diff --git a/docs/FACE_RIG_SPIKE.md b/docs/FACE_RIG_SPIKE.md index 64351134e..0b6d2c4ac 100644 --- a/docs/FACE_RIG_SPIKE.md +++ b/docs/FACE_RIG_SPIKE.md @@ -88,7 +88,7 @@ Template 26,719v → user 12,763v (60%-decimated, different topology): | ARKit shape | max displacement | verts moved | semantics check | |---|---|---|---| | NRICP surface fit | mean **0.003%** / max **0.59%** of diag | — | template lands on user surface | -| jawOpen | 9.0% | 6,771 | lower face ΔY −0.32 (drops), forehead |Δ| 0.002 (still) ✅ | +| jawOpen | 9.0% | 6,771 | lower face ΔY −0.32 (drops), forehead \|Δ\| 0.002 (still) ✅ | | mouthSmileLeft | 3.3% | 3,552 | localized to mouth ✅ | | eyeBlinkLeft | 3.8% | 1,328 | localized to eye ✅ | | browInnerUp | 1.3% | 2,181 | localized to brow ✅ | diff --git a/scripts/export-arkit-template.py b/scripts/export-arkit-template.py index 1275f11ee..ee41b8e0b 100644 --- a/scripts/export-arkit-template.py +++ b/scripts/export-arkit-template.py @@ -16,7 +16,7 @@ magic "QMFRT1\0\0" (8 bytes) int32 vertexCount V int32 faceCount F - int32 shapeCount S (== 52) + int32 shapeCount S (== 51 — ICT has no tongueOut) float32 neutral[V*3] (template neutral positions) int32 faces[F*3] (triangle vertex indices) then S shape records: @@ -72,6 +72,10 @@ ("mouthStretchLeft", ["mouthStretch_L"]), ("mouthStretchRight", ["mouthStretch_R"]), ("mouthUpperUpLeft", ["mouthUpperUp_L"]), ("mouthUpperUpRight", ["mouthUpperUp_R"]), ("noseSneerLeft", ["noseSneer_L"]), ("noseSneerRight", ["noseSneer_R"]), + # NOTE: tongueOut (ARKit channel 52) is deliberately ABSENT — ICT-FaceKit + # ships no tongue expression (the tongue isn't part of its face topology), + # so the packed template carries 51 real shapes. Emitting a zero-delta + # tongueOut would only inflate the bundle and show a dead slider. ] MAGIC = b"QMFRT1\0\0" diff --git a/src/FaceRig/ArkitTemplate.cpp b/src/FaceRig/ArkitTemplate.cpp index bf65a6783..33ad01661 100644 --- a/src/FaceRig/ArkitTemplate.cpp +++ b/src/FaceRig/ArkitTemplate.cpp @@ -1,6 +1,7 @@ #include "ArkitTemplate.h" #include "../ModelDownloader.h" +#include "../SentryReporter.h" #include #include @@ -155,15 +156,18 @@ QString ArkitTemplate::ensureModelBlocking() const QString url = base + QString::fromLatin1(kModelFile); const QString label = QStringLiteral("ARKit face template"); + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QStringLiteral("ARKit template download start")); + QEventLoop loop; - bool ok = false, timedOut = false; + bool ok = false, timedOut = false, done = false; auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, [&](const QString& name, const QString&) { - if (name == label) { ok = true; loop.quit(); } + if (name == label) { ok = true; done = true; loop.quit(); } }); auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, [&](const QString& name, const QString&) { - if (name == label) { ok = false; loop.quit(); } + if (name == label) { ok = false; done = true; loop.quit(); } }); QTimer timeout; timeout.setSingleShot(true); @@ -172,14 +176,23 @@ QString ArkitTemplate::ensureModelBlocking() timeout.start(300000); // 5 min — the template is ~17 MB dl->startDownload(url, dest, label); - loop.exec(); + // `done` guards the synchronous-failure case: startDownload can emit + // downloadError DURING the call (another download active, .part file + // unopenable) — entering the loop then would block for the full timeout. + if (!done) + loop.exec(); QObject::disconnect(onDone); QObject::disconnect(onErr); if (timedOut && dl) dl->cancelDownload(); - return (ok && !timedOut && QFileInfo::exists(dest)) ? dest : QString(); + const bool success = ok && !timedOut && QFileInfo::exists(dest); + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + success ? QStringLiteral("ARKit template download ok") + : QStringLiteral("ARKit template download failed%1") + .arg(timedOut ? QStringLiteral(" (timeout)") : QString())); + return success ? dest : QString(); } } // namespace FaceRig diff --git a/src/FaceRig/DeformationTransfer.cpp b/src/FaceRig/DeformationTransfer.cpp index ffcff82fc..064dc88ef 100644 --- a/src/FaceRig/DeformationTransfer.cpp +++ b/src/FaceRig/DeformationTransfer.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace FaceRig { @@ -81,6 +82,15 @@ bool DeformationTransfer::init(const std::vector& tmplNeutral, const int F = int(faces.size() / 3); if (N < 3 || F < 1 || int(fitted.size() / 3) != N) return false; + // Reject malformed buffers outright: trailing floats/indices that don't + // form whole vertices/triangles, and face indices outside [0, N) — those + // would be dereferenced by vert() below. + if (tmplNeutral.size() % 3 != 0 || faces.size() % 3 != 0 + || fitted.size() % 3 != 0) + return false; + for (int idx : faces) + if (idx < 0 || idx >= N) + return false; m_n = N; m_tmplNeutral = tmplNeutral; @@ -138,20 +148,43 @@ bool DeformationTransfer::init(const std::vector& tmplNeutral, } }; - // rows: 3 per triangle (gradient columns) + ONE anchor row. - // columns: N real verts + F free 4th-vertices. + // rows: 3 per triangle (gradient columns) + ONE anchor row PER CONNECTED + // COMPONENT. columns: N real verts + F free 4th-vertices. // // Deformation gradients are translation-invariant, so the system is only - // determined up to a global translation. We pin exactly ONE real vertex - // (index 0) to fix that gauge — anchoring EVERY vertex would fight the - // shape (the gradients want expr, the anchor wants neutral) and pull the - // solution back toward the neutral, corrupting the transfer. + // determined up to a translation PER TRIANGLE ISLAND — the ICT template + // has dozens of them (eyeballs, corneas, teeth, mouth interior). One + // anchor per island fixes each gauge; anchoring EVERY vertex would fight + // the shape (the gradients want expr, the anchor wants neutral) and pull + // the solution back toward the neutral, corrupting the transfer. for (int f = 0; f < F; ++f) pushGrad(3 * f, f); const double anchorW = 1.0; - trip.push_back({double(3 * F), 0.0, anchorW}); + std::vector comp(size_t(N), 0); + for (int i = 0; i < N; ++i) comp[size_t(i)] = i; + auto findRoot = [&comp](int a) { + while (comp[size_t(a)] != a) { + comp[size_t(a)] = comp[size_t(comp[size_t(a)])]; + a = comp[size_t(a)]; + } + return a; + }; + for (int f = 0; f < F; ++f) { + const int a = findRoot(m_faces[size_t(f)*3]); + comp[size_t(findRoot(m_faces[size_t(f)*3+1]))] = a; + comp[size_t(findRoot(m_faces[size_t(f)*3+2]))] = a; + } + m_anchors.clear(); + std::set anchoredRoots; + for (int i = 0; i < N; ++i) { + if (anchoredRoots.insert(findRoot(i)).second) { + trip.push_back({double(3 * F + int(m_anchors.size())), + double(i), anchorW}); + m_anchors.push_back(i); + } + } - m_A.fromTriplets(3 * F + 1, N + F, trip); + m_A.fromTriplets(3 * F + int(m_anchors.size()), N + F, trip); // source (template) rest inverse frames — for building S_f per shape for (int f = 0; f < F; ++f) { @@ -187,7 +220,8 @@ std::vector DeformationTransfer::transfer( // column g; the free 4th vertex is an unknown (its normal equation exists // in the matrix), so the rhs is just S — no rest-approximation term. const int nCols = m_n + F; - std::vector> rhs(3, std::vector(size_t(3 * F + 1), 0.0)); + std::vector> rhs( + 3, std::vector(size_t(3 * F) + m_anchors.size(), 0.0)); for (int f = 0; f < F; ++f) { const int i0 = m_faces[size_t(f)*3]; @@ -205,14 +239,18 @@ std::vector DeformationTransfer::transfer( for (int g = 0; g < 3; ++g) rhs[size_t(axis)][size_t(3*f+g)] = S[size_t(axis*3+g)]; } - // anchor rhs: pin vertex 0 to (fitted rest + the template's vertex-0 - // displacement) — fixes the translation gauge while letting the shape - // move vertex 0 the same way the source moved its vertex 0. For the - // identity fit this is exactly expr[0]. + // anchor rhs: pin one vertex PER ISLAND to (fitted rest + the template's + // displacement of that vertex) — fixes each island's translation gauge + // while letting the shape move it the same way the source moved it. For + // the identity fit this is exactly expr[anchor]. const double anchorW = 1.0; - for (int axis = 0; axis < 3; ++axis) - rhs[size_t(axis)][size_t(3*F)] = - anchorW * (m_fitted[axis] + tmplExprDelta[size_t(axis)]); + for (size_t ai = 0; ai < m_anchors.size(); ++ai) { + const size_t v = size_t(m_anchors[ai]); + for (int axis = 0; axis < 3; ++axis) + rhs[size_t(axis)][size_t(3*F) + ai] = + anchorW * (m_fitted[v*3 + size_t(axis)] + + tmplExprDelta[v*3 + size_t(axis)]); + } // solve per axis (warm-start real verts at the fitted rest, 4th verts at // their rest normal offset). Read back only the N real vertices as a delta. diff --git a/src/FaceRig/DeformationTransfer.h b/src/FaceRig/DeformationTransfer.h index 52aa37584..a92da110e 100644 --- a/src/FaceRig/DeformationTransfer.h +++ b/src/FaceRig/DeformationTransfer.h @@ -62,6 +62,7 @@ class DeformationTransfer { std::vector> m_srcRestInv; // 3x3 per tri std::vector> m_tgtRestInv; std::vector> m_tgtNormalV4; // fitted 4th vertex/tri + std::vector m_anchors; // one gauge-anchor vertex per island }; } // namespace FaceRig diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp index dc2d99513..93f2353da 100644 --- a/src/FaceRig/FaceRigAttach.cpp +++ b/src/FaceRig/FaceRigAttach.cpp @@ -89,7 +89,16 @@ FaceRigGeometry extractGeometry(Ogre::Entity* entity) appendIndices(id, base, geo.userF); }; - if (mesh->sharedVertexData) { + // Only extract the shared pool when a submesh actually references it — + // some importers allocate sharedVertexData that no submesh uses, and the + // orphan vertices would join the fit (and the head mask / bounding + // computations) without any triangles. + bool anyShared = false; + for (unsigned short si = 0; si < mesh->getNumSubMeshes(); ++si) { + Ogre::SubMesh* sm = mesh->getSubMesh(si); + if (sm && sm->useSharedVertices) { anyShared = true; break; } + } + if (mesh->sharedVertexData && anyShared) { // shared pool → handle 0; its indices live per-submesh. std::vector pos; if (extractPositions(mesh->sharedVertexData, pos)) { diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index f42d69d26..b8ca95e6f 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -461,12 +461,15 @@ FaceRigResult buildFaceRig(const std::vector& userV, // humanoid-only gate: a bad fit means this isn't a face — refuse. A NRICP // fit that couldn't converge onto the surface reports non-finite or huge // residuals (e.g. a plane template forced onto a sphere), which we treat - // as a hard reject alongside the mean-residual threshold. The max residual - // catches shapes that fit on average but blow up in a region. + // as a hard reject. `maxFitResidualPct` gates the MAX residual directly — + // the knob is advertised as `--max-residual`, so it must mean what it says + // (it previously allowed up to 6x the supplied value). The mean gate at a + // quarter of it catches fits that never blow up locally but drape the + // whole surface badly (healthy fits measure mean <= 0.1%, max <= ~4%). const bool nonFinite = !std::isfinite(r.fitMeanResidualPct) || !std::isfinite(r.fitMaxResidualPct); - if (nonFinite || r.fitMeanResidualPct > opts.maxFitResidualPct || - r.fitMaxResidualPct > opts.maxFitResidualPct * 6.0) { + if (nonFinite || r.fitMaxResidualPct > opts.maxFitResidualPct || + r.fitMeanResidualPct > opts.maxFitResidualPct * 0.25) { r.error = "mesh does not fit the human face template (mean residual " + std::to_string(r.fitMeanResidualPct) + "%, max " + std::to_string(r.fitMaxResidualPct) + diff --git a/src/FaceRig/NonRigidICP.cpp b/src/FaceRig/NonRigidICP.cpp index b8651434e..ef4e85295 100644 --- a/src/FaceRig/NonRigidICP.cpp +++ b/src/FaceRig/NonRigidICP.cpp @@ -105,6 +105,47 @@ struct KDTree { nearestRec(near, q, best, bestD2); if (diff*diff < bestD2) nearestRec(far, q, best, bestD2); } + // K nearest centroid indices (broad phase). The caller runs the exact + // point-triangle test over these — a large sliver triangle's true surface + // can be closer than the triangle whose CENTROID is closest, so a single + // centroid winner picks the wrong correspondence on non-uniform meshes. + void nearestK(const Vec3& q, int K, std::vector& out) const + { + out.clear(); + std::vector> heap; // max-heap by distance + nearestKRec(0, q, K, heap); + out.reserve(heap.size()); + for (const auto& [d2, pi] : heap) out.push_back(pi); + } + void nearestKRec(int n, const Vec3& q, int K, + std::vector>& heap) const + { + const Node& nd = nodes[n]; + if (nd.axis < 0) { + for (int k = 0; k < nd.count; ++k) { + const int pi = idx[nd.start+k]; + const Vec3 d = vsub(pts[pi], q); + const double d2 = vdot(d, d); + if (int(heap.size()) < K) { + heap.emplace_back(d2, pi); + std::push_heap(heap.begin(), heap.end()); + } else if (d2 < heap.front().first) { + std::pop_heap(heap.begin(), heap.end()); + heap.back() = {d2, pi}; + std::push_heap(heap.begin(), heap.end()); + } + } + return; + } + const double diff = q[nd.axis] - nd.split; + const int near = diff < 0 ? nd.lo : nd.hi; + const int far = diff < 0 ? nd.hi : nd.lo; + nearestKRec(near, q, K, heap); + const double worstNow = int(heap.size()) < K + ? std::numeric_limits::max() : heap.front().first; + if (diff*diff < worstNow) + nearestKRec(far, q, K, heap); + } }; // ---- CSR sparse matrix + CG on the normal equations (AᵀA x = Aᵀb) ---------- @@ -260,14 +301,26 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, int levelIdx = 0; for (double alpha : opts.stiffness) { for (int iter = 0; iter < opts.itersPerLevel; ++iter) { - // find closest surface point per current X_i + // find closest surface point per current X_i: broad-phase K + // nearest centroids, then the EXACT point-triangle distance picks + // among them — the centroid-nearest triangle alone mis-corresponds + // next to large/sliver triangles on non-uniform meshes. std::vector target(Nt); + std::vector cand; for (int i = 0; i < Nt; ++i) { - const int cf = tree.nearest(X[i]); - // refine: check that triangle + a few neighbors would need the - // full tree; centroid-nearest triangle is a good approximation - // for a fitted template already close to the surface. - target[i] = closestPointTriangle(X[i], utri[cf][0], utri[cf][1], utri[cf][2]); + tree.nearestK(X[i], 4, cand); + double bestD2 = std::numeric_limits::max(); + Vec3 bestP{0,0,0}; + for (int cf : cand) { + const Vec3 p = closestPointTriangle( + X[i], utri[cf][0], utri[cf][1], utri[cf][2]); + const Vec3 d = vsub(p, X[i]); + const double d2 = vdot(d, d); + if (d2 < bestD2) { bestD2 = d2; bestP = p; } + } + target[i] = cand.empty() + ? X[i] + : bestP; } // Landmark anchors that reference a valid template vertex. Weight From 3ac9deffe8421969cde127d41c7088d1562be108 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 23:55:58 -0400 Subject: [PATCH 34/38] ci: trigger PR checks after retargeting the consolidated epic PR to master Co-Authored-By: Claude Fable 5 From cf2df15cc0d8cc184cc0f9ee797e444d0c51c0a4 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 18 Jul 2026 00:45:03 -0400 Subject: [PATCH 35/38] =?UTF-8?q?fix(#889):=20add=20FaceRig=20sources=20to?= =?UTF-8?q?=20the=20tests'=20common=20library=20=E2=80=94=20CI=20link=20fa?= =?UTF-8?q?ilure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-suite test executables link against tests/CMakeLists.txt's own qtmesh_test_common, whose explicit source list didn't include the FaceRig sources while MCPServer.cpp / CLIPipeline.cpp / mainwindow.cpp in that same list now reference them (undefined references on unit-tests-linux). The UnitTests target was unaffected (it GLOBs src recursively), which is why the local build masked it. This was PR #902's P1 review finding — confirmed real by CI. Co-Authored-By: Claude Fable 5 --- tests/CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 85090bc64..4ef2ac01e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -75,6 +75,17 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/BoneWeightOverlay.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/NormalVisualizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/AnimationMerger.cpp + # Face auto-rig (#889): MCPServer/CLIPipeline/mainwindow reference + # these — without them the per-suite test executables fail to link. + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRigController.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/ArkitTemplate.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/NonRigidICP.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/SparseSolve.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/DeformationTransfer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/FaceRigger.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/FaceRigAttach.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/FaceLandmarkDetector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/FaceRigLandmarks.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MCPServer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MCPSettingsDialog.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MeshInfoOverlay.cpp From 1ecd88e5307aeb3ba541b2d5a3163b2961279d0d Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 18 Jul 2026 00:55:06 -0400 Subject: [PATCH 36/38] fix(#889): address second CodeRabbit review round on the consolidated PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - NonRigidICP: reject out-of-range face indices in BOTH the template and user buffers at the fit() boundary (critical finding); a cancelled fit now reports ok=false even when the partial residuals look finite. - ArkitTemplate::load: validate face indices and reject non-finite positions/deltas — a corrupt bundle can no longer reach the solvers. - FaceRigAttach::headSubmesh: bound-check triangle indices before indexing fullToSub (only negatives were rejected). - FaceRigger: normalise the amplitude out of the invisible-shapes safety net so a healthy rig at low amplitude doesn't trigger the unanchored rerun. - DeformationTransfer: scale island gauge-anchor deltas by the global template->fitted size ratio (template-space deltas had the wrong amplitude on scaled fits; rotationless fits are the pipeline contract). - MCPServer: drop the redundant sidecar write after export — the exporter already writes the FULL deduplicated pose-name list; overwriting it with only the newly-attached names dropped pre-existing morph targets. - FaceLandmarkDetector: download breadcrumbs + the synchronous-failure done-guard (same pattern as ArkitTemplate). - CLIPipeline: facerig import breadcrumb logs the extension only, not the absolute path (telemetry privacy). - scripts: spike-facerig.py fan-triangulates ICT quads (same bug as the packer); export-arkit-template.py aborts on missing expressions or a face-list mismatch instead of packaging dead zero shapes; upload-facerig-template.sh refuses to publish without the template and ICT license. All 34 FaceRig/NRICP/DT tests pass; reference rig unchanged (max residual 0.35%). Co-Authored-By: Claude Fable 5 --- scripts/export-arkit-template.py | 9 +++++++-- scripts/spike-facerig.py | 7 ++++++- scripts/upload-facerig-template.sh | 6 +++++- src/CLIPipeline.cpp | 8 ++++++-- src/FaceRig/ArkitTemplate.cpp | 19 +++++++++++++++--- src/FaceRig/DeformationTransfer.cpp | 30 ++++++++++++++++++++++++---- src/FaceRig/DeformationTransfer.h | 1 + src/FaceRig/FaceLandmarkDetector.cpp | 22 +++++++++++++++----- src/FaceRig/FaceRigAttach.cpp | 3 ++- src/FaceRig/FaceRigger.cpp | 3 +++ src/FaceRig/NonRigidICP.cpp | 17 ++++++++++++---- src/MCPServer.cpp | 6 ++++-- 12 files changed, 106 insertions(+), 25 deletions(-) diff --git a/scripts/export-arkit-template.py b/scripts/export-arkit-template.py index ee41b8e0b..77f818c82 100644 --- a/scripts/export-arkit-template.py +++ b/scripts/export-arkit-template.py @@ -117,13 +117,18 @@ def main(): if not os.path.exists(p): missing.append(stem) continue - ev, _ = load_obj(p) + ev, ef = load_obj(p) if len(ev) != V: sys.exit(f"topology mismatch: {stem} has {len(ev)} verts, " f"neutral {V}") + if not np.array_equal(ef, faces): + sys.exit(f"topology mismatch: {stem} face list differs from " + f"the neutral mesh (same vert count is not enough)") delta += ev - neutral if missing: - print(f" !! {arkit_name}: missing ICT {missing} — zero shape") + # A missing expression would silently ship as a dead zero-delta + # shape — fail so a corrupt bundle can't be packaged/uploaded. + sys.exit(f"ABORT: {arkit_name}: missing ICT {missing}") # count verts moved by a MEANINGFUL amount (0.1% of the head # diagonal), not floating-point dust, so the log reflects real motion. _diag = float(np.linalg.norm(neutral.max(0) - neutral.min(0))) diff --git a/scripts/spike-facerig.py b/scripts/spike-facerig.py index 64b6088e5..84e2ad683 100644 --- a/scripts/spike-facerig.py +++ b/scripts/spike-facerig.py @@ -81,7 +81,12 @@ def load_obj(path): if ln.startswith("v "): V.append([float(x) for x in ln.split()[1:4]]) elif ln.startswith("f "): - F.append([int(t.split("/")[0]) - 1 for t in ln.split()[1:4]]) + # Fan-triangulate: ICT-FaceKit meshes are QUADS (same fix as + # export-arkit-template.py — first-3-indices dropped half the + # triangles). + idx = [int(t.split("/")[0]) - 1 for t in ln.split()[1:]] + for k in range(1, len(idx) - 1): + F.append([idx[0], idx[k], idx[k + 1]]) return np.asarray(V, dtype=np.float64), np.asarray(F, dtype=np.int64) diff --git a/scripts/upload-facerig-template.sh b/scripts/upload-facerig-template.sh index 779642910..42cccbaf9 100755 --- a/scripts/upload-facerig-template.sh +++ b/scripts/upload-facerig-template.sh @@ -34,8 +34,12 @@ upload() { # fi } +# Required artifacts — refuse to publish an incomplete release. Only the +# landmark model below is optional (hosted separately by the mocap epic). +[ -f "$OUT/arkit_template.bin" ] || { echo "ABORT: missing $OUT/arkit_template.bin"; exit 1; } +[ -n "$ICT_LICENSE" ] && [ -f "$ICT_LICENSE" ] || { echo "ABORT: ICT_LICENSE not set or missing (the MIT license must ship next to the template)"; exit 1; } upload "$OUT/arkit_template.bin" "facerig/arkit_template.bin" -[ -n "$ICT_LICENSE" ] && upload "$ICT_LICENSE" "facerig/ICT-FaceKit-LICENSE.txt" +upload "$ICT_LICENSE" "facerig/ICT-FaceKit-LICENSE.txt" # Facial-landmark model (MediaPipe FaceMesh V2, Apache-2.0) that anchors the # NRICP fit to real face features (#889 landmark pass). It is the SAME graph the diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 5b35b76f5..8536b03db 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -9402,8 +9402,10 @@ int CLIPipeline::cmdRig(int argc, char* argv[]) SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.auto_rig"), QString("rig .%1 template=%2 algo=%3 skin=%4") .arg(fi.suffix(), templateName, algoName).arg(alsoSkin)); + // Extension only — absolute paths leak usernames/directory structure + // into telemetry. SentryReporter::addBreadcrumb(QStringLiteral("file.import"), - QString("Importing %1").arg(fi.absoluteFilePath())); + QString("Importing .%1 for facerig").arg(fi.suffix())); MeshImporterExporter::importer({fi.absoluteFilePath()}); QList meshEntities; @@ -9522,8 +9524,10 @@ int CLIPipeline::cmdFaceRig(int argc, char* argv[]) SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), QString("facerig .%1 max_shapes=%2").arg(fi.suffix()).arg(opts.maxShapes)); + // Extension only — absolute paths leak usernames/directory structure + // into telemetry. SentryReporter::addBreadcrumb(QStringLiteral("file.import"), - QString("Importing %1").arg(fi.absoluteFilePath())); + QString("Importing .%1 for facerig").arg(fi.suffix())); MeshImporterExporter::importer({fi.absoluteFilePath()}); QList meshEntities; diff --git a/src/FaceRig/ArkitTemplate.cpp b/src/FaceRig/ArkitTemplate.cpp index 33ad01661..88cce110d 100644 --- a/src/FaceRig/ArkitTemplate.cpp +++ b/src/FaceRig/ArkitTemplate.cpp @@ -12,6 +12,7 @@ #include #include +#include #include namespace FaceRig { @@ -93,11 +94,19 @@ bool ArkitTemplate::load(const QString& path, QString* error) m_vertexCount = V; m_faceCount = F; m_neutral.resize(size_t(V) * 3); - for (auto& v : m_neutral) + for (auto& v : m_neutral) { v = rdF32(p); + if (!std::isfinite(v)) + return fail(QStringLiteral("non-finite neutral position")); + } m_faces.resize(size_t(F) * 3); - for (auto& i : m_faces) + for (auto& i : m_faces) { i = rdI32(p); + // out-of-range indices would be dereferenced by the fit / renders + if (i < 0 || i >= V) + return fail(QStringLiteral("face index %1 out of range (V=%2)") + .arg(i).arg(V)); + } m_shapes.clear(); m_shapes.reserve(S); @@ -108,8 +117,12 @@ bool ArkitTemplate::load(const QString& path, QString* error) shape.name = QString::fromLatin1(p, qstrnlen(p, kNameLen)); p += kNameLen; shape.deltas.resize(size_t(V) * 3); - for (auto& d : shape.deltas) + for (auto& d : shape.deltas) { d = rdF32(p); + if (!std::isfinite(d)) + return fail(QStringLiteral("non-finite delta in shape %1") + .arg(shape.name)); + } m_shapes.push_back(std::move(shape)); } return true; diff --git a/src/FaceRig/DeformationTransfer.cpp b/src/FaceRig/DeformationTransfer.cpp index 064dc88ef..176c6a106 100644 --- a/src/FaceRig/DeformationTransfer.cpp +++ b/src/FaceRig/DeformationTransfer.cpp @@ -96,6 +96,24 @@ bool DeformationTransfer::init(const std::vector& tmplNeutral, m_tmplNeutral = tmplNeutral; m_faces = faces; m_fitted = fitted; + // global template->fitted size ratio, used to scale island gauge-anchor + // deltas into fitted space (see the anchor rhs in transfer()). + { + auto diagOf = [](const std::vector& v) { + float lo[3] = {1e30f, 1e30f, 1e30f}, hi[3] = {-1e30f, -1e30f, -1e30f}; + for (size_t i = 0; i + 2 < v.size(); i += 3) + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], v[i + size_t(a)]); + hi[a] = std::max(hi[a], v[i + size_t(a)]); + } + double s2 = 0; + for (int a = 0; a < 3; ++a) + s2 += double(hi[a] - lo[a]) * (hi[a] - lo[a]); + return std::sqrt(s2); + }; + const double td = diagOf(tmplNeutral); + m_fitScale = td > 1e-12 ? diagOf(fitted) / td : 1.0; + } m_srcRestInv.assign(size_t(F), {}); m_tgtRestInv.assign(size_t(F), {}); m_tgtNormalV4.assign(size_t(F), {}); @@ -240,16 +258,20 @@ std::vector DeformationTransfer::transfer( rhs[size_t(axis)][size_t(3*f+g)] = S[size_t(axis*3+g)]; } // anchor rhs: pin one vertex PER ISLAND to (fitted rest + the template's - // displacement of that vertex) — fixes each island's translation gauge - // while letting the shape move it the same way the source moved it. For - // the identity fit this is exactly expr[anchor]. + // displacement of that vertex, SCALED into fitted space) — fixes each + // island's translation gauge while letting the shape move it the way the + // source moved it. The gradient system cannot represent rigid + // translation, and the raw template-space delta has the wrong amplitude + // when the fit changed scale; the global size ratio corrects that (a + // rotationless fit is the pipeline contract — the similarity prealign + // has no rotation term). const double anchorW = 1.0; for (size_t ai = 0; ai < m_anchors.size(); ++ai) { const size_t v = size_t(m_anchors[ai]); for (int axis = 0; axis < 3; ++axis) rhs[size_t(axis)][size_t(3*F) + ai] = anchorW * (m_fitted[v*3 + size_t(axis)] - + tmplExprDelta[v*3 + size_t(axis)]); + + m_fitScale * tmplExprDelta[v*3 + size_t(axis)]); } // solve per axis (warm-start real verts at the fitted rest, 4th verts at diff --git a/src/FaceRig/DeformationTransfer.h b/src/FaceRig/DeformationTransfer.h index a92da110e..da4cd73c7 100644 --- a/src/FaceRig/DeformationTransfer.h +++ b/src/FaceRig/DeformationTransfer.h @@ -63,6 +63,7 @@ class DeformationTransfer { std::vector> m_tgtRestInv; std::vector> m_tgtNormalV4; // fitted 4th vertex/tri std::vector m_anchors; // one gauge-anchor vertex per island + double m_fitScale = 1.0; // template->fitted global size ratio }; } // namespace FaceRig diff --git a/src/FaceRig/FaceLandmarkDetector.cpp b/src/FaceRig/FaceLandmarkDetector.cpp index 62448c4a7..1ba874f20 100644 --- a/src/FaceRig/FaceLandmarkDetector.cpp +++ b/src/FaceRig/FaceLandmarkDetector.cpp @@ -2,6 +2,7 @@ #include "ArkitTemplate.h" // reuse its model dir + base-url convention #include "../ModelDownloader.h" +#include "../SentryReporter.h" #include #include @@ -93,15 +94,18 @@ QString FaceLandmarkDetector::ensureModelBlocking() const QString url = base + QString::fromLatin1(kModelFile); const QString label = QStringLiteral("face landmark model"); + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QStringLiteral("face landmark model download start")); + QEventLoop loop; - bool ok = false, timedOut = false; + bool ok = false, timedOut = false, done = false; auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, [&](const QString& name, const QString&) { - if (name == label) { ok = true; loop.quit(); } + if (name == label) { ok = true; done = true; loop.quit(); } }); auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, [&](const QString& name, const QString&) { - if (name == label) { ok = false; loop.quit(); } + if (name == label) { ok = false; done = true; loop.quit(); } }); QTimer timeout; timeout.setSingleShot(true); @@ -110,14 +114,22 @@ QString FaceLandmarkDetector::ensureModelBlocking() timeout.start(120000); // 2 min — the model is small (~3 MB) dl->startDownload(url, dest, label); - loop.exec(); + // done-guard: a synchronous downloadError would otherwise block the + // loop for the full timeout (same pattern as ArkitTemplate). + if (!done) + loop.exec(); QObject::disconnect(onDone); QObject::disconnect(onErr); if (timedOut && dl) dl->cancelDownload(); - return (ok && !timedOut && QFileInfo::exists(dest)) ? dest : QString(); + const bool success = ok && !timedOut && QFileInfo::exists(dest); + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + success ? QStringLiteral("face landmark model download ok") + : QStringLiteral("face landmark model download failed%1") + .arg(timedOut ? QStringLiteral(" (timeout)") : QString())); + return success ? dest : QString(); } bool FaceLandmarkDetector::isAvailable() const { return d && d->loaded; } diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp index 93f2353da..70b37ab2b 100644 --- a/src/FaceRig/FaceRigAttach.cpp +++ b/src/FaceRig/FaceRigAttach.cpp @@ -211,7 +211,8 @@ void headSubmesh(const FaceRigGeometry& geo, } for (size_t f = 0; f + 2 < geo.userF.size(); f += 3) { const int a = geo.userF[f], b = geo.userF[f+1], c = geo.userF[f+2]; - if (a < 0 || b < 0 || c < 0) continue; + const int nv = int(fullToSub.size()); + if (a < 0 || b < 0 || c < 0 || a >= nv || b >= nv || c >= nv) continue; const int sa = fullToSub[size_t(a)], sb = fullToSub[size_t(b)], sc = fullToSub[size_t(c)]; if (sa >= 0 && sb >= 0 && sc >= 0) diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index b8ca95e6f..9ab849145 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -664,6 +664,9 @@ FaceRigResult buildFaceRig(const std::vector& userV, double maxAmp = 0; for (const auto& sh : r.shapes) maxAmp = std::max(maxAmp, double(sh.maxDisp)); + // maxDisp includes the user amplitude — normalise it out so a healthy + // rig at amplitude 0.1 doesn't read as "invisible" and rerun. + maxAmp /= std::clamp(opts.amplitude, 0.1, 5.0); if (maxAmp < 0.005 * diag) { std::fprintf(stderr, "[facerig] anchored fit produced invisible " "shapes (max %.5f on diag %.3f) — retrying " diff --git a/src/FaceRig/NonRigidICP.cpp b/src/FaceRig/NonRigidICP.cpp index ef4e85295..e23af239d 100644 --- a/src/FaceRig/NonRigidICP.cpp +++ b/src/FaceRig/NonRigidICP.cpp @@ -231,6 +231,14 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, const int Fu = int(userF.size()/3); if (Nt < 3 || Fu < 1 || userV.size() < 9) return res; + bool aborted = false; + // Reject malformed face buffers at the public boundary — indices are + // dereferenced when building triangles/edges below. + const int Nu = int(userV.size()/3); + for (int idx : userF) + if (idx < 0 || idx >= Nu) return res; + for (int idx : tmplF) + if (idx < 0 || idx >= Nt) return res; // user bbox diagonal Vec3 mn = at(userV,0), mx = mn; @@ -247,7 +255,6 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, for (int a=0;a<3;++a){ tmn[a]=std::min(tmn[a],p[a]); tmx[a]=std::max(tmx[a],p[a]); } } tc = vscale(tc, 1.0/Nt); - const int Nu = int(userV.size()/3); for (int i = 0; i < Nu; ++i) uc = vadd(uc, at(userV,i)); uc = vscale(uc, 1.0/Nu); const double tdiag = std::sqrt(vdot(vsub(tmx,tmn), vsub(tmx,tmn))); @@ -400,8 +407,10 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, } ++levelIdx; if (progress && !progress(levelIdx, levelCount)) { - // caller aborted — bail with the best fit so far (ok stays false - // below because the residual pass computes finiteCount honestly). + // caller aborted — bail with the best fit so far, and force + // ok=false regardless of how the residuals turn out (the + // finite-count check alone can read a half-finished fit as ok). + aborted = true; break; } } @@ -442,7 +451,7 @@ NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, res.maxResidual = (divergedCount > Nt / 20) // > 5% diverged ? std::numeric_limits::infinity() : mx2; - res.ok = finiteCount > Nt / 2; // need at least half the verts to have fit + res.ok = !aborted && finiteCount > Nt / 2; // half the verts + not cancelled return res; } diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 56703ea1a..ed6117f98 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -2348,8 +2348,10 @@ QJsonObject MCPServer::toolAddArkitBlendshapes(const QJsonObject &args) return makeErrorResult( QStringLiteral("Error: blendshapes attached but export to '%1' " "failed (code %2)").arg(outputPath).arg(rc)); - // Sidecar with the ordered ARKit names (Assimp drops targetNames). - FaceRig::writeArkitSidecar(outputPath, rep.shapeNames); + // NOTE: no sidecar write here — MeshImporterExporter::exporter + // already writes the FULL deduplicated pose-name sidecar; + // overwriting it with only the newly-attached names would drop + // pre-existing morph targets and misalign indices. } } catch (const Ogre::Exception& e) { return makeErrorResult(QStringLiteral("Ogre error: %1") From 13ee9521c28cc10fa6b766b661a4b3937016a054 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 18 Jul 2026 02:03:59 -0400 Subject: [PATCH 37/38] fix(#889): EnvGatedRealBundle passes as a no-op instead of GTEST_SKIP The CI harness treats ANY skipped test as a suite failure (the sole remaining red in unit-tests-linux: 462/463 suites). Follow the SkinEvaluate env-gated convention: SUCCEED() + return when QTMESH_FACERIG_TEMPLATE is unset. Co-Authored-By: Claude Fable 5 --- src/FaceRig/ArkitTemplate_test.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/FaceRig/ArkitTemplate_test.cpp b/src/FaceRig/ArkitTemplate_test.cpp index 6fdbea338..ebab4e467 100644 --- a/src/FaceRig/ArkitTemplate_test.cpp +++ b/src/FaceRig/ArkitTemplate_test.cpp @@ -122,8 +122,12 @@ TEST(ArkitTemplate, MissingFileFails) TEST(ArkitTemplate, EnvGatedRealBundle) { const QByteArray p = qgetenv("QTMESH_FACERIG_TEMPLATE"); - if (p.isEmpty()) - GTEST_SKIP() << "set QTMESH_FACERIG_TEMPLATE to arkit_template.bin"; + if (p.isEmpty()) { + // pass as a no-op — the CI harness treats ANY skipped test as a suite + // failure (same convention as SkinEvaluate's env-gated reference test) + SUCCEED() << "QTMESH_FACERIG_TEMPLATE not set — real bundle not exercised"; + return; + } FaceRig::ArkitTemplate t; QString err; ASSERT_TRUE(t.load(QString::fromUtf8(p), &err)) << err.toStdString(); From 0827124ff9948dd15426d79a373803ae359024de Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 18 Jul 2026 03:21:33 -0400 Subject: [PATCH 38/38] =?UTF-8?q?fix(#889):=20third=20CodeRabbit=20round?= =?UTF-8?q?=20=E2=80=94=20bounds,=20far-query=20fallback,=20breadcrumb=20c?= =?UTF-8?q?ategory,=20atomic=20template=20load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - FaceRigger: upper-bound face-index check in the headMask subset path (critical — malformed faces reached fullToSub before NRICP's own validation); the delta-resample grid's nearest() now clamps its SEARCH ORIGIN into the populated bounds so a query far outside the grid can't exhaust the shell cap and silently drop that vertex's transferred delta (distances still measured to the real query — result unchanged). - MCPServer: add_arkit_blendshapes breadcrumb uses the ai.tool_call category per the MCP house rule (and drops the entity name). - ArkitTemplate::load: parse into temporaries and commit only after all validation passes — no half-populated object on a corrupt bundle. All FaceRig suites pass; reference rig unchanged (max residual 0.35%). Co-Authored-By: Claude Fable 5 --- src/FaceRig/ArkitTemplate.cpp | 24 +++++++++++++++--------- src/FaceRig/FaceRigger.cpp | 14 ++++++++++++-- src/MCPServer.cpp | 5 ++--- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/FaceRig/ArkitTemplate.cpp b/src/FaceRig/ArkitTemplate.cpp index 88cce110d..f30d8047d 100644 --- a/src/FaceRig/ArkitTemplate.cpp +++ b/src/FaceRig/ArkitTemplate.cpp @@ -91,16 +91,16 @@ bool ArkitTemplate::load(const QString& path, QString* error) return fail(QStringLiteral("template truncated (need %1 bytes, have %2)") .arg(need).arg(blob.size())); - m_vertexCount = V; - m_faceCount = F; - m_neutral.resize(size_t(V) * 3); - for (auto& v : m_neutral) { + // Parse into TEMPORARIES and commit only after every check passes — a + // validation failure mid-file must not leave the object half-populated. + std::vector neutral(size_t(V) * 3, 0.0f); + for (auto& v : neutral) { v = rdF32(p); if (!std::isfinite(v)) return fail(QStringLiteral("non-finite neutral position")); } - m_faces.resize(size_t(F) * 3); - for (auto& i : m_faces) { + std::vector faces(size_t(F) * 3, 0); + for (auto& i : faces) { i = rdI32(p); // out-of-range indices would be dereferenced by the fit / renders if (i < 0 || i >= V) @@ -108,8 +108,8 @@ bool ArkitTemplate::load(const QString& path, QString* error) .arg(i).arg(V)); } - m_shapes.clear(); - m_shapes.reserve(S); + std::vector shapes; + shapes.reserve(S); for (int s = 0; s < S; ++s) { if (p + kNameLen > end) return fail(QStringLiteral("shape %1 name overruns").arg(s)); @@ -123,8 +123,14 @@ bool ArkitTemplate::load(const QString& path, QString* error) return fail(QStringLiteral("non-finite delta in shape %1") .arg(shape.name)); } - m_shapes.push_back(std::move(shape)); + shapes.push_back(std::move(shape)); } + + m_vertexCount = V; + m_faceCount = F; + m_neutral = std::move(neutral); + m_faces = std::move(faces); + m_shapes = std::move(shapes); return true; } diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp index 9ab849145..d6432d457 100644 --- a/src/FaceRig/FaceRigger.cpp +++ b/src/FaceRig/FaceRigger.cpp @@ -48,7 +48,15 @@ class PointGrid { int nearest(const float* q) const { if (!m_pts || m_cells.empty()) return -1; - const std::array c = cellOf(q); + // Clamp the SEARCH ORIGIN into the populated bounds: for a query far + // outside the grid, the shell cap below (grid span) could otherwise + // terminate before any populated cell is reached and silently return + // -1 (dropping that vertex's transferred delta). Distances are still + // measured to the real q, so the nearest result is unchanged. + float qc[3]; + for (int a = 0; a < 3; ++a) + qc[a] = std::min(std::max(q[a], m_lo[a]), m_hi[a]); + const std::array c = cellOf(qc); // absolute cap on the shell radius = span of the grid in cells + 1, // guarantees termination even if q is far outside the populated region. int spanCells = 1; @@ -317,7 +325,9 @@ FaceRigResult buildFaceRig(const std::vector& userV, // keep faces whose 3 verts are all head; remap to sub indices for (size_t f = 0; f + 2 < userF.size(); f += 3) { const int a = userF[f], b = userF[f+1], c = userF[f+2]; - if (a < 0 || b < 0 || c < 0) continue; + const int nFull = int(fullToSub.size()); + if (a < 0 || b < 0 || c < 0 || a >= nFull || b >= nFull || c >= nFull) + continue; const int sa = fullToSub[size_t(a)], sb = fullToSub[size_t(b)], sc = fullToSub[size_t(c)]; if (sa >= 0 && sb >= 0 && sc >= 0) diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index ed6117f98..0934ec45f 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -2324,9 +2324,8 @@ QJsonObject MCPServer::toolAddArkitBlendshapes(const QJsonObject &args) return makeErrorResult("No selected entity."); Ogre::Entity* entity = resolved.first(); - SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), - QStringLiteral("add_arkit_blendshapes entity=%1 max_shapes=%2") - .arg(QString::fromStdString(entity->getName())).arg(opts.maxShapes)); + SentryReporter::addBreadcrumb(QStringLiteral("ai.tool_call"), + QStringLiteral("add_arkit_blendshapes max_shapes=%1").arg(opts.maxShapes)); FaceRig::AttachReport rep; try {