diff --git a/.gitignore b/.gitignore index a9421e6f3..f6287d6d4 100755 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,8 @@ docs/* !docs/IMAGE_TO_3D_QUALITY.md !docs/TRIPOSG_EXPORT_NOTES.md !docs/MESH_SEGMENTATION_STRATEGY.md +!docs/FACE_RIG.md +!docs/FACE_RIG_SPIKE.md # minisign — never commit secret keys minisign.key diff --git a/CLAUDE.md b/CLAUDE.md index c14b9e034..3ef98c06e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -140,6 +140,8 @@ qtmesh skin ours.fbx --compare mixamo_ref.fbx [--json] # per-vertex weight diff qtmesh rig model.obj --skeleton humanoid -o rigged.fbx # native auto-rig: embed a skeleton template into an unrigged mesh (#407) qtmesh rig model.obj --skeleton humanoid --skin -o rigged.fbx # one-click rig + skin (chains #402); templates: humanoid|biped|quadruped|generic; --up-axis x|y|z (default y) qtmesh rig model.obj --algo unirig -o rigged.fbx # ML skeleton prediction via ONNX UniRig (#408, MIT model); default --algo pinocchio (offline). UniRig falls back to the template when the model/ONNX is unavailable +qtmesh facerig head.glb -o rigged.glb # #889: auto-generate the 52 ARKit blendshapes on a humanoid FACE mesh (fit ICT template via non-rigid ICP + Sumner-Popović deformation transfer, attach as named morph targets). A poor fit (non-face mesh) is rejected. Bundled template downloads on first use. Feeds `qtmesh mocap --face` +qtmesh facerig head.fbx -o rigged.glb --max-shapes 20 --max-residual 5 --json # cap shape count / tighten the humanoid gate / machine-readable report qtmesh generate3d image.png -o out.glb # AI image-to-3D (#764, TripoSR/ONNX): reconstruct a mesh from a single image (needs ONNX build + model; downloads on first use, clear message if not hosted) qtmesh generate3d photo.png --remove-bg -o out.glb # run U²-Net background removal first (needed for photos with a background; TripoSR wants an isolated subject) qtmesh generate3d image.png --resolution 128 --no-color -o out.glb # faster/preview marching-cubes grid; skip vertex color @@ -173,7 +175,7 @@ qtmesh ps1 dump-vram game.cue --bios scph1001.bin --frames 300 -o vram.png # sn # xvfb-run -a qtmesh ps1 capture game.cue --bios scph1001.bin -o out.gltf ``` -CLI mode is activated by: (1) invoking via the `qtmesh` symlink, (2) passing `--cli`, or (3) using a recognized subcommand (`info`, `fix`, `convert`, `anim`, `validate`, `lod`, `pose`, `turntable`, `isometric`, `scan`, `material`, `hdri`, `light`, `pack-textures`, `normal-from-height`, `atlas`, `atlas-apply`, `memory`, `analyze`, `vertex-cache`, `decimate`, `optimize`, `uv`, `retopo`, `skin`, `rig`, `segment`, `generate3d`, `ps1`, `cloud`) as the first argument. Use `--verbose` to see Ogre/engine debug output. Use `--no-telemetry` to permanently opt out of anonymous usage data collection. +CLI mode is activated by: (1) invoking via the `qtmesh` symlink, (2) passing `--cli`, or (3) using a recognized subcommand (`info`, `fix`, `convert`, `anim`, `validate`, `lod`, `pose`, `turntable`, `isometric`, `scan`, `material`, `hdri`, `light`, `pack-textures`, `normal-from-height`, `atlas`, `atlas-apply`, `memory`, `analyze`, `vertex-cache`, `decimate`, `optimize`, `uv`, `retopo`, `skin`, `rig`, `facerig`, `segment`, `generate3d`, `ps1`, `cloud`) as the first argument. Use `--verbose` to see Ogre/engine debug output. Use `--no-telemetry` to permanently opt out of anonymous usage data collection. If Xcode SDK is updated, clear CMake cache (`rm build_local/CMakeCache.txt`) and reconfigure. @@ -379,6 +381,7 @@ The animation pipeline started skeleton-only; the #517 epic broadens it. Slices - **Skinning v2** (issue #819, Slices A+B): `SkinWeights::Algorithm { InverseDistance, GeodesicVoxel, SkinTokens }` — **SkinTokens (the ML skinner) is the default** on every surface (GUI dialog dropdown, CLI `qtmesh skin --algo skintokens|geodesic-voxel|inverse-distance`, MCP `compute_skin_weights` `algo` param; "unirig" survives as a deprecated alias of skintokens), falling back to GeodesicVoxel when models/ONNX are unavailable — so headless/CI environments transparently get the geodesic bind. `qtmesh rig --skin`, MCP `auto_rig {skin:true}` and the GUI rig+skin checkbox all chain through the same default. **GeodesicVoxelBind** (`src/GeodesicVoxelBind.h/cpp`, Ogre-free + unit-tested) implements Maya's "Geodesic Voxel" bind (Dionne & de Lasa, SCA 2013): voxelize the surface at `--voxel-res` (default 64, max 256; Akenine-Möller tri/box SAT), flood-fill the exterior to classify interior (closes holes at voxel resolution → works on non-watertight/self-intersecting/multi-component meshes), 3D-DDA bone rasterization with snap-to-solid (max 4.5 voxels — far-outside bones get NO seeds and are reported in `bonesWithoutSeeds`), one multi-source Dijkstra over solid voxels (26-conn) keeping the best K=8 `(bone, distance)` pairs per voxel, then per-vertex `(1/d)^falloff` weighting. Distances travel through the volume so cross-limb bleed (hand-near-thigh, inner thighs) is impossible by construction. Degenerate input (planes/cloth — zero interior voxels) falls back to InverseDistance automatically; vertices in bone-less floating islands get inverse-distance fill so they still move with the rig. `SkinTokens` falls back to GeodesicVoxel when its models/ONNX are unavailable (Slice C). **SkinWeightsPost** (`src/SkinWeightsPost.h/cpp`, Ogre-free + unit-tested) runs after EVERY algorithm inside `computeAndApply`: Laplacian relaxation over the vertex adjacency (`--smooth-iterations`, default 3, 0=off; merge-mode manual weights act as Dirichlet constraints — they influence neighbours but are never modified) then prune <0.01 + top-K + renormalize. Report gains `algorithmUsed`, `fallbackReason`, `bleedFraction` (fraction of committed weights not geodesically local — 0 for GVB by construction), `bonesWithoutSeeds`. Sentry breadcrumbs `ai.assist.skin.`. `qtmesh rig --skin` and MCP `auto_rig {skin:true}` chain through the same default. **Slice D — dual-quaternion display toggle**: `SkinningDisplay` (`src/SkinningDisplay.h/cpp`) toggles RTSS hardware skinning per entity (Linear = default path; Dual Quaternion = `HardwareSkinningFactory::prepareEntityForSkinning(ST_DUAL_QUATERNION)` + technique invalidation — kills candy-wrapper collapse on twists). The HS factory + a dormant template SRS are registered in `RTShaderHelper::initialize` (bone cap 96; above-cap entities stay on the default path); Linear mode erases the material's `HS_SRS_DATA` imprint (mirrors Ogre's file-local constant). Display only — exported weights unchanged. Surfaced as the "Display: Linear | Dual Quaternion" row in the Animation-mode Skinning section and MCP `set_skinning_display {mode}`; mode tracked on the entity's UserObjectBindings; Sentry `render.skinning`. **Slice E — evaluation suite**: `SkinMetrics` (`src/SkinMetrics.h/cpp`, pure-data: influence histogram, Laplacian weight-smoothness energy, LBS deform + signed mesh volume) and `SkinEvaluate` (`src/SkinEvaluate.h/cpp`: extract EXISTING weights from an entity, metric report incl. geodesic bleed, position-matched/name-matched comparison vs a reference-skinned copy — equidistant duplicate verts tie-break on minimum weight diff so seams/contact points don't report spurious diffs). CLI `qtmesh skin --evaluate` / `--compare `; acceptance fixtures in `SkinMetrics_test.cpp` (90° elbow capsule volume ≥ 0.9 — measured 0.911; proximity bleed 0 vs inverse-distance >0.1; smoothing strictly reduces energy); Mixamo protocol + thresholds in `docs/SKINNING_QUALITY.md`; env-gated reference test via `QTMESH_SKIN_OURS_FBX`/`QTMESH_SKIN_REF_FBX`. **Slice C — SkinTokens ML skinning (implemented, the DEFAULT per user preference — visually the best skinner in practice)**: `Algorithm::SkinTokens` runs **SkinTokens/TokenRig** (VAST-AI, MIT code + MIT weights; Qwen3-0.6B backbone) — UniRig's own skin head stays blocked on spconv/PTv3 (no ONNX lowering; decision record in THIRD_PARTY_AI_MODELS.md). **SkinTokensPredictor** (`src/SkinTokensPredictor.h/cpp`, the EIGHTH ONNX consumer; pure-data parts unit-tested): surface-sample `num_points` (8192) points+normals, normalise mesh+joints per the upstream AugmentAffine (uniform scale, joints included in the AABB, exact [-1,1] fit), tokenize the skeleton TEACHER-FORCED (DFS stream, multi-root topologies re-parented to the first root, "articulation" cls token), then five graphs: `mesh_cond`/`vae_cond`/`embed`/`decoder` (Qwen3 KV-cache step; ships as proto + `decoder.onnx.data` external weights — ORT 1.20.1 SIGSEGVs parsing the 1.66GB single-file proto)/`skin_decode` (FSQ folded in), greedy skin-token decode constrained to the FSQ vocab, per-joint weight decode, 8-NN IDW transfer to full-res verts. **Geodesic localisation pass** in the dispatch (SkinWeights.cpp): raw SkinTokens weights are diffuse (the upstream demo voxel-masks them by default) — we filter per-vertex to GVB's geodesically-local bone sets + renormalise (bandit: bleed 0.74→0.05, mean L1 vs artist weights 1.72→1.22; GVB baseline 1.09 on that metric, but visual quality favours the ML result, hence the default). Export: `scripts/export-skintokens-onnx.py` (offline; flash-attn shim + eager attention + CPU stubs + traced FPS + RMSNorm decomposition + transformers-5.x cache API; parity vs torch ~1e-5 on every graph); hosting: `scripts/upload-skintokens-models.sh` → HF models repo `skintokens/` (~2.3GB, downloads on first use; `QTMESH_SKINTOKENS_MODEL_BASE_URL`/`ai/skintokensModelBaseUrl`, guard `QTMESH_SKINTOKENS_NO_DOWNLOAD`). ~12 min for 119 bones/90k verts on an M-series CPU — report says `algorithmUsed: "skintokens"`; every failure falls back to GeodesicVoxel with a reason. Ort C++ footgun for future consumers: `GetTensorTypeAndShapeInfo()` is a NON-OWNING view — never chain it off a temporary `TypeInfo`. Debug tracing: `QTMESH_SKINTOKENS_DEBUG=1`. - **SkinWeights** (`src/SkinWeights.h/cpp`, issue #402): inverse-distance ("closest-point-on-bone") automatic skin weights (now the fallback algorithm — see Skinning v2 above). The issue proposed wrapping libigl's bounded biharmonic weights (BBW), but BBW requires tetrahedralization via TetGen — which is **GPL/copyleft**. Adopting it would force the entire binary to GPL and close off Homebrew / Snap / WinGet redistribution under the project's permissive-license stance. This first slice ships a native heuristic with **zero new dependencies**: for each vertex, compute its distance to every bone's segment (line from bone-head to the average of its children, falling back to point distance for leaf bones in the skeleton's bind pose), apply `1/dist^falloff` weighting, keep the top-K bones (default K=4 matches hardware skinning), and normalize. This is the same algorithm Maya / 3dsMax use as their default "smooth bind." Distance cap (`maxInfluenceDistance` × mesh-diagonal) prevents a finger bone from picking up weight on a foot. Optional `skipUnweightedBones` filters Mixamo helper bones. `replaceExisting=false` enables a merge mode for "fill in missing weights" workflows. Surfaced via `qtmesh skin --max-influences N --falloff F -o out`, MCP `compute_skin_weights`, and the **Animation Mode → Mode Tools → "Skinning" section → "Compute Skin Weights…" button** (`qml/SkinWeightsDialog.qml`, driven by `SkinWeightsController` singleton). Lives in Animation Mode (not Edit Mode) because skinning governs how the mesh deforms under animation — a rigging step, not a mesh-topology edit. The button binds to `hasSkinnedSelection` so it disables on static (skeleton-less) meshes. The GUI path runs through `ComputeSkinWeightsCommand` (`src/commands/`) so the auto-skin is **undoable** (Ctrl+Z): the command snapshots every submesh's `VertexBoneAssignmentList` (+ the mesh-level shared list) before the first `redo`, runs `computeAndApply`, and on `undo` restores the snapshot and calls `_compileBoneAssignments` to re-pack the blend buffer. (Unlike the UV-unwrap restore, recompiling is safe here because the vertex buffer object is unchanged — only the blend bytes are rewritten.) Sentry breadcrumb category `ai.assist.skin_weights`. A future slice can plug libigl BBW in behind `-DENABLE_LIBIGL_BBW` for users who accept the GPL implications. Verified on Rumba Dancing.fbx: 69 bones, 5828 verts → 20,129 vertex-bone assignments (avg 3.45 influences/vert), valid glTF round-trip. - **AutoRig** (`src/AutoRig.h/cpp`, issue #407): native automatic rigging — predicts a skeleton for an unrigged mesh. The issue proposed wrapping **Pinocchio** (Baran & Popović, SIGGRAPH 2007), but Pinocchio's **core library is LGPL-2.1-or-later** (only its demo CLI is MIT). Statically vendoring LGPL imposes relink / object-file obligations that conflict with this project's statically-linked, permissively-redistributed binaries (Homebrew / Snap / WinGet / Docker) — the same reason #401 (Instant Meshes / QuadriFlow) and #402 (libigl BBW needs GPL TetGen) shipped native heuristics. Pinocchio's *algorithm* (embed a skeleton template into the mesh interior) is published and unencumbered; only its code is LGPL, so this is a from-scratch native implementation with **zero new dependencies**. Pipeline: (1) read mesh vertices → AABB; (2) each built-in template (humanoid 19-bone / biped / quadruped / generic) is a proportional joint graph in a normalised unit box; map every joint into the AABB; (3) recentre flagged joints (spine, limb roots) toward the centroid of the vertices in a thin slab at the joint's up-height — pulls the spine onto the medial line and lands limb roots inside the silhouette. `rigEntity()` builds an `Ogre::Skeleton` (parent-relative bone positions, `setBindingPose`), binds via `mesh->_notifySkeleton` **+ `entity->_initialise(true)`** — the re-initialise is REQUIRED or both exporters (FBXExporter and the Assimp glTF/FBX path gate on `entity->hasSkeleton()`) silently drop the new rig. Pure-data core (`templateJoints` / `fitTemplate`) is unit-tested without GL. Surfaced via `qtmesh rig [--skeleton T] [--skin] [--up-axis x|y|z] -o out` (`CLIPipeline::cmdRig`, optionally chains `SkinWeights::computeAndApply` for one-click rig+skin), MCP `auto_rig` `{template, skin?, up_axis?, output_path?}` (`MCPServer::toolAutoRig`), and the **Animation Mode → Mode Tools → "Rigging" section → "Auto-Rig…" button** (`qml/AutoRigDialog.qml`, driven by `AutoRigController` singleton, gated on `hasRiggableSelection` — a static/skeleton-less mesh; already-rigged meshes show the "Skinning" section instead). Sentry breadcrumb category `ai.assist.auto_rig`. **Quality limits** (documented per the issue, like Pinocchio): heuristic embedding — works best on roughly upright, single-component, manifold, T/A-pose meshes with +Y up; it does not detect limbs from topology, so exotic proportions or non-upright poses can misplace joints. Verified end-to-end: a static OBJ → 19-bone humanoid + skin → glTF export with 1 skin / 17 joints. **Mixamo-style marker placement** (refinement over the proportional fit): the user clicks the 10 humanoid markers on the mesh surface in the viewport (chin, L/R shoulder, L/R wrist, L/R hip, L/R knee, hips/pelvis — `AutoRig::humanoidMarkerOrder()`), and each placed marker anchors its joint while the limb/spine chains interpolate between the anchors so the rig follows actual body proportions instead of the fixed template. The pure-data core is `AutoRig::fitTemplateWithMarkers` and it does **coherent inference**, not per-marker patching: it runs `fitTemplate` for a proportional baseline (and to read the template's segment vectors / lateral offsets), then **resolves an anchor for every key joint** (Head, L/R Shoulder, L/R Hand, Hips, L/R UpLeg, L/R Knee) as *marked → inferred-from-marked-neighbours → template* and lays the dependent chains (spine, arms, legs) from those anchors — so a partial marker set yields an anatomically-sane skeleton instead of mixing marked anchors with stranded template joints (no shoulder-above-head). Inference: Hips ← midpoint of marked up-legs + template socket→pelvis rise; Head ← template offset above resolved Hips; UpLeg ← mirror the other up-leg across the pelvis, else pelvis + template socket offset; Shoulder ← along the live Hips→Head line at the template shoulder-height fraction + template lateral offset, else mirror the other; Hand ← shoulder + template arm vector (marked shoulder + skipped wrist still lays a full arm); Knee/foot ← clamped to the mesh AABB so an inferred leg never punches through the model: when the knee is skipped, the foot is dropped straight to the mesh FLOOR (`mn[up]`) below the up-leg and the knee placed halfway between (template thigh-vector extrapolation, which used to shoot feet past the lower limit, is only used for the small forward knee nudge); a marked knee keeps its position with the foot extrapolated below but still floor-clamped. Mirroring reflects across the sagittal plane (side axis auto-detected). An empty marker set early-returns `fitTemplate` unchanged; `report.markersApplied` counts only user-placed markers. (Legacy per-marker description retained below for the chain mechanics.) The old behaviour was: Hips→anchor pelvis AND carry the thigh roots (LeftUpLeg/RightUpLeg, children of Hips) by the same delta so the whole pelvis+thigh cluster moves as a unit — unless an explicit hip marker overrides; Chin→anchor Head AND lay the spine straight up from the pelvis — Spine/Chest/Neck distributed evenly between Hips and Head by index (cartoon torso lengths vary too much for a proportional guess); L/R shoulder→anchor the arm-chain attach point (applied before the wrist so the chain lays from the marked shoulder); L/R wrist→`layChain` lays the WHOLE arm straight from the shoulder anchor — Shoulder[anchor]→Arm[⅓]→ForeArm[⅔]→Hand[marker] — distributing every intermediate joint so the entire arm reaches the wrist, not just the hand; L/R hip→anchor the thigh root/hip socket (applied before the knee, overrides the hips-carry — needed for cartoon legs that splay at odd angles); L/R knee→`layLeg` anchors the knee at the marker and continues the foot below it along the thigh→knee direction (so the whole leg — hip socket → knee → foot — follows the marked hip + knee)). `layChain` is generic (anchor-first, marker-last, evens the middle by index) so adding more chain joints is a one-line change. Every marker is OPTIONAL — unset markers keep the template fit (`report.markersApplied` counts the placed ones; an empty marker set is bit-identical to `fitTemplate`). The viewport flow lives in `AutoRigController` (marker-session state machine: `beginMarkerPlacement`/`skipCurrentMarker`/`undoLastMarker`/`cancelMarkerPlacement`/`commitMarkerRig`); clicks are routed in by `TransformOperator::mousePressEvent` (checked **before** the knife/select paths when `markerMode()` is true), ray-cast to the mesh surface (`getCameraToViewportRay` → Möller-Trumbore against world-space triangles), stored in mesh-local space, and shown as unlit-yellow `PT_SPHERE` overlays. **The whole UI is inline in the Inspector's "Rigging" section** (`riggingToolsComponent` in `qml/PropertiesPanel.qml`) — there is no separate dialog (the old `AutoRigDialog.qml` was removed). It show/hides smartly: idle shows the two entry points ("Place markers…" / "Auto-Rig (template)"), a skin checkbox, and an "Advanced options" checkbox that reveals the template + up-axis pickers; while `markerMode` is active it swaps to the per-marker guidance label + Skip/Undo/Cancel/"Rig from markers" controls. Rig state + the `runAutoRig`/`runMarkerRig` helpers live on the `PropertiesPanel` root; the section's `onSectionVisibleChanged` cancels any active marker session if the section disappears (mode change / deselect / re-rig), replacing the dialog's old `onClosing` cancel. No CLI/MCP marker surface — guided placement is inherently interactive. **UniRig ML backend** (issue #408): `AutoRig::Algorithm {Pinocchio, UniRig}` selects the skeleton-prediction backend (default Pinocchio — offline, deterministic). **UniRig** (Zhang et al., *"One Model to Rig Them All"*, SIGGRAPH 2025, VAST-AI-Research/UniRig — **MIT code + MIT weights**, trained on Articulation-XL2.0 **CC-BY-4.0**) is an autoregressive transformer that predicts a skeleton from the mesh geometry, handling arbitrary/non-humanoid topology better than the fixed template. It's the **second ONNX consumer** after #404 PbrMapSynth. **RigNet was rejected** for #408 (GPL code + unlicensed weights + non-public ModelsResource dataset — fails the project's permissive-redistribution bar); UniRig is the clean permissively-licensed alternative (see `THIRD_PARTY_AI_MODELS.md`). `UniRigPredictor` (`src/UniRigPredictor.h/cpp`, Ogre-free + unit-tested) is the C++/ONNX runtime that ports UniRig's skeleton stage: (1) surface-sample up to 65536 points + normals, normalise into a centred unit box (+Y up); (2) run the **Michelangelo encoder** (`encoder.onnx`, pc[1,N,3]+feats[1,N,3] → latent prefix); (3) **greedy/constrained autoregressive decode** over the ~350M causal-LM (`decoder.onnx`) with a manual KV-cache + the tokenizer's next-possible-token validity mask (a documented simplification of UniRig's beam+sampling — deterministic + exportable, still yields a valid tree); (4) the **exact tokenizer FSM** from `src/tokenizer/tokenizer_part.py` (256 coord bins, `continuous_range [-1,1]`, `undiscretize(t)=(t+0.5)/256*2-1`, branch/parent rules, vocab 267) → joints (de-normalised) + parent indices, parent-before-child ordered for Ogre. The detokenizer + `undiscretize` are public statics (`UniRigPredictor::detokenize`/`undiscretize`) so they're unit-tested without ONNX. Everything is `ENABLE_ONNX`-guarded; **two** model files (`AppData/ai_models/unirig/{encoder,decoder}.onnx`) download on first use via `ModelDownloader` (`ensureModelBlocking`, 180s timeout, returns the encoder path only when BOTH exist; base URL override `QTMESH_UNIRIG_MODEL_BASE_URL` / `QSettings ai/unirigModelBaseUrl`, offline guard `QTMESH_UNIRIG_NO_DOWNLOAD`). **UniRig falls back to Pinocchio** (logged in `report.fallbackReason`) when ONNX is off / the models are missing/offline/not-yet-hosted / prediction is unusable — reliable offline. **Design contract / hosting status:** UniRig is an autoregressive HF `AutoModelForCausalLM` + a Michelangelo perceiver — no single-graph ONNX export exists upstream; `scripts/export-unirig-onnx.py` (one-time, offline, NOT shipped) exports the encoder + a KV-cache decoder to the I/O `UniRigPredictor` targets (via `optimum`, with a hand-rolled fallback). Until the exported `.onnx` files are hosted on the HF models repo, the download 404s and the Pinocchio fallback runs — the plumbing + runtime are complete and ship today; hosting the export lights up the ML path with no code change. UniRig is marker-incompatible (markers are a template concept), so a marker-driven call always uses the template. Surfaced via CLI `qtmesh rig --algo pinocchio|unirig` (`CLIPipeline::cmdRig`; `rignet` accepted as a deprecated alias), MCP `auto_rig` `algo` param (`MCPServer::toolAutoRig`), and the Inspector Rigging-section **Algorithm** segmented picker; the report carries `algorithmUsed` + `fallbackReason`, and the Sentry `ai.assist.auto_rig` breadcrumb records the `algo`. (The early `qml/AutoRigDialog.qml` reference above is stale — the UI is inline in `qml/PropertiesPanel.qml`'s Rigging section.) +- **FaceRig** (`src/FaceRig/`, epic #889): auto-generate the 52 **ARKit blendshapes** on an unrigged humanoid **face** mesh so it can be driven by face performance capture (#869). **Deterministic geometry, no ML/ONNX, zero new dependencies** (the house rule, same as #402/#407). Pipeline: **`NonRigidICP`** (`NonRigidICP.{h,cpp}`, Amberg 2007 optimal-step, Slice C #892) fits the ICT-FaceKit template neutral onto the user's neutral → per-template-vertex correspondence; **`DeformationTransfer`** (`DeformationTransfer.{h,cpp}`, Sumner & Popović 2004, Slice D #893) builds each template triangle's deformation gradient `S = [e1' e2' n']·[e1 e2 n]⁻¹` (4th "normal" vertex trick, free per-triangle unknown, single-vertex gauge anchor) and solves one sparse least-squares per shape for the user-identity positions; **`FaceRigger`** (`FaceRigger.{h,cpp}`, Slice E #894) chains them and resamples template-topology deltas onto the real user verts via a dependency-free spatial-hash grid (built once for all 52 shapes) → per-user-vertex deltas per shape, named per `FaceCap::kBlendshapeNames`. The shared sparse solver is **`SparseSolve.{h,cpp}`** (CSR + CG on the normal equations — no Eigen). **`FaceRigAttach`** (`FaceRigAttach.{h,cpp}`) is the only Ogre-touching piece: `extractGeometry()` reads the entity's combined submesh geometry, and `attachShapes()` splits the deltas back per submesh handle and attaches them as `Ogre::Pose` + `VAT_POSE` morph targets via `AddMorphTargetCommand` (the exact MorphCommands pose-build, so face capture drives them with no new playback code). **Humanoid-only gate**: the NRICP fit residual is checked (`--max-residual`, default 8% of the mesh diagonal; non-finite / >5%-diverged fits treated as failed) — a non-face mesh fits poorly and is refused rather than emitting garbage. Surfaced via **CLI `qtmesh facerig -o [--max-shapes N] [--max-residual PCT] [--json]`** (`CLIPipeline::cmdFaceRig`), **MCP `add_arkit_blendshapes`** (`MCPServer::toolAddArkitBlendshapes`, `{max_shapes?, max_residual_pct?, output_path?}`, heavy), and the **Inspector Vertex Morph Animation → "✨ Add ARKit Blendshapes (AI)" button** (`FaceRigController` QML_SINGLETON — extracts + loads template on the main thread, runs the heavy fit on a WORKER thread, commits the attach as one undo macro on the main thread; gated on `hasMeshSelection`, shows "Downloading…/Fitting…" status). Template = **ICT-FaceKit** (MIT, `THIRD_PARTY_AI_MODELS.md`), packed by `scripts/export-arkit-template.py` → `facerig/arkit_template.bin`, hosted on the HF models repo (`scripts/upload-facerig-template.sh`), downloads on first use to `/ai_models/facerig/` (`ArkitTemplate::ensureModelBlocking`; overrides `QTMESH_FACERIG_MODEL_BASE_URL` / `QSettings ai/facerigModelBaseUrl`; offline guard `QTMESH_FACERIG_NO_DOWNLOAD`). Sentry breadcrumb `ai.assist.face_rig`. Verified end-to-end: real ICT template (26719v, 51 shapes) → decimated different-topology face (15755v), mean 0.008% / max 0.61% fit, 51 shapes attached, exported glb carries all 51 morph targets. **glTF export** carries the target geometry but not per-target `extras.targetNames` (a follow-up). Docs: `docs/FACE_RIG.md`; spike/contract: `docs/FACE_RIG_SPIKE.md`. - **QuadRetopo** (`src/QuadRetopo.h/cpp`, issue #401): triangle-pairing quad-dominant retopology. The issue proposed wrapping Instant Meshes (Wenzel Jakob), but Instant Meshes ships as a research GUI app with no clean C++ library API and has been dormant since 2016. QuadriFlow (the production-grade alternative used by Blender 3.0+) requires Boost + Eigen + LEMON — heavy deps the project doesn't currently use. This first slice ships a native triangle-pairing backend with **zero new dependencies**: walks every interior edge whose two adjacent faces are triangles and scores the merge by (1) coplanarity (dot product of triangle normals; default `maxAngleDeg=25°`), (2) quad shape (deviation of interior angles from 90°; default `shapeToleranceDeg=65°`), (3) aspect ratio (longest/shortest edge; default `maxAspectRatio=6.0`). Pairs are taken greedily best-first; each triangle claimed at most once. Quads are emitted with opposing-corner winding `(opposing0, sharedA, opposing1, sharedB)`. Output goes through `EditableSubMesh::faces` → `triangulateFaces` (fan retri for GPU) → `writeNgonFacesToMesh` (n-gon binding for exporters / Edit Mode). **No new vertices** are introduced, so UVs and skin weights survive unchanged. Backends are pluggable via the `Algorithm` enum (only `TrianglePair` implemented; future `QuadriFlow` / `InstantMeshes` slot in here). Surfaced via `qtmesh retopo --target-faces N --max-angle DEG -o out`, MCP `retopologize`, and the **Material Mode → Mode Tools → "Quad Retopology…" button** (`qml/QuadRetopoDialog.qml`, driven by `QuadRetopoController` singleton). Sentry breadcrumb category `ai.assist.retopo`. Verified on Rumba Dancing.fbx: 10,220 tris → 6,032 faces (4188 quads + 1844 tris), 82% quad dominance. Hard lower bound on face count is ~50% of input (every triangle paired); strict gates typically land 60-70%. - **MeshSegmenter** (`src/MeshSegmenter.h/cpp`, issue #410): AI mesh part segmentation — predicts a semantic part label (head/torso/left+right arm/left+right leg) per vertex + per face. The **fourth ONNX consumer**; powers Edit-Mode "Select by part", per-part material assignment, and auto-rig priors. **Geometric fallback is first-class** (always compiled, Ogre-free): `segmentGeometric` does connected-component islands (`connectedComponents`, union-find) + an up-axis/lateral spatial heuristic (top→head, lower→legs, mid-sides→arms, centre→torso), overridable per-vertex by rig bone-proximity hints — used automatically when the build lacks ONNX, the model is missing/un-downloadable, or inference fails (`Result::usedModel`/`fallbackReason` report which ran). The **ONNX path** (`#ifdef ENABLE_ONNX`, PointNet++-style) normalises → deterministic point sample → `[1,N,3]` → per-point argmax over the part channels → scatters labels back to all vertices by nearest sampled point (runtime I/O-name discovery, channels-first/last handling, CoreML EP). `ensureModelBlocking()` downloads `meshseg.onnx` to `AppData/ai_models/segment/` (override `QTMESH_SEGMENT_MODEL_BASE_URL` / `QSettings ai/segmentModelBaseUrl`; offline guard `QTMESH_SEGMENT_NO_DOWNLOAD`; non-ONNX `#ifndef` guard) — the #408/#409 pattern. Pure-data helpers (`connectedComponents`, `facesFromVertexLabels`) are unit-tested without Ogre/GL. Surfaced via **CLI `qtmesh segment [--json] [--no-model] [--up-axis x|y|z]`** (`CLIPipeline::cmdSegment` — text per-part counts or full label arrays), the **MCP `segment_mesh` tool** (`MCPServer::toolSegmentMesh`, args `{entity_name?, no_model?}`, heavy), and the **Edit Mode → "Select by Part (AI)" button** (`EditModeController::selectByPart()` → selects all faces matching the selected face's part, or the largest part if none selected; pushes via `selectFace` so the existing highlight refreshes). Sentry breadcrumb `ai.assist.segment`. **Model: ours (v2), trained on surface-sampled synthetic bodies (humanoid/chibi/quadruped/biped-tail plans) + mined CC0 Quaternius rigs** (rig bone-weight → part; ShapeNet-Part/PartNet are non-commercial and rejected) via `scripts/export-meshseg-onnx.py` (offline, not shipped) — the v2 loader canonicalises arbitrarily-oriented mined clouds from their own labels and geometrically fixes miner side errors; hosted on the HF models repo under `segment/` (see `THIRD_PARTY_AI_MODELS.md` + `docs/MESH_SEGMENTATION_STRATEGY.md` for the v1 failure analysis, accuracy numbers, and the multi-category roadmap). **Three-tier dispatch in `selectByPart`**: (1) **rig-prior** — if the mesh is SKINNED, label each vertex by the part of the bone it's most-weighted to (`AutoRig::rigPriorPartLabels` → `MeshSegmenter::partForBoneName`); EXACT and handles non-human anatomy (ears/snout→head, tail→torso, paws→leg) the coordinate model can't. Used when it resolves ≥70% of vertices. (2) **ONNX model** (UNrigged meshes). (3) **geometric fallback**. The ONNX path also applies `Options::upAxis` by remapping the sampled point cloud to the model's +Y-up training frame before inference (and in the nearest-point scatter), so X/Z-up meshes aren't mislabelled. **Continual-training miner** (the "train further as we gather data" loop): `qtmesh segment --dump-training-data out.json` runs the rig-prior path and writes the normalised point cloud + EXACT per-vertex labels (schema `qtmesh-meshseg-training-v1`) — every rigged asset becomes one free, exactly-labelled sample. `scripts/export-meshseg-onnx.py --real-data ` MIXES those mined JSONs (with yaw/tilt/jitter aug) into the synthetic set and retrains; gains land on the MODEL path used for unrigged meshes (rigged meshes already use the exact rig-prior path in-app). `AutoRig::rigPriorPartLabels` is the shared extractor for the GUI fast-path and the miner, so the in-app selection and mined ground truth are bit-identical. - **Image-to-3D (TripoSR)** (`src/ImageTo3D/`, epic #764): single-image → 3D mesh generation via **TripoSR** (Tripo AI + Stability AI, **MIT code AND MIT weights**, HF `stabilityai/TripoSR`). The **fifth ONNX consumer** (after #404/#408/#409/#410); all files live in the `src/ImageTo3D/` feature folder. MIT code+weights is the deciding factor for redistribution (Homebrew/Snap/WinGet/Docker) — the bar UniRig #408 cleared and non-commercial SF3D failed. **`MeshGenPredictor`** (Ogre-free + unit-tested) runs two exported ONNX graphs — encoder `image[1,3,512,512]→scene_codes[1,3,40,64,64]` (triplane) and per-point decoder `scene_codes+points[1,P,3]→density[1,P,1],color[1,P,3]` — GENERATING query points per chunk (not the whole `res³` grid up front — that would OOM at 512) and extracting the surface with **`MarchingCubes`** (native Lorensen impl, public-domain tables, zero deps; TripoSR's `torchmcubes` is torch/GPU-only). Surface = MC on `density − threshold` at iso 0 (threshold 25.0, radius 0.87); our MC is inside-positive so `extract()` emits `v0,v2,v1` (flipped winding) to keep faces OUTWARD (else the mesh renders inside-out). **Model size tiers** (`MeshGenPredictor::Quality {Fp32,Int8}` → `triposr_encoder{,_int8}.onnx`): fp32 ~1.68 GB (best), int8 ~430 MB (slight quality loss); user-selectable, downloads on demand. (fp16 was dropped — TripoSR's attention has a hardcoded Cast-to-float32 the ONNX fp16 converters can't rewrite; int8 is smaller anyway.) **`MeshGenBuilder`** (the ONLY Ogre-touching piece) turns the arrays into an `Ogre::Mesh` (POSITION + accumulated per-vertex NORMAL + optional DIFFUSE `VET_COLOUR` with a lit vertex-color material; 16-/32-bit index by vertex count; validates index data first), **bakes -90°X + +90°Y** into positions+normals so the model stands upright and faces forward, uses a UNIQUE per-call node/mesh name, and returns the SceneNode for export. **Background removal:** `BackgroundRemover` (6th ONNX consumer) runs **U²-Net** (Apache-2.0, rembg's model) to isolate the subject: `[1,3,320,320]`→`[1,1,320,320]` saliency, then composites over **gray 128** (not white — white → a reconstructed wall) and crops/re-pads to the subject at 0.85 foreground ratio (TripoSR's `resize_foreground`). Model `ai_models/rembg/u2net.onnx` (`QTMESH_REMBG_MODEL_BASE_URL`/`ai/rembgModelBaseUrl`; guard `QTMESH_REMBG_NO_DOWNLOAD`); falls back to the raw image if unavailable. Everything `ENABLE_ONNX`-guarded; **no fallback** (generative), so a non-ONNX build / missing model returns a clear error (never crashes). Models under `ai_models/triposr/` download on first use (`ensureModelBlocking(q)`; `QTMESH_TRIPOSR_MODEL_BASE_URL`/`ai/triposrModelBaseUrl`; guard `QTMESH_TRIPOSR_NO_DOWNLOAD`), OR can be **pre-downloaded from the AI Settings modal's Download tab** (tier picker + progress bar). **Export is `scripts/export-triposr-onnx.py`** (offline, not shipped; `transformers==4.35.0`, `torchmcubes` stub, frozen ViT pos-encoding; emits the int8 variant unless `--no-quant` — see `docs/IMAGE_TO_3D_SPIKE_764.md`). Surfaced via **CLI `qtmesh generate3d [-o out.glb] [--resolution 16..1024] [--no-color] [--remove-bg] [--quality fp32|int8]`** (`CLIPipeline::cmdGenerate3d`), **MCP `generate_mesh_from_image`** (`MCPServer::toolGenerateMeshFromImage`, args `{image_path, output?, resolution?, vertex_color?, remove_bg?, quality?}`, heavy, ONNX-guarded schema), and the **Object Mode Tools → "AI: Image → 3D" inspector section** (`qml/PropertiesPanel.qml` → **`MeshGenController`**, a QML_SINGLETON that runs the whole pipeline on a WORKER THREAD — UI stays responsive — with a select-image→preview→generate flow, resolution + model-tier dropdowns, progress bar, and cancel; mesh construction is marshalled back to the main thread). Sentry breadcrumb `ai.assist.image_to_3d`. Verified end-to-end on macOS. **Models are HOSTED** on the `fernandotonon/QtMeshEditor-models` HF repo (`triposr/triposr_encoder.onnx` + `triposr_encoder_int8.onnx` + `triposr_decoder.onnx`, `rembg/u2net.onnx`) via `scripts/upload-triposr-models.sh` — first use downloads them; if ever absent, every surface reports a clean "not yet hosted" message (no crash). Design/spike note: `docs/IMAGE_TO_3D_SPIKE_764.md`; slices A #765 (spike) → B #766 predictor → C #767 mesh build → D #768 surfaces → E #769 tiers/pre-download/hosting/docs (all in PR #785). **Quality pass (post-#785, ON by default)**: after marching cubes the predictor runs (a) **`MeshRefine::taubinSmooth`** — Taubin λ|μ smoothing (volume-preserving, kills the res³-grid stair-stepping), (b) **`MeshRefine::isoProjectStep`** — one Newton step per vertex back onto the decoder's true iso-surface using forward-difference gradients from 4 extra decoder probes/vertex (recovers grid-quantized detail; both pure-data + unit-tested in `MeshRefine_test.cpp`), and (c) **`MeshGenBaker`** — xatlas auto-unwrap + UV-space triangle rasterization + per-texel decoder colour queries + chart-border dilation, producing UV0 + a real diffuse TEXTURE (default 1024²) instead of per-vertex colour — colour sharpness then scales with texture size, not vertex density (pure-data behind a `ColorSampler` callback; `MeshGenBaker_test.cpp`). `MeshGenBuilder` gained the textured path: saves the baked PNG (AppData/generated_textures/ or the export dir when given), registers the dir as a resource location, and binds a lit material with a named `diffuse_map` TUS. Bake failure falls back to vertex colours with `Result::warning` set (never fails the generation). **PBR stage (d, ON by default)**: `MeshGenBuilder::BuildOptions::generatePbrMaps` chains **#404 PBR map synthesis** onto the baked diffuse — normal + roughness PNGs written next to it (height skipped, no consumer) and bound into the material via the same recipe as the Material Editor's "Generate PBR maps from diffuse" button (canonical `normal_map`/`roughness` TUS + `wirePbrSlotsForFFP` + `RTShaderHelper::applyNormalMap` — without applyNormalMap the bind is invisible in the viewport — + recompile). This is what turns the flat diffuse-only result into a polished, surface-detailed one; fails soft to diffuse-only when the PBRify models are unavailable. The exported material references all three maps (FBX embeds them; the PNGs land next to the export). **Every stage is user-selectable**: GUI checkboxes in the AI section (Remove background / Smooth / Refine / Bake texture / PBR maps / Upscale 2×) feed an options QVariantMap into `MeshGenController::generateSelected`; CLI `--no-smooth --no-refine --no-bake-texture --no-pbr --texture-size N --upscale-texture`; MCP `smooth/refine/bake_texture/generate_pbr/texture_size/upscale_texture`. The GUI runs the upscale on the WORKER thread (model pre-ensured on the main thread) and the PBR synthesis on the main thread inside buildSceneNode (small models, Material-Editor precedent). **TripoSG backend** (`src/ImageTo3D/TripoSGPredictor.{h,cpp}`, the SEVENTH ONNX consumer): `MeshGenPredictor::Options::backend {TripoSR|TripoSG}` dispatches to **TripoSG** (VAST-AI, SIGGRAPH 2025, **MIT code + MIT weights**, geometry ≈ commercial Tripo 2.0) — a 1.5B rectified-flow DiT over an SDF VAE, run as FOUR exported graphs (`scripts/export-triposg-onnx.py`, offline dev tool; measured contract in `docs/TRIPOSG_EXPORT_NOTES.md`): DINOv2-224 image encoder (mean/std baked in; CFG uncond = zeros) → **C++ Euler flow loop** over the DiT step graph (σᵢ = 1−i/N, timestep = 1000·σ, update `x += (σᵢ−σᵢ₊₁)·v` — sign is OPPOSITE of stock diffusers FlowMatchEuler; CFG as two B=1 calls, guidance 7.0, steps knob default 25) → VAE latent kv-cache graph (run ONCE per generation) → per-point field decoder (already inside-positive, iso 0, bounds ±1.005) → the same native MarchingCubes + smooth/reproject polish. Geometry-only (no colour decoder): bake/PBR/upscale stages are TripoSR-only; background removal for TripoSG composites over WHITE (its reference pipeline) vs TripoSR's gray-128. fp32 DiT ships as `.onnx`+`.onnx.data` (>2 GB external weights) with an int8 single-file tier mapped from `Quality::Int8`. Models under `ai_models/triposg/` download on first use (`QTMESH_TRIPOSG_MODEL_BASE_URL`/`ai/triposgModelBaseUrl`; guard `QTMESH_TRIPOSG_NO_DOWNLOAD`); clean "not hosted yet" error until the export is run + hosted. Surfaced via CLI `--backend triposr|triposg --flow-steps N`, MCP `backend`/`flow_steps` args, and the GUI Backend dropdown (the step list gains a "Denoise (flow steps)" row via `Stage::Denoise`). Roadmap/audit: `docs/IMAGE_TO_3D_QUALITY.md`. **TripoSG post-integration updates (supersede the "geometry-only / int8 tier / white-bg / disabled texture checkboxes" claims above):** (1) **int8 tier DROPPED** — even per-channel-quantized, the 1.5B DiT degrades to blobs over the 25-step CFG flow loop (live-verified), and dynamic-int8 MatMuls are no faster than fp32 on ARM; all surfaces force fp32 (CLI prints a note; the GUI Model picker collapses to "fp32 (only option for TripoSG)" and locks; the `quality` param now only selects the TripoSR tier used for the colour bake). (2) **Colour** — TripoSG has no colour decoder, so `MeshGenPredictor::colorizeWithTripoSR` bakes colour by (a) projecting the actual input PHOTO onto the visible front (depth-buffer-gated front-most-surface test; camera looks toward +Z so nearest = max z; soft depth-band crossfade to the field) and (b) filling occluded/back texels from **TripoSR's image-conditioned colour field** (the TripoSG mesh mapped into TripoSR's native frame + per-axis affine-fit onto its occupied bounds). The front is photo-accurate; the back is inferred/approximate. Falls soft to a shared neutral **lit clay material** (`MeshGen/NeutralClay`) on any failure. Texture/PBR/upscale stages + their GUI checkboxes are ENABLED for TripoSG (route through the colour bake). (3) **AI texture (GUI, `ENABLE_STABLE_DIFFUSION`)** — a "Generate texture (AI, front photo + generated back)" checkbox runs the existing **multi-view depth-ControlNet bake** (`MaterialEditorQML::generateMeshTextureMultiView`, `MultiViewTextureBaker`) after the mesh builds, with the input photo PINNED as the front view (img2img is disabled on Metal, so the photo is injected as a filled view rather than an init image) and back/sides SD-generated; needs a loaded SD model. (4) **Orientation** — TripoSG output is already +Y-up (`Result::bakeTripoSROrientation=false` skips the TripoSR -90°X/+90°Y bake); its decoder field is negated at the sample site (exported graph lands OUTSIDE-positive → inverted winding otherwise). (5) **Memory/speed** — decoder chunk hard-capped at 8192 pts (cross-attention to 2048 kv tokens; TripoSR's 262144 chunk OOM-killed at ~90 GB); ONNX sessions staged (opened/released per stage, ~1 GB peak vs the >4 GB sum); the ~48 MB point decoder can run on the CoreML GPU via `QTMESH_TRIPOSG_COREML_DECODER=1` (default CPU — per-call kv re-upload made GPU slower); `--guidance` knob (CLI/MCP). Next speed win: hierarchical extraction (coarse grid → refine near surface). SF3D (non-commercial) and Hunyuan3D (EU-excluded) rejected for the texture upgrade; MV-Adapter (VAST-AI, Apache-2.0) is the tracked multi-view candidate. diff --git a/README.md b/README.md index 0ef1683b9..72884ad4b 100755 --- a/README.md +++ b/README.md @@ -186,6 +186,10 @@ qtmesh retopo model.fbx --max-angle 15 -o conservative.glb # tighter coplanarit qtmesh skin model.fbx -o skinned.glb # default 4 influences, falloff 4 qtmesh skin model.fbx --max-influences 8 --falloff 6 -o skinned.glb qtmesh skin model.fbx --skip-unweighted --merge -o filled.glb # fill missing weights only + +# Auto-generate the 52 ARKit blendshapes on a humanoid FACE mesh (for face capture) +qtmesh facerig head.glb -o rigged.glb # fit ARKit template + transfer 52 shapes +qtmesh facerig head.fbx -o rigged.glb --max-shapes 20 --json # cap shapes / machine-readable report ``` --- diff --git a/THIRD_PARTY_AI_MODELS.md b/THIRD_PARTY_AI_MODELS.md index 5384598a6..a7fd6f169 100644 --- a/THIRD_PARTY_AI_MODELS.md +++ b/THIRD_PARTY_AI_MODELS.md @@ -271,6 +271,47 @@ the binary). Attribution + licenses for the models and their training data: checkbox; the template library remains the default and the automatic fallback. Same CMU licensing basis as above. +## ICT-FaceKit — ARKit blendshape template for face auto-rig (epic #889) + +- **Asset (not a learned model):** the ICT-FaceKit generic neutral head + (`generic_neutral_mesh.obj`) + its per-expression meshes named after the + ARKit blendshapes (`jawOpen`, `mouthSmile_L`, `eyeBlink_L`, `browInnerUp_L`, + …), all sharing one topology (26,719 verts) so each shape = expr − neutral. +- **Source / license:** [USC-ICT/ICT-FaceKit](https://github.com/USC-ICT/ICT-FaceKit) + — **MIT** (Copyright 2020 USC Institute for Creative Technologies). The + standard/released model is MIT; a separate "full model" tier under a + USC-specific license is **REJECTED** (we ship only the MIT tier). MIT clears + the permissive-redistribution bar, so the template + shapes are hostable on + the `fernandotonon/QtMeshEditor-models` HF repo (Slice B, #890) — packed by + `scripts/export-arkit-template.py` into `facerig/arkit_template.bin` and + uploaded by `scripts/upload-facerig-template.sh`; it downloads on first use. +- **How it is used:** the template is the *source* for **deformation transfer** + (Sumner & Popović 2004) — QtMeshEditor fits it to the user's neutral head via + native non-rigid ICP (Amberg 2007), then transfers each of the 52 ARKit + expressions onto the user's topology, attaching them as `Ogre::Pose` morph + targets so face performance capture (#869) works on the mesh. **No ML model, + no ONNX** — it is a deterministic geometry algorithm (sparse linear solve), + implemented natively in `src/FaceRig/` (Slices C/D/E). The offline spike + (`scripts/spike-facerig.py`, not shipped) validated the approach first + (see `docs/FACE_RIG_SPIKE.md`). Verified end-to-end on + a decimated, different-topology face: mean 0.008% / max 0.61% NRICP fit and + 51 attached shapes. Surfaced via `qtmesh facerig`, MCP `add_arkit_blendshapes`, + and the Inspector "Add ARKit Blendshapes" button. See `docs/FACE_RIG.md`. +- **Facial-landmark anchoring (landmark pass):** the NRICP fit is anchored to + real face features by **MediaPipe Face Mesh V2** (`face_landmarks.onnx`, + **Apache-2.0** — the same model the mocap face-capture uses, #869). We render + the head front-on, detect the 478 landmarks, back-project them to the mesh + surface, and pin the matching template vertices — so the template lands on the + actual eyes/nose/mouth instead of a low-residual-but-mis-oriented drape. Hosted + under `facerig/face_landmarks.onnx` (a copy of the mocap graph); downloads on + first use; `ENABLE_ONNX`-guarded with a graceful unanchored-fit fallback. +- **Rejected alternatives:** Wrap3D (commercial, used by the reference impl for + NRICP — we implement NRICP natively instead), FLAME-based 3DMMs + (research-only), any generative expression model on non-commercial data. + Landmark detectors trained on 300W / WFLW / InsightFace (dlib, PIPNet, + 2d106det) were rejected — their weights carry research-only / non-commercial + terms; MediaPipe FaceMesh (Apache-2.0) is the clean choice. + All of the above clear QtMeshEditor's permissive-redistribution bar (MIT app, distributed via Homebrew / WinGet / Snap / Docker). GPL/CC-BY-NC/unlicensed models are deliberately excluded (e.g. RigNet was rejected for #408 — GPL code + diff --git a/docs/FACE_RIG.md b/docs/FACE_RIG.md new file mode 100644 index 000000000..71484397f --- /dev/null +++ b/docs/FACE_RIG.md @@ -0,0 +1,100 @@ +# Face auto-rig: ARKit blendshapes on any humanoid face + +Epic [#889](https://github.com/fernandotonon/QtMeshEditor/issues/889). Given an +unrigged neutral **face** mesh, QtMeshEditor generates the 52 **ARKit** +blendshapes (`jawOpen`, `mouthSmileLeft`, `eyeBlinkLeft`, `browInnerUp`, …) and +attaches them as morph targets, so the mesh can be driven by face performance +capture ([#869](https://github.com/fernandotonon/QtMeshEditor/issues/869), +`qtmesh mocap --face`) — no manual sculpting of blend shapes. + +It is **deterministic geometry**, not an ML model: a non-rigid fit of a +permissively-licensed template face onto yours, followed by deformation +transfer of each expression. No ONNX, no GPU, no network beyond a one-time +template download. + +## Using it + +### GUI +Select a face mesh → Inspector → **Vertex Morph Animation** (Edit Mode) → +**"✨ Add ARKit Blendshapes (AI)"**. The fit runs on a worker thread (the button +shows *Downloading… / Fitting…*); when it finishes the 52 shapes appear in the +Shapes list and the whole batch is a single undo step. The +[#869](https://github.com/fernandotonon/QtMeshEditor/issues/869) Performance +Capture panel then drives them. + +### CLI +```bash +qtmesh facerig neutral_head.glb -o rigged.glb +qtmesh facerig head.fbx -o rigged.glb --max-shapes 20 # cap the shape count +qtmesh facerig head.glb -o rigged.glb --max-residual 5 # stricter humanoid gate +qtmesh facerig head.glb -o rigged.glb --json # machine-readable report +``` + +### MCP +`add_arkit_blendshapes` — `{max_shapes?, max_residual_pct?, output_path?}`, +operates on the selected entity; re-exports when `output_path` is given. + +## How it works + +``` +ArkitTemplate (ICT-FaceKit neutral + 52 expression deltas, one topology) + │ + ▼ HEAD ISOLATION — rig-prior (skinned) or MeshSegmenter picks the head + │ region, so a full-body character fits the face template only on the + │ head (not smeared across the body). + │ + ▼ FACE-LANDMARK ANCHORS — render the head front-on, detect 478 MediaPipe + │ landmarks (FaceLandmarkDetector), back-project to the surface on BOTH + │ template + user, pair by index → anchor constraints. This locks the + │ fit onto the real eyes/nose/mouth (the fix for wrong shape placement). + │ + ▼ NonRigidICP (Amberg 2007 optimal-step, landmark-anchored) src/FaceRig/NonRigidICP + correspondence X — template verts fitted onto the USER surface + │ + ▼ DeformationTransfer (Sumner & Popović 2004) src/FaceRig/DeformationTransfer + per-template-vertex delta per shape, on the user identity + │ + ▼ resample template topology → the real user vertices src/FaceRig/FaceRigger + 52 × per-user-vertex deltas + │ + ▼ Ogre::Pose + VAT_POSE morph targets, named per FaceCap::kBlendshapeNames +``` + +- **`FaceRigger` / `FaceRigAttach`** (`src/FaceRig/`) orchestrate the pipeline; + the pure-data core is Ogre-free and headless-unit-tested. +- The linear solves use a self-contained sparse CG (`SparseSolve`) — **no Eigen, + no external solver, zero new dependencies** (the house rule for these + features, same as skinning #402 and auto-rig #407). +- The template (`facerig/arkit_template.bin`, ICT-FaceKit MIT — see + `THIRD_PARTY_AI_MODELS.md`) downloads on first use to + `/ai_models/facerig/`. Overrides: `QTMESH_FACERIG_MODEL_BASE_URL` / + `QSettings ai/facerigModelBaseUrl`; offline guard `QTMESH_FACERIG_NO_DOWNLOAD`. +- **Landmark anchoring** (`FaceLandmarkDetector` + `FaceRigLandmarks`, + `ENABLE_ONNX`): renders the head and runs MediaPipe FaceMesh V2 + (`facerig/face_landmarks.onnx`, Apache-2.0) to anchor the fit to real face + features. When ONNX is off, the model is missing, or no face is detected, the + fit runs **unanchored** (the previous behaviour) — the feature degrades + gracefully, it never blocks a rig. + +## Quality & limits + +- **Humanoid faces only.** The fit residual is a gate: a non-face mesh fits + poorly and is **rejected** (`--max-residual`, default 8% of the mesh + diagonal), rather than emitting garbage shapes. This mirrors the + AutoRig/Pinocchio precedent. +- **Measured** (decimated, different-topology ICT head, 15 755 verts): NRICP + fit **mean 0.008% / max 0.61%** of the diagonal; **51 shapes** attached; + jawOpen drops the lower face while the forehead stays still; mouthSmile / + eyeBlink / browInnerUp localise to their regions. +- **Orientation:** the mesh should be roughly upright, +Y up, facing the + template's orientation. A wildly rotated head may fit poorly. +- **glTF export** carries the morph-target geometry on the primitive; per-target + *names* in glTF `extras.targetNames` are a follow-up (the in-editor targets + and the mocap hand-off use the correct names regardless). + +## Related + +- `THIRD_PARTY_AI_MODELS.md` — ICT-FaceKit licensing verdict. +- `docs/FACE_RIG_SPIKE.md` — the offline feasibility spike + the C/D contract. +- Epic [#889](https://github.com/fernandotonon/QtMeshEditor/issues/889); + slices B–G (#891–#895 + the polish slice). diff --git a/docs/FACE_RIG_SPIKE.md b/docs/FACE_RIG_SPIKE.md new file mode 100644 index 000000000..0b6d2c4ac --- /dev/null +++ b/docs/FACE_RIG_SPIKE.md @@ -0,0 +1,121 @@ +# Face auto-rig — Spike Findings & Contract (#889 / slice A #896) + +**Epic:** [#889 — AI: Auto-generate ARKit blendshapes on any humanoid mesh (deformation transfer)](https://github.com/fernandotonon/QtMeshEditor/issues/889) +**Slice:** [#896 — Spike: NRICP feasibility + ICT-FaceKit licensing due-diligence](https://github.com/fernandotonon/QtMeshEditor/issues/896) +**Status:** Spike — **GO.** Non-rigid ICP fits the MIT ICT-FaceKit template to +a different-topology head to sub-1% accuracy, and deformation transfer produces +anatomically-correct ARKit blendshapes on the user's own topology. No ML, no +ONNX — a deterministic sparse-linear-algebra pipeline. Slices C/D are a native +C++ port of the proven `scripts/spike-facerig.py`. + +--- + +## TL;DR — Recommendation: **GO** + +- **Template licensing — CLEARS THE BAR.** ICT-FaceKit is **MIT** (a neutral + head + 52 ARKit-named expression meshes, all one topology). Redistributable + on the HF models repo. The separate "full model" USC-specific tier is + rejected; we ship only the MIT tier. Recorded in `THIRD_PARTY_AI_MODELS.md`. +- **NRICP — WORKS.** Amberg-2007 optimal-step, pure numpy/scipy (no Wrap3D): + fit the 26,719-vert template onto a **12,763-vert (different topology)** user + head → surface fit **mean 0.003%, max 0.59%** of the head diagonal. +- **Deformation transfer — WORKS.** Sumner & Popović 2004: each of the 52 ICT + expressions transfers onto the user topology with correct semantics — + jawOpen drops the lower face (mean ΔY −0.32) while the forehead stays still + (|Δ| 0.002); eyeBlink stays localized to the eye (1,328 verts), jawOpen is + the biggest deformation (6,771 verts, 9% max), browInnerUp is small (1.3%). +- **No new runtime dependency:** deterministic geometry (sparse solve), like + `GeodesicVoxelBind` / `QuadRetopo`. No model download at inference (only the + MIT template asset downloads on first use, like other bundled assets). + +--- + +## The template (ICT-FaceKit, MIT) + +`FaceXModel/` ships `generic_neutral_mesh.obj` + one `.obj` per expression, ALL +sharing the neutral's topology (26,719 verts / 26,384 tris), so a blendshape is +simply `expr_obj − neutral_obj` (per-vertex delta). ICT uses `_L/_R` +stems; the spike maps them to the canonical `FaceCap::kBlendshapeNames` (the +mocap-52 order) — full table in `scripts/spike-facerig.py::ICT_TO_ARKIT`. +Slice B bakes this template into a compact bundled form + hosts it. + +> **OBJ gotcha:** these OBJs are multi-group; trimesh loads them as a Scene and +> reorders/duplicates vertices, breaking the shared-topology assumption. Parse +> `v`/`f` manually and preserve order (the spike + Slice B loader both do). + +## The pipeline (what Slices C/D implement natively) + +``` +user neutral head (arbitrary topology, roughly humanoid, +Y up, facing +Z) + │ + 1. rigid pre-align: centroid + bbox-scale match template→user + │ (correspondence-free; NRICP refines. A real user mesh may need + │ up/forward-axis detection first — the spike's decimated head shared + │ ICT's orientation so identity axes sufficed.) + │ + 2. NRICP (Amberg 2007 optimal-step): + │ unknown = per-template-vertex 3×4 affine A_i + │ minimize ‖A_i·ṽ_i − closest_point_on_user_surface(X_i)‖² (data) + │ + α·‖(A_i − A_j)‖² over template edges (i,j) (stiffness) + │ stiffness annealed α = 50→20→8→3→1→0.5, ~3 inner iters each; + │ closest point via point-to-triangle projection; one sparse lsqr per axis. + │ → fitted template verts X (lie on the user surface) = CORRESPONDENCE. + │ + 3. deformation transfer (Sumner & Popović 2004), per ARKit expression: + │ template expression correspondence = X + (expr_tmpl − neutral_tmpl) + │ per-user-vertex delta = displacement of the nearest correspondence point + │ (the spike's simplified transfer; the full C++ form solves the + │ per-triangle deformation-gradient least-squares — see below). + │ + → 52 per-user-vertex deltas → Ogre::Pose morph targets named per kBlendshapeNames +``` + +### Note on the transfer step (Slice D must upgrade the spike form) + +The spike transfers by **nearest-correspondence-point displacement**, which is +enough to prove semantics and fit quality. The production Slice D should use the +**full deformation-gradient transfer**: build each template triangle's affine +`S_j = V_expr · V_neutral⁻¹` (with the 4th "normal" vertex trick), then solve one +sparse least-squares `min ‖A_userTri − S_j‖²` for the user vertex positions +(the Sumner-Popović matrix). This is more faithful for large/rotational +deformations (jaw) than nearest-point displacement. The linear solver is shared +with NRICP. + +## Measured quality (2026-07-14, `scripts/spike-facerig.py`) + +Template 26,719v → user 12,763v (60%-decimated, different topology): + +| ARKit shape | max displacement | verts moved | semantics check | +|---|---|---|---| +| NRICP surface fit | mean **0.003%** / max **0.59%** of diag | — | template lands on user surface | +| jawOpen | 9.0% | 6,771 | lower face ΔY −0.32 (drops), forehead \|Δ\| 0.002 (still) ✅ | +| mouthSmileLeft | 3.3% | 3,552 | localized to mouth ✅ | +| eyeBlinkLeft | 3.8% | 1,328 | localized to eye ✅ | +| browInnerUp | 1.3% | 2,181 | localized to brow ✅ | + +## Risks / limits (carry into the epic) + +1. **Humanoid-only.** Transfer from a human template only makes sense for + roughly human face meshes; a prop/creature yields garbage. Slice E gates on + this (NRICP fit-quality metric + a clear error), the AutoRig/Pinocchio + precedent. +2. **Orientation.** The spike's user head shared ICT's axes. A real arbitrary + mesh needs up/forward detection (or a user hint) before the rigid pre-align + — fold into Slice C/E. +3. **Correspondence quality drives shape quality.** Landmark constraints (eye + corners / nose / mouth) may be needed on faces far from the template + proportions; the spike didn't need them on the decimated ICT head — revisit + on real Ready-Player-Me / scanned heads in Slice C. +4. **Full deformation-gradient transfer** (Slice D) over the nearest-point + spike form, for faithful large deformations. +5. **Performance.** The python spike's dense per-vertex assembly is slow (~minutes + at 26k verts); the C++ port must assemble the sparse system directly and use a + real sparse solver (the project's existing linear-algebra path) — target a + few seconds, worker-threaded in the GUI. + +## Go/No-Go + +**GO.** Both algorithms proven on a real, different-topology head with a +permissive (MIT) template. Slices B→G are an engineering port of a working +prototype, not open research. Ship the full deformation-gradient transfer + +landmark option as the two quality upgrades over the spike. diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 5f2d66cf9..672d3d7e4 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -9381,6 +9381,293 @@ Rectangle { } } + // ── Auto-generate ARKit blendshapes (#895) ────────────────── + // One-click: fit the ARKit template onto the selected FACE mesh + // (NRICP + deformation transfer) and attach the 52 ARKit-named + // morph targets, so the #869 face-capture panel drives them. + // Heavy — runs on a worker thread via FaceRigController; the + // button shows progress and disables while busy. + Rectangle { + id: arkitBtn + width: parent.width + height: 24 + radius: 3 + property bool canRun: FaceRigController.hasMeshSelection + && !FaceRigController.busy + opacity: canRun ? 1.0 : 0.5 + color: arkitMa.containsMouse && canRun + ? Qt.lighter(PropertiesPanelController.highlightColor, 1.1) + : PropertiesPanelController.highlightColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: FaceRigController.busy + ? (FaceRigController.status !== "" + ? FaceRigController.status : "Working…") + : "✨ Add ARKit Blendshapes (AI)" + color: PropertiesPanelController.textColor + font.pixelSize: 10 + font.bold: true + } + MouseArea { + id: arkitMa + anchors.fill: parent + hoverEnabled: true + enabled: arkitBtn.canRun + cursorShape: enabled ? Qt.PointingHandCursor : Qt.ForbiddenCursor + onClicked: FaceRigController.addArkitBlendshapesAsync(0, 8.0) + ToolTip.visible: containsMouse + ToolTip.text: FaceRigController.hasMeshSelection + ? "Auto-fit the ARKit blendshape template onto this face " + + "mesh and attach the 52 ARKit shapes. Humanoid faces " + + "only; a poor fit is rejected." + : "Select a face mesh first." + } + } + + // ── Face markers (auto-seed, user adjusts) — the reliable path + // for cartoon/stylized faces MediaPipe can't detect. #889. + Rectangle { + id: markerBtn + width: parent.width + height: 22 + visible: !FaceRigController.markerMode + property bool canRun: FaceRigController.hasMeshSelection + && !FaceRigController.busy + opacity: canRun ? 1.0 : 0.5 + radius: 3 + color: markerMa.containsMouse && canRun + ? Qt.lighter(PropertiesPanelController.headerColor, 1.3) + : PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: "◎ Place / adjust face markers…" + color: PropertiesPanelController.textColor + font.pixelSize: 9 + } + MouseArea { + id: markerMa + anchors.fill: parent + hoverEnabled: true + enabled: markerBtn.canRun + cursorShape: enabled ? Qt.PointingHandCursor : Qt.ForbiddenCursor + onClicked: FaceRigController.beginFaceMarkers() + ToolTip.visible: containsMouse + ToolTip.text: "Auto-place face anchors (eyes, nose, mouth, " + + "chin) and drag any that are off, then rig from them. " + + "Use this for cartoon/stylized faces where auto-detect " + + "struggles." + } + } + + // Marker-editing panel — shown only during a marker session. + Column { + width: parent.width + visible: FaceRigController.markerMode + spacing: 4 + Text { + width: parent.width + wrapMode: Text.Wrap + font.pixelSize: 9 + color: PropertiesPanelController.textColor + text: (FaceRigController.markersSeededFromDetection + ? "Auto-detected. " : "Auto-detect was weak — ") + + "Click a marker below, then click on the face to move " + + "it. Cyan = selected. Left/Right = the CHARACTER's " + + "side (a mirrored placement is auto-corrected)." + } + // Marker chips — click to select which one the next mesh + // click will move. After placing, selection auto-advances + // to the NEXT chip in this order. + Flow { + width: parent.width + spacing: 3 + Repeater { + model: FaceRigController.markerLabels + Rectangle { + height: 18 + width: chipText.implicitWidth + 12 + radius: 3 + property bool sel: index === FaceRigController.selectedMarker + // NB: markerPlaced() is an invokable, not a + // property — reference selectedMarker in the + // binding so it re-evaluates on markersChanged + // (otherwise the chip state goes stale). + property bool placed: { + var _dep = FaceRigController.selectedMarker + return FaceRigController.markerPlaced(index) + } + color: sel ? "#2ae6ff" + : placed ? PropertiesPanelController.highlightColor + : PropertiesPanelController.controlBgColor + border.color: sel ? "#ffffff" + : PropertiesPanelController.borderColor + border.width: sel ? 2 : 1 + opacity: placed || sel ? 1.0 : 0.6 + Text { + id: chipText + anchors.centerIn: parent + text: modelData + font.pixelSize: 8 + font.bold: sel + color: sel ? "#003" : PropertiesPanelController.textColor + } + MouseArea { + anchors.fill: parent + cursorShape: Qt.PointingHandCursor + onClicked: FaceRigController.selectMarker(index) + } + } + } + } + // Strength (amplitude) — exaggeration multiplier for the + // transferred shapes; stylized faces often want >1. + RowLayout { + width: parent.width + spacing: 6 + Text { + text: "Strength" + font.pixelSize: 9 + color: PropertiesPanelController.textColor + } + Slider { + id: ampSlider + Layout.fillWidth: true + from: 0.5; to: 3.0; value: 1.5 + stepSize: 0.1 + } + Text { + text: "×" + ampSlider.value.toFixed(1) + font.pixelSize: 9 + color: PropertiesPanelController.textColor + } + } + RowLayout { + width: parent.width + spacing: 4 + Rectangle { + Layout.fillWidth: true + Layout.preferredHeight: 22 + radius: 3 + color: rigMkMa.containsMouse + ? Qt.lighter(PropertiesPanelController.highlightColor, 1.1) + : PropertiesPanelController.highlightColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: "✓ Rig from markers" + font.pixelSize: 9; font.bold: true + color: PropertiesPanelController.textColor + } + MouseArea { + id: rigMkMa + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + onClicked: FaceRigController.rigFromMarkers(0, 8.0, ampSlider.value) + } + } + Rectangle { + Layout.preferredWidth: 56 + Layout.preferredHeight: 22 + radius: 3 + color: cancelMkMa.containsMouse + ? Qt.lighter(PropertiesPanelController.headerColor, 1.3) + : PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: "Cancel"; font.pixelSize: 9 + color: PropertiesPanelController.textColor + } + MouseArea { + id: cancelMkMa + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + onClicked: FaceRigController.cancelFaceMarkers() + } + } + } + } + // Progress bar + Cancel, shown only while the worker runs. + RowLayout { + width: parent.width + visible: FaceRigController.busy + spacing: 6 + // Track + determinate fill (indeterminate look when total==0). + Rectangle { + Layout.fillWidth: true + Layout.preferredHeight: 6 + radius: 3 + color: PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + Rectangle { + height: parent.height + radius: 3 + color: PropertiesPanelController.highlightColor + width: FaceRigController.progressTotal > 0 + ? parent.width * FaceRigController.progress + / FaceRigController.progressTotal + : parent.width * 0.15 + } + } + Text { + visible: FaceRigController.progressTotal > 0 + text: FaceRigController.progress + "/" + FaceRigController.progressTotal + color: PropertiesPanelController.textColor + font.pixelSize: 9 + } + Rectangle { + Layout.preferredWidth: 48 + Layout.preferredHeight: 18 + radius: 3 + color: cancelMa.containsMouse + ? Qt.lighter(PropertiesPanelController.headerColor, 1.3) + : PropertiesPanelController.controlBgColor + border.color: PropertiesPanelController.borderColor + Text { + anchors.centerIn: parent + text: "Cancel" + color: PropertiesPanelController.textColor + font.pixelSize: 9 + } + MouseArea { + id: cancelMa + anchors.fill: parent + hoverEnabled: true + cursorShape: Qt.PointingHandCursor + onClicked: FaceRigController.cancel() + } + } + } + Connections { + target: FaceRigController + function onError(message) { + arkitStatus.text = "⚠ " + message + arkitStatus.color = "#d66" + } + function onCompleted(report) { + arkitStatus.text = "✓ Attached " + report.shapesAttached + + " ARKit shapes (fit " + + report.fitMeanResidualPct.toFixed(2) + "% mean, jawOpen amp " + + report.jawOpenDisp.toFixed(4) + ", max amp " + + report.maxShapeDisp.toFixed(4) + ")." + arkitStatus.color = PropertiesPanelController.textColor + morphCol.targets = MorphAnimationManager.morphTargetsForSelection() || [] + } + } + Text { + id: arkitStatus + width: parent.width + visible: text !== "" + wrapMode: Text.Wrap + font.pixelSize: 9 + color: PropertiesPanelController.textColor + text: "" + } + // ── Section 2: ANIMATION CLIPS ────────────────────────────── // Section header, shown only once shapes exist (clips animate // shapes, so they're meaningless without any). diff --git a/scripts/export-arkit-template.py b/scripts/export-arkit-template.py new file mode 100644 index 000000000..77f818c82 --- /dev/null +++ b/scripts/export-arkit-template.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Pack the ICT-FaceKit ARKit template into QtMeshEditor's face-rig bundle (#890). + +ONE-TIME, OFFLINE developer tool — NOT shipped. Produces the compact binary +`arkit_template.bin` that `src/FaceRig/ArkitTemplate.cpp` reads and the app +downloads on first use to AppData/ai_models/facerig/. The face-rig feature +(#889) uses it as the deformation-transfer SOURCE. + +INPUT + A directory of ICT-FaceKit FaceXModel .obj files (MIT, USC-ICT): + generic_neutral_mesh.obj + the per-expression meshes (same topology, so a + blendshape = expr - neutral). Download from + https://github.com/USC-ICT/ICT-FaceKit/tree/master/FaceXModel + +OUTPUT arkit_template.bin — little-endian: + magic "QMFRT1\0\0" (8 bytes) + int32 vertexCount V + int32 faceCount F + int32 shapeCount S (== 51 — ICT has no tongueOut) + float32 neutral[V*3] (template neutral positions) + int32 faces[F*3] (triangle vertex indices) + then S shape records: + char[32] name (ASCII, NUL-padded — a FaceCap::kBlendshapeNames entry) + float32 delta[V*3] (expr - neutral; most verts are ~0) + ("_neutral" is NOT stored as a shape — it is the base.) + +The 52 ARKit names + the ICT->ARKit mapping are baked here; some ARKit +channels are SINGLE/centered (browInnerUp, cheekPuff, mouthClose, mouthFunnel, +mouthPucker, jaw*, mouthRoll*/Shrug*) while ICT splits a few as _L/_R — for +those the ARKit delta is the SUM of the ICT halves. The rest map 1:1. + +USAGE + python scripts/export-arkit-template.py --ict-dir .facerig_work/ict_full \ + --out .facerig_work/out/arkit_template.bin +""" + +import argparse +import os +import struct +import sys + +import numpy as np + +# 52 ARKit names in FaceCap::kBlendshapeNames order (index 0 "_neutral" is the +# base, not a shape). Each maps to a list of ICT stems whose deltas SUM to it. +ARKIT = [ + ("browDownLeft", ["browDown_L"]), ("browDownRight", ["browDown_R"]), + ("browInnerUp", ["browInnerUp_L", "browInnerUp_R"]), + ("browOuterUpLeft", ["browOuterUp_L"]), ("browOuterUpRight", ["browOuterUp_R"]), + ("cheekPuff", ["cheekPuff_L", "cheekPuff_R"]), + ("cheekSquintLeft", ["cheekSquint_L"]), ("cheekSquintRight", ["cheekSquint_R"]), + ("eyeBlinkLeft", ["eyeBlink_L"]), ("eyeBlinkRight", ["eyeBlink_R"]), + ("eyeLookDownLeft", ["eyeLookDown_L"]), ("eyeLookDownRight", ["eyeLookDown_R"]), + ("eyeLookInLeft", ["eyeLookIn_L"]), ("eyeLookInRight", ["eyeLookIn_R"]), + ("eyeLookOutLeft", ["eyeLookOut_L"]), ("eyeLookOutRight", ["eyeLookOut_R"]), + ("eyeLookUpLeft", ["eyeLookUp_L"]), ("eyeLookUpRight", ["eyeLookUp_R"]), + ("eyeSquintLeft", ["eyeSquint_L"]), ("eyeSquintRight", ["eyeSquint_R"]), + ("eyeWideLeft", ["eyeWide_L"]), ("eyeWideRight", ["eyeWide_R"]), + ("jawForward", ["jawForward"]), ("jawLeft", ["jawLeft"]), + ("jawOpen", ["jawOpen"]), ("jawRight", ["jawRight"]), + ("mouthClose", ["mouthClose"]), + ("mouthDimpleLeft", ["mouthDimple_L"]), ("mouthDimpleRight", ["mouthDimple_R"]), + ("mouthFrownLeft", ["mouthFrown_L"]), ("mouthFrownRight", ["mouthFrown_R"]), + ("mouthFunnel", ["mouthFunnel"]), ("mouthLeft", ["mouthLeft"]), + ("mouthLowerDownLeft", ["mouthLowerDown_L"]), + ("mouthLowerDownRight", ["mouthLowerDown_R"]), + ("mouthPressLeft", ["mouthPress_L"]), ("mouthPressRight", ["mouthPress_R"]), + ("mouthPucker", ["mouthPucker"]), ("mouthRight", ["mouthRight"]), + ("mouthRollLower", ["mouthRollLower"]), ("mouthRollUpper", ["mouthRollUpper"]), + ("mouthShrugLower", ["mouthShrugLower"]), ("mouthShrugUpper", ["mouthShrugUpper"]), + ("mouthSmileLeft", ["mouthSmile_L"]), ("mouthSmileRight", ["mouthSmile_R"]), + ("mouthStretchLeft", ["mouthStretch_L"]), ("mouthStretchRight", ["mouthStretch_R"]), + ("mouthUpperUpLeft", ["mouthUpperUp_L"]), ("mouthUpperUpRight", ["mouthUpperUp_R"]), + ("noseSneerLeft", ["noseSneer_L"]), ("noseSneerRight", ["noseSneer_R"]), + # NOTE: tongueOut (ARKit channel 52) is deliberately ABSENT — ICT-FaceKit + # ships no tongue expression (the tongue isn't part of its face topology), + # so the packed template carries 51 real shapes. Emitting a zero-delta + # tongueOut would only inflate the bundle and show a dead slider. +] + +MAGIC = b"QMFRT1\0\0" + + +def load_obj(path): + V, F = [], [] + with open(path) as f: + for ln in f: + if ln.startswith("v "): + V.append([float(x) for x in ln.split()[1:4]]) + elif ln.startswith("f "): + # Fan-triangulate: ICT-FaceKit meshes are QUADS. Taking only + # the first 3 indices dropped every quad's second triangle, + # shipping a template with half its faces missing (holed + # renders, fragmented connectivity). + idx = [int(t.split("/")[0]) - 1 for t in ln.split()[1:]] + for k in range(1, len(idx) - 1): + F.append([idx[0], idx[k], idx[k + 1]]) + return np.asarray(V, np.float64), np.asarray(F, np.int32) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--ict-dir", required=True) + ap.add_argument("--out", default=".facerig_work/out/arkit_template.bin") + args = ap.parse_args() + + neutral, faces = load_obj(os.path.join(args.ict_dir, "generic_neutral_mesh.obj")) + V = len(neutral) + print(f"neutral: {V} verts / {len(faces)} tris") + + shapes = [] + for arkit_name, ict_stems in ARKIT: + delta = np.zeros((V, 3), np.float64) + missing = [] + for stem in ict_stems: + p = os.path.join(args.ict_dir, stem + ".obj") + if not os.path.exists(p): + missing.append(stem) + continue + ev, ef = load_obj(p) + if len(ev) != V: + sys.exit(f"topology mismatch: {stem} has {len(ev)} verts, " + f"neutral {V}") + if not np.array_equal(ef, faces): + sys.exit(f"topology mismatch: {stem} face list differs from " + f"the neutral mesh (same vert count is not enough)") + delta += ev - neutral + if missing: + # A missing expression would silently ship as a dead zero-delta + # shape — fail so a corrupt bundle can't be packaged/uploaded. + sys.exit(f"ABORT: {arkit_name}: missing ICT {missing}") + # count verts moved by a MEANINGFUL amount (0.1% of the head + # diagonal), not floating-point dust, so the log reflects real motion. + _diag = float(np.linalg.norm(neutral.max(0) - neutral.min(0))) + moved = int((np.linalg.norm(delta, axis=1) > 1e-3 * _diag).sum()) + shapes.append((arkit_name, delta.astype(np.float32))) + print(f" {arkit_name:22s} <- {'+'.join(ict_stems):24s} moved {moved} verts") + + os.makedirs(os.path.dirname(args.out), exist_ok=True) + with open(args.out, "wb") as f: + f.write(MAGIC) + f.write(struct.pack(" {args.out} ({size/1e6:.1f} MB)") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/spike-facerig.py b/scripts/spike-facerig.py new file mode 100644 index 000000000..84e2ad683 --- /dev/null +++ b/scripts/spike-facerig.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Face-rig spike (#896): prove auto-ARKit-blendshape generation is viable. + +ONE-TIME, OFFLINE feasibility prototype — NOT shipped, NOT wired to CMake/CI. +It validates the two algorithms Slices C/D will implement natively in C++: + + 1. Non-rigid ICP (NRICP, Amberg et al. 2007 optimal-step) to fit the + ICT-FaceKit generic-neutral template to an arbitrary USER neutral head, + producing a per-template-vertex correspondence on the user surface. + 2. Deformation transfer (Sumner & Popovic 2004) of each of ICT's 52 + ARKit-named expression shapes onto the USER mesh topology. + +Output: for each expression, a new blendshape (per-user-vertex delta) that, +added to the user neutral, reproduces that expression on the user's identity. + +DATA + LICENSE + Template = ICT-FaceKit (USC-ICT), MIT — generic_neutral_mesh.obj + the + per-expression *.obj (same topology; shape = expr - neutral). See + THIRD_PARTY_AI_MODELS.md. + +CONTRACT this measures (for docs/FACE_RIG_SPIKE.md / Slices C-D): + - template: N_t verts, F_t tris; 52 ARKit shape names (ICT naming -> + FaceCap::kBlendshapeNames mapping printed below). + - NRICP: point-to-point + stiffness-annealed regularization on the template + edge graph; returns fitted template vertex positions lying on the user + surface (= correspondence). + - deformation transfer: per-triangle affine from (neutral tri -> expr tri) + on the template, retargeted to the user tris via the correspondence, + solved as one sparse least-squares for user vertex positions. + +USAGE (offline, venv with numpy scipy trimesh): + python scripts/spike-facerig.py \ + --template-dir .facerig_work/ict \ + --user .facerig_work/user_neutral.obj \ + --out-dir .facerig_work/out [--shapes jawOpen,mouthSmile_L,...] +""" + +import argparse +import glob +import os +import sys + +import numpy as np +import trimesh +from scipy.sparse import coo_matrix, csr_matrix, vstack +from scipy.sparse.linalg import lsqr +from scipy.spatial import cKDTree + + +# ICT expression-file stem -> ARKit-52 canonical name (FaceCap order). ICT +# splits L/R and uses slightly different stems; this is the mapping Slice B +# bakes in. (Subset shown covers the spike; full 52 in the doc.) +ICT_TO_ARKIT = { + "browDown_L": "browDownLeft", "browDown_R": "browDownRight", + "browInnerUp_L": "browInnerUp", "browInnerUp_R": "browInnerUp", + "browOuterUp_L": "browOuterUpLeft", "browOuterUp_R": "browOuterUpRight", + "cheekPuff_L": "cheekPuff", "cheekPuff_R": "cheekPuff", + "cheekSquint_L": "cheekSquintLeft", "cheekSquint_R": "cheekSquintRight", + "eyeBlink_L": "eyeBlinkLeft", "eyeBlink_R": "eyeBlinkRight", + "eyeLookDown_L": "eyeLookDownLeft", "eyeLookDown_R": "eyeLookDownRight", + "eyeLookIn_L": "eyeLookInLeft", "eyeLookIn_R": "eyeLookInRight", + "eyeLookOut_L": "eyeLookOutLeft", "eyeLookOut_R": "eyeLookOutRight", + "eyeLookUp_L": "eyeLookUpLeft", "eyeLookUp_R": "eyeLookUpRight", + "eyeSquint_L": "eyeSquintLeft", "eyeSquint_R": "eyeSquintRight", + "eyeWide_L": "eyeWideLeft", "eyeWide_R": "eyeWideRight", + "jawForward": "jawForward", "jawLeft": "jawLeft", "jawRight": "jawRight", + "jawOpen": "jawOpen", + "mouthClose": "mouthClose", + "mouthSmile_L": "mouthSmileLeft", "mouthSmile_R": "mouthSmileRight", + "mouthFrown_L": "mouthFrownLeft", "mouthFrown_R": "mouthFrownRight", +} + + +def load_obj(path): + # Manual v/f parse — trimesh loads these OBJs as multi-group Scenes and + # reorders/duplicates verts, which breaks the shared-topology assumption + # (shape delta = expr - neutral requires identical vertex order). + V, F = [], [] + with open(path) as f: + for ln in f: + if ln.startswith("v "): + V.append([float(x) for x in ln.split()[1:4]]) + elif ln.startswith("f "): + # Fan-triangulate: ICT-FaceKit meshes are QUADS (same fix as + # export-arkit-template.py — first-3-indices dropped half the + # triangles). + idx = [int(t.split("/")[0]) - 1 for t in ln.split()[1:]] + for k in range(1, len(idx) - 1): + F.append([idx[0], idx[k], idx[k + 1]]) + return np.asarray(V, dtype=np.float64), np.asarray(F, dtype=np.int64) + + +# --------------------------------------------------------------------------- +# rigid pre-align (Umeyama with scale) — template onto user +# --------------------------------------------------------------------------- + +def umeyama(src, dst): + mu_s, mu_d = src.mean(0), dst.mean(0) + S, D = src - mu_s, dst - mu_d + cov = D.T @ S / len(src) + U, d, Vt = np.linalg.svd(cov) + R = U @ Vt + if np.linalg.det(R) < 0: + U[:, -1] *= -1 + R = U @ Vt + var = (S ** 2).sum() / len(src) + s = d.sum() / var + t = mu_d - s * R @ mu_s + return s, R, t + + +# --------------------------------------------------------------------------- +# NRICP (Amberg 2007 optimal-step, point-to-point) +# --------------------------------------------------------------------------- + +def edge_incidence(faces, n): + edges = set() + for a, b, c in faces: + for i, j in ((a, b), (b, c), (c, a)): + edges.add((min(i, j), max(i, j))) + edges = list(edges) + rows, cols, vals = [], [], [] + for k, (i, j) in enumerate(edges): + rows += [k, k]; cols += [i, j]; vals += [-1.0, 1.0] + M = coo_matrix((vals, (rows, cols)), shape=(len(edges), n)) + return M.tocsr() + + +def nricp(tmpl_v, tmpl_f, user_v, user_f, + stiffness=(50, 20, 8, 3, 1, 0.5), iters_per=3): + """Fit template verts onto the user surface. Returns fitted positions X + (template-indexed, lying near the user surface) = the correspondence.""" + n = len(tmpl_v) + # correspondence-free rigid pre-align: match centroid + bbox scale (the + # point sets differ in count, so a Procrustes needs correspondence we + # don't have yet; NRICP refines from this coarse start). Axes already + # agree (both +Y up, facing +Z) since ICT and the decimated user share + # the source orientation; a real user mesh may need axis detection first. + s = np.linalg.norm(user_v.max(0) - user_v.min(0)) / \ + max(np.linalg.norm(tmpl_v.max(0) - tmpl_v.min(0)), 1e-9) + R = np.eye(3) + t = user_v.mean(0) - s * tmpl_v.mean(0) + X = (s * (tmpl_v @ R.T)) + t + user_tri = user_v[user_f] + user_centroids = user_tri.mean(1) + tree = cKDTree(user_centroids) + + M = edge_incidence(tmpl_f, n) # (E, n) node-arc incidence + G = np.array([1, 1, 1, 1.0]) # per-vertex 3x4 affine weight + kron = None # built lazily + + # unknown: per-vertex 3x4 affine A_i; template homogeneous coords + Th = np.hstack([tmpl_v, np.ones((n, 1))]) # (n,4) + + for alpha in stiffness: + for _ in range(iters_per): + # data term: A_i * th_i ~= closest user point to current X_i + _, tri_idx = tree.query(X) + # closest point = project X onto that triangle (approx: centroid-nudged + # to nearest vertex of the tri for a cheap point-to-point target) + target = closest_on_tris(X, user_v, user_f, tri_idx) + + # Build sparse system: [ alpha * (M kron G) ; D ] A = [ 0 ; target ] + # A is stacked as (4n x 3). D picks th_i per row. + D = csr_matrix((np.repeat(1.0, n), + (np.arange(n), np.arange(n))), shape=(n, n)) + # data rows: for each vertex, th_i (1x4) times A_i (4x3) -> point + data_rows = build_data(Th) # (n, 4n) + stiff = build_stiffness(M, alpha) # (4E, 4n) + Amat = vstack([stiff, data_rows]).tocsr() + rhs = np.vstack([np.zeros((stiff.shape[0], 3)), target]) + Asol = np.zeros((4 * n, 3)) + for axis in range(3): + Asol[:, axis] = lsqr(Amat, rhs[:, axis], atol=1e-6, btol=1e-6, + iter_lim=400)[0] + X = apply_affine(Th, Asol) + return X + + +def build_data(Th): + n = len(Th) + rows, cols, vals = [], [], [] + for i in range(n): + for k in range(4): + rows.append(i); cols.append(4 * i + k); vals.append(Th[i, k]) + return coo_matrix((vals, (rows, cols)), shape=(n, 4 * n)).tocsr() + + +def build_stiffness(M, alpha): + # (M kron I4) * alpha, gamma weighting on the translation column left at 1 + E, n = M.shape + Mc = M.tocoo() + rows, cols, vals = [], [], [] + for r, c, v in zip(Mc.row, Mc.col, Mc.data): + for k in range(4): + rows.append(4 * r + k); cols.append(4 * c + k); vals.append(alpha * v) + return coo_matrix((vals, (rows, cols)), shape=(4 * E, 4 * n)).tocsr() + + +def apply_affine(Th, Asol): + n = len(Th) + X = np.zeros((n, 3)) + for i in range(n): + Ai = Asol[4 * i:4 * i + 4, :] # (4,3) + X[i] = Th[i] @ Ai + return X + + +def closest_on_tris(P, V, F, tri_idx): + """Point-to-triangle projection of each P[i] onto tri F[tri_idx[i]].""" + out = np.empty_like(P) + for i in range(len(P)): + a, b, c = V[F[tri_idx[i]]] + out[i] = closest_point_triangle(P[i], a, b, c) + return out + + +def closest_point_triangle(p, a, b, c): + ab, ac, ap = b - a, c - a, p - a + d1, d2 = ab @ ap, ac @ ap + if d1 <= 0 and d2 <= 0: + return a + bp = p - b + d3, d4 = ab @ bp, ac @ bp + if d3 >= 0 and d4 <= d3: + return b + vc = d1 * d4 - d3 * d2 + if vc <= 0 and d1 >= 0 and d3 <= 0: + return a + (d1 / (d1 - d3)) * ab + cp = p - c + d5, d6 = ab @ cp, ac @ cp + if d6 >= 0 and d5 <= d6: + return c + vb = d5 * d2 - d1 * d6 + if vb <= 0 and d2 >= 0 and d6 <= 0: + return a + (d2 / (d2 - d6)) * ac + va = d3 * d6 - d5 * d4 + if va <= 0 and (d4 - d3) >= 0 and (d5 - d6) >= 0: + return b + ((d4 - d3) / ((d4 - d3) + (d5 - d6))) * (c - b) + denom = 1.0 / (va + vb + vc) + v, w = vb * denom, vc * denom + return a + ab * v + ac * w + + +# --------------------------------------------------------------------------- +# deformation transfer (Sumner & Popovic 2004), correspondence-based +# --------------------------------------------------------------------------- + +def tri_frame(v0, v1, v2): + """3x3 frame [e1 e2 n] for a triangle (n = normalized cross / sqrt).""" + e1 = v1 - v0 + e2 = v2 - v0 + n = np.cross(e1, e2) + ln = np.linalg.norm(n) + n = n / np.sqrt(ln) if ln > 1e-12 else n + return np.column_stack([e1, e2, n]) + + +def deformation_transfer(user_v, user_f, src_neutral_corr, src_expr_corr): + """Transfer the src (template, expressed via the correspondence) per-tri + deformation onto the user mesh. src_*_corr are template-vertex positions + (neutral / expression) already living on the user identity via NRICP; we + solve for user vertex positions whose per-tri deformation matches. + + Simplified spike form: since the correspondence maps template verts onto + the user surface 1:1, the transferred user delta is the correspondence + delta resampled to user verts via nearest correspondence point.""" + # correspondence points move by (expr - neutral); map that displacement to + # each user vertex by nearest correspondence point (KD-tree). + disp = src_expr_corr - src_neutral_corr + tree = cKDTree(src_neutral_corr) + _, idx = tree.query(user_v) + return disp[idx] # per-user-vertex delta + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--template-dir", default=".facerig_work/ict") + ap.add_argument("--user", required=True, help="user neutral head .obj") + ap.add_argument("--out-dir", default=".facerig_work/out") + ap.add_argument("--shapes", default="", + help="comma ICT stems (default: all *.obj present)") + ap.add_argument("--gt-dir", default="", + help="optional: user-identity ground-truth expr .obj dir " + "(same names as template) for RMS quality") + args = ap.parse_args() + + os.makedirs(args.out_dir, exist_ok=True) + tv, tf = load_obj(os.path.join(args.template_dir, "generic_neutral_mesh.obj")) + uv, uf = load_obj(args.user) + print(f"template: {len(tv)} verts / {len(tf)} tris") + print(f"user: {len(uv)} verts / {len(uf)} tris") + + print("running NRICP (template -> user neutral)...") + corr = nricp(tv, tf, uv, uf) + fit_err = np.linalg.norm( + corr - closest_on_tris(corr, uv, uf, cKDTree(uv[uf].mean(1)).query(corr)[1]), + axis=1) + diag = np.linalg.norm(uv.max(0) - uv.min(0)) + print(f" NRICP surface fit: mean {fit_err.mean()/diag*100:.3f}% of diag, " + f"max {fit_err.max()/diag*100:.3f}%") + + if args.shapes: + stems = args.shapes.split(",") + else: + stems = [os.path.splitext(os.path.basename(p))[0] + for p in glob.glob(os.path.join(args.template_dir, "*.obj")) + if "neutral" not in p] + + report = {"template_verts": len(tv), "user_verts": len(uv), + "nricp_mean_pct": float(fit_err.mean() / diag * 100), "shapes": {}} + for stem in stems: + ep = os.path.join(args.template_dir, stem + ".obj") + if not os.path.exists(ep): + print(f" skip {stem} (missing)"); continue + ev, _ = load_obj(ep) + # express the template expression through the SAME correspondence: + # correspondence of the expression = corr + (expr - neutral) template delta + expr_corr = corr + (ev - tv) + user_delta = deformation_transfer(uv, uf, corr, expr_corr) + arkit = ICT_TO_ARKIT.get(stem, stem) + mag = np.linalg.norm(user_delta, axis=1) + entry = {"arkit": arkit, "max_disp_pct": float(mag.max() / diag * 100), + "moved_verts": int((mag > 1e-4 * diag).sum())} + # optional GT RMS + gt = os.path.join(args.gt_dir, stem + ".obj") if args.gt_dir else "" + if gt and os.path.exists(gt): + gtv, _ = load_obj(gt) + rms = np.sqrt(((uv + user_delta - gtv) ** 2).sum(1).mean()) + entry["rms_pct"] = float(rms / diag * 100) + report["shapes"][stem] = entry + # write the blendshape target (neutral + delta) for visual inspection + trimesh.Trimesh(uv + user_delta, uf, process=False).export( + os.path.join(args.out_dir, f"user_{arkit}.obj")) + print(f" {stem:16s} -> {arkit:18s} maxDisp {entry['max_disp_pct']:.2f}% " + f"moved {entry['moved_verts']} verts" + + (f" RMS {entry['rms_pct']:.3f}%" if "rms_pct" in entry else "")) + + import json + with open(os.path.join(args.out_dir, "facerig_spike_report.json"), "w") as f: + json.dump(report, f, indent=2) + print(f"\nreport -> {os.path.join(args.out_dir, 'facerig_spike_report.json')}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/upload-facerig-template.sh b/scripts/upload-facerig-template.sh new file mode 100755 index 000000000..42cccbaf9 --- /dev/null +++ b/scripts/upload-facerig-template.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Upload the ARKit face-rig template to the QtMeshEditor HF models repo +# (epic #889, slice #890). ONE-TIME, run by a maintainer with write access. +# +# The app downloads this on first use from +# https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/facerig/arkit_template.bin +# so the file name + subdir MUST match ArkitTemplate::modelPath() / +# kDefaultModelBaseUrl. +# +# The template is derived from ICT-FaceKit (USC-ICT), MIT — ship the ICT MIT +# LICENSE next to it (see THIRD_PARTY_AI_MODELS.md). +# +# Prereqs: +# pip install -U huggingface_hub +# hf auth login # a token with write access to the repo +# scripts/export-arkit-template.py already run -> OUT holds arkit_template.bin +# +# Usage: +# OUT=.facerig_work/out ICT_LICENSE=.facerig_work/ICT_LICENSE.txt \ +# ./scripts/upload-facerig-template.sh +set -euo pipefail + +REPO="${REPO:-fernandotonon/QtMeshEditor-models}" +OUT="${OUT:?set OUT to the dir holding arkit_template.bin}" +ICT_LICENSE="${ICT_LICENSE:-}" + +upload() { # + local src="$1" dst="$2" + if [ -f "$src" ]; then + echo ">> uploading $src -> $REPO:$dst" + hf upload "$REPO" "$src" "$dst" + else + echo "!! skip (missing): $src" + fi +} + +# Required artifacts — refuse to publish an incomplete release. Only the +# landmark model below is optional (hosted separately by the mocap epic). +[ -f "$OUT/arkit_template.bin" ] || { echo "ABORT: missing $OUT/arkit_template.bin"; exit 1; } +[ -n "$ICT_LICENSE" ] && [ -f "$ICT_LICENSE" ] || { echo "ABORT: ICT_LICENSE not set or missing (the MIT license must ship next to the template)"; exit 1; } +upload "$OUT/arkit_template.bin" "facerig/arkit_template.bin" +upload "$ICT_LICENSE" "facerig/ICT-FaceKit-LICENSE.txt" + +# Facial-landmark model (MediaPipe FaceMesh V2, Apache-2.0) that anchors the +# NRICP fit to real face features (#889 landmark pass). It is the SAME graph the +# mocap face-capture uses (mocap/face/face_landmarks.onnx); we host a copy under +# facerig/ so the face-rig feature is self-contained. Point FACE_LMK at the +# converted onnx (e.g. .mocap_work/out/face/face_landmarks.onnx). +FACE_LMK="${FACE_LMK:-}" +[ -n "$FACE_LMK" ] && upload "$FACE_LMK" "facerig/face_landmarks.onnx" + +echo "done. Verify: curl -sI https://huggingface.co/$REPO/resolve/main/facerig/arkit_template.bin | head -1" diff --git a/src/AppLaunchHandler.cpp b/src/AppLaunchHandler.cpp index 670deeef0..3293263ba 100644 --- a/src/AppLaunchHandler.cpp +++ b/src/AppLaunchHandler.cpp @@ -27,7 +27,8 @@ bool isCliSubcommand(const QString& arg) QStringLiteral("optimize"), QStringLiteral("bake-vertex-colors"), QStringLiteral("vat"), QStringLiteral("uv"), QStringLiteral("hdri"), QStringLiteral("light"), QStringLiteral("retopo"), - QStringLiteral("skin"), QStringLiteral("rig"), QStringLiteral("segment"), + QStringLiteral("skin"), QStringLiteral("rig"), QStringLiteral("facerig"), + QStringLiteral("segment"), QStringLiteral("generate3d"), QStringLiteral("morph"), QStringLiteral("nodeanim"), QStringLiteral("ps1"), QStringLiteral("cloud"), diff --git a/src/Assimp/Importer.cpp b/src/Assimp/Importer.cpp index 84768f68f..af6e9eb4f 100644 --- a/src/Assimp/Importer.cpp +++ b/src/Assimp/Importer.cpp @@ -37,6 +37,10 @@ THE SOFTWARE. #include #include +#include +#include +#include +#include namespace { @@ -235,6 +239,26 @@ Ogre::MeshPtr AssimpToOgreImporter::loadModel(const std::string& path, bool conv // Process the root node recursively (meshes) MeshProcessor meshProcessor(skeleton, isZup); + + // ARKit blendshape name sidecar (`.arkit.json`, schema + // qtmesh-arkit-blendshapes-v1): Assimp's glTF2 exporter drops + // `targetNames`, so shapes in a re-imported glb arrive nameless and would + // degrade to "Shape_N". Restore the authored ARKit names from the sidecar + // the face-rig exporters write next to the mesh. + { + QFile sidecar(QString::fromStdString(path) + ".arkit.json"); + if (sidecar.exists() && sidecar.open(QIODevice::ReadOnly)) { + const QJsonObject root = QJsonDocument::fromJson(sidecar.readAll()).object(); + if (root.value("schema").toString().startsWith("qtmesh-arkit-blendshapes")) { + std::vector names; + for (const auto& v : root.value("names").toArray()) + names.push_back(v.toString().toStdString()); + if (!names.empty()) + meshProcessor.setMorphNameHints(std::move(names)); + } + } + } + meshProcessor.processNode(scene->mRootNode, scene); Ogre::MeshPtr ogreMesh = meshProcessor.createMesh(modelName, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME, materialProcessor); diff --git a/src/Assimp/MeshProcessor.cpp b/src/Assimp/MeshProcessor.cpp index a3427750c..f38d705f0 100644 --- a/src/Assimp/MeshProcessor.cpp +++ b/src/Assimp/MeshProcessor.cpp @@ -135,12 +135,27 @@ SubMeshData* MeshProcessor::processMesh(aiMesh* mesh, const aiScene* scene) { // time, where we already have the base positions in hand for the // Ogre::Pose constructor. Apply the same Z-up axis bake the base // vertex pass uses so the shape and base agree on coordinate frame. + // Sidecar name hints only apply when the scene has exactly ONE morphed + // mesh — the unambiguous case (a flat ordered name list can't be split + // across submeshes safely). + bool useNameHints = false; + if (!m_nameHints.empty()) { + unsigned morphedMeshes = 0; + for (auto mi = 0u; mi < scene->mNumMeshes; mi++) + if (scene->mMeshes[mi] && scene->mMeshes[mi]->mNumAnimMeshes > 0) + morphedMeshes++; + useNameHints = (morphedMeshes == 1); + } for(auto am = 0u; am < mesh->mNumAnimMeshes; am++) { const aiAnimMesh* anim = mesh->mAnimMeshes[am]; if (!anim || !anim->mVertices || anim->mNumVertices != mesh->mNumVertices) continue; MorphTargetData target; - target.name = anim->mName.length > 0 ? anim->mName.C_Str() - : (std::string("Shape_") + std::to_string(am)); + if (anim->mName.length > 0) + target.name = anim->mName.C_Str(); + else if (useNameHints && am < m_nameHints.size() && !m_nameHints[am].empty()) + target.name = m_nameHints[am]; + else + target.name = std::string("Shape_") + std::to_string(am); target.positions.reserve(anim->mNumVertices); for (auto i = 0u; i < anim->mNumVertices; i++) { Ogre::Vector3 v(anim->mVertices[i].x, anim->mVertices[i].y, anim->mVertices[i].z); diff --git a/src/Assimp/MeshProcessor.h b/src/Assimp/MeshProcessor.h index ef4c112d6..a2504804c 100644 --- a/src/Assimp/MeshProcessor.h +++ b/src/Assimp/MeshProcessor.h @@ -33,6 +33,14 @@ class MeshProcessor { void processNode(aiNode* node, const aiScene* scene); Ogre::MeshPtr createMesh(const Ogre::String& name, const Ogre::String& group, MaterialProcessor &materialProcessor); + // Morph-target name hints from a `.arkit.json` sidecar (ordered + // names). Assimp's glTF2 EXPORTER drops `targetNames`, so a re-imported + // rigged glb otherwise degrades to generated "Shape_N" names — losing the + // ARKit vocabulary face capture matches on. Applied only when the aiMesh + // itself carries no names and the scene has a single morphed mesh (the + // unambiguous case — e.g. the ARKit reference head). + void setMorphNameHints(std::vector names) { m_nameHints = std::move(names); } + protected: // Protected for testing purposes SubMeshData* processMesh(aiMesh* mesh, const aiScene* scene); @@ -42,4 +50,5 @@ class MeshProcessor { std::vector boneAssignments; Ogre::SkeletonPtr skeleton; bool m_isZup; + std::vector m_nameHints; }; diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 550866f06..8536b03db 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -42,6 +42,9 @@ #include "SkinWeights.h" #include "SkinEvaluate.h" #include "AutoRig.h" +#include "FaceRig/ArkitTemplate.h" +#include "FaceRig/FaceRigAttach.h" +#include "FaceRig/FaceRigLandmarks.h" #include "ImageTo3D/MeshGenPredictor.h" #include "ImageTo3D/TripoSGPredictor.h" #include "ImageTo3D/MeshGenBuilder.h" @@ -861,6 +864,13 @@ void CLIPipeline::printUsage() " Per-vertex weight diff vs a reference-skinned copy of the same asset\n" " (e.g. Mixamo) — vertices matched by position, bones by name. See\n" " docs/SKINNING_QUALITY.md for the comparison protocol.\n" + " facerig -o [--max-shapes N] [--max-residual PCT] [--json]\n" + " Auto-generate the 52 ARKit blendshapes on a humanoid\n" + " FACE mesh: fit the ARKit template (non-rigid ICP) and\n" + " transfer each expression (Sumner-Popovic) onto the\n" + " mesh, attaching them as named morph targets. A poor\n" + " fit (non-face mesh) is rejected. The bundled template\n" + " downloads on first use. Feeds `qtmesh mocap --face`.\n" " morph --list [--json] List morph targets / blend shapes on a mesh. (Set/add/delete\n" " land in follow-up slices once authoring is in place.)\n" " nodeanim --list [--json] List node-animation clips on a scene (props, doors, machinery,\n" @@ -1599,6 +1609,7 @@ int CLIPipeline::run(int argc, char* argv[]) else if (cmd == "retopo") rc = cmdRetopo(argc, argv); else if (cmd == "skin") rc = cmdSkin(argc, argv); else if (cmd == "rig") rc = cmdRig(argc, argv); + else if (cmd == "facerig") rc = cmdFaceRig(argc, argv); else if (cmd == "segment") rc = cmdSegment(argc, argv); else if (cmd == "generate3d") rc = cmdGenerate3d(argc, argv); else if (cmd == "morph") rc = cmdMorph(argc, argv); @@ -1625,6 +1636,7 @@ int CLIPipeline::run(int argc, char* argv[]) {QStringLiteral("uv"), QStringLiteral("uv_unwrap")}, {QStringLiteral("skin"), QStringLiteral("skin_weights")}, {QStringLiteral("rig"), QStringLiteral("auto_rig")}, + {QStringLiteral("facerig"), QStringLiteral("auto_rig")}, {QStringLiteral("anim"), QStringLiteral("animation_blend")}, {QStringLiteral("morph"), QStringLiteral("morph")}, {QStringLiteral("vat"), QStringLiteral("vat_bake")}, @@ -9390,8 +9402,10 @@ int CLIPipeline::cmdRig(int argc, char* argv[]) SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.auto_rig"), QString("rig .%1 template=%2 algo=%3 skin=%4") .arg(fi.suffix(), templateName, algoName).arg(alsoSkin)); + // Extension only — absolute paths leak usernames/directory structure + // into telemetry. SentryReporter::addBreadcrumb(QStringLiteral("file.import"), - QString("Importing %1").arg(fi.absoluteFilePath())); + QString("Importing .%1 for facerig").arg(fi.suffix())); MeshImporterExporter::importer({fi.absoluteFilePath()}); QList meshEntities; @@ -9464,6 +9478,252 @@ int CLIPipeline::cmdRig(int argc, char* argv[]) return 0; } +int CLIPipeline::cmdFaceRig(int argc, char* argv[]) +{ + // Parse: facerig [-o out] [--max-shapes N] [--max-residual PCT] + // [--json] + QString inputPath, outputPath; + bool jsonOutput = false; + FaceRig::FaceRigOptions opts; + + for (int i = 1; i < argc; ++i) { + const QString arg = QString::fromLocal8Bit(argv[i]); + if (arg == "facerig" || arg == "--cli") continue; + if (arg == "--json") { jsonOutput = true; continue; } + if ((arg == "-o" || arg == "--output") && i + 1 < argc) { + outputPath = QString::fromLocal8Bit(argv[++i]); continue; + } + if (arg == "--max-shapes" && i + 1 < argc) { + opts.maxShapes = QString::fromLocal8Bit(argv[++i]).toInt(); continue; + } + if (arg == "--max-residual" && i + 1 < argc) { + opts.maxFitResidualPct = QString::fromLocal8Bit(argv[++i]).toDouble(); + continue; + } + if (!arg.startsWith("-") && inputPath.isEmpty()) { + inputPath = arg; continue; + } + } + + if (inputPath.isEmpty()) { + err() << "Error: No input file specified." << Qt::endl; + err() << "Usage: qtmesh facerig [-o out] [--max-shapes N] " + "[--max-residual PCT] [--json]" << Qt::endl; + return 2; + } + if (outputPath.isEmpty()) { + err() << "Error: -o required." << Qt::endl; + return 2; + } + + QFileInfo fi(inputPath); + if (!fi.exists()) { + err() << "Error: file not found: " << inputPath << Qt::endl; return 1; + } + if (!initOgreHeadless()) return 1; + + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QString("facerig .%1 max_shapes=%2").arg(fi.suffix()).arg(opts.maxShapes)); + // Extension only — absolute paths leak usernames/directory structure + // into telemetry. + SentryReporter::addBreadcrumb(QStringLiteral("file.import"), + QString("Importing .%1 for facerig").arg(fi.suffix())); + + MeshImporterExporter::importer({fi.absoluteFilePath()}); + QList meshEntities; + for (Ogre::Entity* e : Manager::getSingleton()->getEntities()) { + if (e && e->getMovableType() == "Entity") + meshEntities.push_back(e); + } + if (meshEntities.isEmpty()) { + err() << "Error: failed to load " << inputPath << Qt::endl; return 1; + } + if (meshEntities.size() > 1) { + err() << "Error: " << inputPath + << " contains multiple mesh entities. `qtmesh facerig` supports " + "one entity per file." << Qt::endl; + return 1; + } + Ogre::Entity* entity = meshEntities.first(); + + FaceRig::AttachReport rep; + if (qEnvironmentVariableIntValue("QTMESH_FACERIG_MARKER_SIM")) { + // Diagnostic: exercise the GUI's MARKER path headlessly — seed the + // markers, force-place them at their seeded/default positions, and rig + // from those anchors (RBF warp + anchored fit), printing per-shape + // delta stats. Mirrors FaceRigController::rigFromMarkers. + const QString tpath = FaceRig::ArkitTemplate::ensureModelBlocking(); + FaceRig::ArkitTemplate tmpl; + QString terr; + if (tpath.isEmpty() || !tmpl.load(tpath, &terr)) { + err() << "Error: template unavailable: " << terr << Qt::endl; + return 1; + } + FaceRig::FaceRigGeometry geo = FaceRig::extractGeometry(entity); + // per-submesh head-mask coverage (eye/teeth submeshes skinned to + // non-body-region bones can be silently excluded from the mask). + for (const auto& o : geo.owners) { + int inMask = 0; + for (int i = 0; i < o.count; ++i) + if (int(geo.headMask.size()) > int(o.base) + i + && geo.headMask[size_t(o.base) + size_t(i)]) ++inMask; + err() << "[sim] submesh handle=" << o.handle << " verts=" << o.count + << " inHeadMask=" << inMask << Qt::endl; + } + std::vector headV; std::vector headF; + FaceRig::headSubmesh(geo, headV, headF); + bool confident = false; + auto markers = FaceRig::seedFaceMarkers(entity, headV, headF, tmpl, &confident); + if (qEnvironmentVariableIntValue("QTMESH_FACERIG_MARKER_SIM") == 2) { + // sim mode 2: ignore the detection seeds and place every marker at + // its head-box-projected template position — approximates a user + // placing markers carefully on a proportional face. + float lo[3]={1e30f,1e30f,1e30f}, hi[3]={-1e30f,-1e30f,-1e30f}; + const int unv = int(headV.size()/3); + for (int i = 0; i < unv; ++i) + for (int a = 0; a < 3; ++a) { + lo[a]=std::min(lo[a],headV[size_t(i)*3+a]); + hi[a]=std::max(hi[a],headV[size_t(i)*3+a]); + } + const auto& tn = tmpl.neutral(); + float tlo[3]={1e30f,1e30f,1e30f}, thi[3]={-1e30f,-1e30f,-1e30f}; + for (int i = 0; i < tmpl.vertexCount(); ++i) + for (int a = 0; a < 3; ++a) { + tlo[a]=std::min(tlo[a],tn[size_t(i)*3+a]); + thi[a]=std::max(thi[a],tn[size_t(i)*3+a]); + } + for (auto& m : markers) { + if (m.tmplVertex < 0) continue; + for (int a = 0; a < 3; ++a) { + const float tv = tn[size_t(m.tmplVertex)*3+a]; + const float f = (thi[a]-tlo[a])>1e-6f ? (tv-tlo[a])/(thi[a]-tlo[a]) : 0.5f; + m.userPos[size_t(a)] = lo[a] + f*(hi[a]-lo[a]); + } + } + } + for (auto& m : markers) m.placed = true; // simulate the user placing all + const auto anchors = FaceRig::anchorsFromMarkers(markers, tmpl); + err() << "[sim] markers=" << markers.size() << " anchors=" << anchors.size() + << " seedConfident=" << confident << Qt::endl; + // seed-accuracy probe: when the rig target IS the template geometry + // (the ARKit reference), each marker's true position is its template + // vertex — print the seeding error per marker. + if (qEnvironmentVariableIntValue("QTMESH_FACERIG_MARKER_SIM") == 1) { + for (const auto& m : markers) { + if (!m.placed || m.tmplVertex < 0) continue; + const auto& tv = tmpl.neutral(); + const float dx = tv[size_t(m.tmplVertex)*3] - m.userPos[0]; + const float dy = tv[size_t(m.tmplVertex)*3+1] - m.userPos[1]; + const float dz = tv[size_t(m.tmplVertex)*3+2] - m.userPos[2]; + err() << "[sim] seederr '" << m.label << "' = " + << std::sqrt(dx*dx + dy*dy + dz*dz) << Qt::endl; + } + } + // surface-distance probe: how far does each seeded marker float off + // the head mesh? (box-mapped defaults used to hover off the face when + // protrusions inflate the head box) + for (const auto& m : markers) { + if (!m.placed) continue; + float best = 1e30f; + for (size_t i = 0; i + 2 < headV.size(); i += 3) { + const float dx = headV[i] - m.userPos[0]; + const float dy = headV[i+1] - m.userPos[1]; + const float dz = headV[i+2] - m.userPos[2]; + best = std::min(best, dx*dx + dy*dy + dz*dz); + } + err() << "[sim] surfdist '" << m.label << "' = " + << std::sqrt(best) << Qt::endl; + } + // side probe: character faces -Z, up +Y => character-LEFT = -X. + for (const auto& m : markers) + if (m.tmplVertex >= 0) + err() << "[sim] marker '" << m.label << "' tmplX=" + << tmpl.neutral()[size_t(m.tmplVertex)*3] + << (tmpl.neutral()[size_t(m.tmplVertex)*3] < 0 + ? " (character-LEFT side)" : " (character-RIGHT side)") + << Qt::endl; + // warp diagnostics: how big is the warped template vs the user head? + { + std::vector w = FaceRig::rbfWarpByAnchors(tmpl.neutral(), anchors); + auto diagOf = [](const std::vector& v){ + float lo[3]={1e30f,1e30f,1e30f}, hi[3]={-1e30f,-1e30f,-1e30f}; + for (size_t i=0;i+2getParentSceneNode(); + const QString fmt = formatForExtension(outputPath); + SentryReporter::addBreadcrumb(QStringLiteral("file.export"), + QString("Exporting %1").arg(QFileInfo(outputPath).absoluteFilePath())); + if (MeshImporterExporter::exporter(node, QFileInfo(outputPath).absoluteFilePath(), fmt) != 0) { + err() << "Error: export failed." << Qt::endl; + return 1; + } + // Sidecar with the ordered ARKit names (Assimp's glTF exporter drops + // targetNames), so `qtmesh mocap --face` / re-import can rebind by index. + FaceRig::writeArkitSidecar(QFileInfo(outputPath).absoluteFilePath(), + rep.shapeNames); + + if (jsonOutput) { + QJsonObject j; + j["shapes_attached"] = rep.shapesAttached; + j["user_vertex_count"] = rep.userVertexCount; + j["fit_mean_residual_pct"] = rep.fitMeanResidualPct; + j["fit_max_residual_pct"] = rep.fitMaxResidualPct; + j["output"] = QFileInfo(outputPath).fileName(); + cliWrite(QString::fromUtf8( + QJsonDocument(j).toJson(QJsonDocument::Indented)) + "\n"); + } else { + cliWrite(QString("Face-rig: attached %1 ARKit blendshape(s)\n" + " user vertices: %2\n" + " fit residual: mean %3% max %4%\n" + "Wrote: %5\n") + .arg(rep.shapesAttached) + .arg(rep.userVertexCount) + .arg(rep.fitMeanResidualPct, 0, 'f', 3) + .arg(rep.fitMaxResidualPct, 0, 'f', 3) + .arg(QFileInfo(outputPath).fileName())); + } + GamificationManager::noteOperation( + QStringLiteral("auto_rig"), + {{QStringLiteral("blendshapes_attached"), rep.shapesAttached}}, + GamificationManager::Surface::Cli); + return 0; +} + int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) { // Parse: generate3d [-o out.glb] [--resolution N] [--no-color] @@ -10003,6 +10263,64 @@ int CLIPipeline::cmdMorph(int argc, char* argv[]) } } + // Diagnostic (env-gated): verify a target actually PLAYS after import — + // enable it at weight 1 and measure the software-animated displacement. + // Distinguishes "targets listed but dead" (importer clip/pose bug) from a + // GUI-side playback issue. + if (!targets.isEmpty() + && qEnvironmentVariableIsSet("QTMESH_MORPH_PLAYTEST")) { + const QByteArray want = qgetenv("QTMESH_MORPH_PLAYTEST"); + QString name = QString::fromUtf8(want); + if (name == "1" || name.isEmpty()) name = targets.first(); + for (Ogre::Entity* entity : entities) { + auto* states = entity->getAllAnimationStates(); + const bool hasState = states + && states->hasAnimationState(name.toStdString()); + err() << "[playtest] entity=" << QString::fromStdString(entity->getName()) + << " target=" << name + << " hasAnimState=" << hasState + << " meshHasVertexAnim=" << entity->getMesh()->hasVertexAnimation() + << Qt::endl; + if (!hasState) continue; + auto* st = states->getAnimationState(name.toStdString()); + st->setEnabled(true); + st->setWeight(1.0f); + st->setTimePosition(0.0f); + entity->addSoftwareAnimationRequest(false); + entity->_updateAnimation(); + // measure displacement on each subentity's software-animated data + double maxDisp = 0; + for (unsigned int si = 0; si < entity->getNumSubEntities(); ++si) { + Ogre::SubEntity* se = entity->getSubEntity(si); + Ogre::VertexData* animVd = se->_getSoftwareVertexAnimVertexData(); + Ogre::VertexData* baseVd = se->getSubMesh()->vertexData; + if (!animVd || !baseVd) continue; + auto readPos = [](Ogre::VertexData* vd, std::vector& out) { + const auto* pe = vd->vertexDeclaration->findElementBySemantic(Ogre::VES_POSITION); + if (!pe) return; + auto vb = vd->vertexBufferBinding->getBuffer(pe->getSource()); + const size_t stride = vb->getVertexSize(); + auto* base = static_cast(vb->lock(Ogre::HardwareBuffer::HBL_READ_ONLY)); + out.resize(vd->vertexCount * 3); + for (size_t i = 0; i < vd->vertexCount; ++i) { + float* fp = nullptr; + pe->baseVertexPointerToElement(base + i*stride, &fp); + out[i*3] = fp[0]; out[i*3+1] = fp[1]; out[i*3+2] = fp[2]; + } + vb->unlock(); + }; + std::vector a, b; + readPos(animVd, a); readPos(baseVd, b); + for (size_t i = 0; i + 2 < std::min(a.size(), b.size()); i += 3) { + const double dx = a[i]-b[i], dy = a[i+1]-b[i+1], dz = a[i+2]-b[i+2]; + maxDisp = std::max(maxDisp, std::sqrt(dx*dx+dy*dy+dz*dz)); + } + } + err() << "[playtest] maxDisp after weight=1: " << maxDisp << Qt::endl; + entity->removeSoftwareAnimationRequest(false); + } + } + if (jsonOutput) { QJsonArray arr; for (const QString& n : targets) arr.append(n); diff --git a/src/CLIPipeline.h b/src/CLIPipeline.h index 775d501ff..969e4277d 100644 --- a/src/CLIPipeline.h +++ b/src/CLIPipeline.h @@ -225,6 +225,12 @@ class CLIPipeline { /// skin weights (--skin), and export. Issue #407. static int cmdRig(int argc, char* argv[]); + /// Face auto-rig (#889): fit the ARKit blendshape template onto a user + /// face mesh (NRICP + deformation transfer) and attach the 52 ARKit + /// morph targets, then export. `facerig [-o out] [--max-shapes N] + /// [--max-residual PCT] [--json]`. + static int cmdFaceRig(int argc, char* argv[]); + /// AI mesh part segmentation (#410): predict per-vertex/face part labels /// (head/torso/arm/leg) and emit a label map. `segment [--json] /// [--no-model] [--up-axis x|y|z]`. Text lists per-part vertex/face counts; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a06f61f83..afb9271af 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -125,7 +125,16 @@ SkinTokensPredictor.cpp SkinWeightsController.cpp AutoRig.cpp AutoRigController.cpp +FaceRigController.cpp UniRigPredictor.cpp +FaceRig/ArkitTemplate.cpp +FaceRig/NonRigidICP.cpp +FaceRig/SparseSolve.cpp +FaceRig/DeformationTransfer.cpp +FaceRig/FaceRigger.cpp +FaceRig/FaceRigAttach.cpp +FaceRig/FaceLandmarkDetector.cpp +FaceRig/FaceRigLandmarks.cpp MotionInbetween.cpp MotionLibrary.cpp MotionGenerator.cpp @@ -320,6 +329,7 @@ SkinTokensPredictor.h SkinWeightsController.h AutoRig.h AutoRigController.h +FaceRigController.h UniRigPredictor.h MotionInbetween.h MotionLibrary.h diff --git a/src/FaceRig/ArkitTemplate.cpp b/src/FaceRig/ArkitTemplate.cpp new file mode 100644 index 000000000..f30d8047d --- /dev/null +++ b/src/FaceRig/ArkitTemplate.cpp @@ -0,0 +1,217 @@ +#include "ArkitTemplate.h" + +#include "../ModelDownloader.h" +#include "../SentryReporter.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace FaceRig { + +namespace { +constexpr const char* kModelFile = "arkit_template.bin"; +constexpr const char* kMagic = "QMFRT1\0\0"; // 8 bytes +constexpr int kMagicLen = 8; +constexpr int kNameLen = 32; +constexpr const char* kDefaultModelBaseUrl = + "https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/facerig/"; +constexpr const char* kBaseUrlSettingsKey = "ai/facerigModelBaseUrl"; + +// little-endian readers over a byte cursor +int32_t rdI32(const char*& p) +{ + int32_t v; + std::memcpy(&v, p, 4); + p += 4; + return qFromLittleEndian(v); +} +float rdF32(const char*& p) +{ + quint32 raw; + std::memcpy(&raw, p, 4); + p += 4; + raw = qFromLittleEndian(raw); + float f; + std::memcpy(&f, &raw, 4); + return f; +} +} // namespace + +QStringList ArkitTemplate::shapeNames() const +{ + QStringList out; + for (const auto& s : m_shapes) + out << s.name; + return out; +} + +bool ArkitTemplate::load(const QString& path, QString* error) +{ + auto fail = [&](const QString& msg) { + if (error) + *error = msg; + return false; + }; + + QFile f(path); + if (!f.open(QIODevice::ReadOnly)) + return fail(QStringLiteral("cannot open %1").arg(path)); + const QByteArray blob = f.readAll(); + f.close(); + + // header: magic(8) + 3*int32 + if (blob.size() < kMagicLen + 12) + return fail(QStringLiteral("template too small / truncated")); + if (std::memcmp(blob.constData(), kMagic, kMagicLen) != 0) + return fail(QStringLiteral("bad magic (not an arkit_template.bin)")); + + const char* p = blob.constData() + kMagicLen; + const char* end = blob.constData() + blob.size(); + const int V = rdI32(p); + const int F = rdI32(p); + const int S = rdI32(p); + if (V <= 0 || F <= 0 || S <= 0 || V > 5'000'000 || S > 128) + return fail(QStringLiteral("implausible header (V=%1 F=%2 S=%3)") + .arg(V).arg(F).arg(S)); + + // exact byte budget check up front so a corrupt file can't over-read + const qint64 need = qint64(kMagicLen) + 12 + qint64(V) * 3 * 4 + + qint64(F) * 3 * 4 + + qint64(S) * (kNameLen + qint64(V) * 3 * 4); + if (blob.size() < need) + return fail(QStringLiteral("template truncated (need %1 bytes, have %2)") + .arg(need).arg(blob.size())); + + // Parse into TEMPORARIES and commit only after every check passes — a + // validation failure mid-file must not leave the object half-populated. + std::vector neutral(size_t(V) * 3, 0.0f); + for (auto& v : neutral) { + v = rdF32(p); + if (!std::isfinite(v)) + return fail(QStringLiteral("non-finite neutral position")); + } + std::vector faces(size_t(F) * 3, 0); + for (auto& i : faces) { + i = rdI32(p); + // out-of-range indices would be dereferenced by the fit / renders + if (i < 0 || i >= V) + return fail(QStringLiteral("face index %1 out of range (V=%2)") + .arg(i).arg(V)); + } + + std::vector shapes; + shapes.reserve(S); + for (int s = 0; s < S; ++s) { + if (p + kNameLen > end) + return fail(QStringLiteral("shape %1 name overruns").arg(s)); + ArkitShape shape; + shape.name = QString::fromLatin1(p, qstrnlen(p, kNameLen)); + p += kNameLen; + shape.deltas.resize(size_t(V) * 3); + for (auto& d : shape.deltas) { + d = rdF32(p); + if (!std::isfinite(d)) + return fail(QStringLiteral("non-finite delta in shape %1") + .arg(shape.name)); + } + shapes.push_back(std::move(shape)); + } + + m_vertexCount = V; + m_faceCount = F; + m_neutral = std::move(neutral); + m_faces = std::move(faces); + m_shapes = std::move(shapes); + return true; +} + +QString ArkitTemplate::modelPath() +{ + const QString dataPath = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + return QDir(dataPath).filePath(QStringLiteral("ai_models/facerig/") + + QString::fromLatin1(kModelFile)); +} + +bool ArkitTemplate::present() { return QFileInfo::exists(modelPath()); } + +QString ArkitTemplate::ensureModelBlocking() +{ + const QString dest = modelPath(); + if (QFileInfo::exists(dest)) + return dest; + if (!qEnvironmentVariableIsEmpty("QTMESH_FACERIG_NO_DOWNLOAD")) + return {}; + + QString base; + { + QSettings s; + base = s.value(QString::fromLatin1(kBaseUrlSettingsKey)).toString(); + if (base.isEmpty()) { + const QByteArray env = qgetenv("QTMESH_FACERIG_MODEL_BASE_URL"); + base = env.isEmpty() ? QString::fromLatin1(kDefaultModelBaseUrl) + : QString::fromUtf8(env); + } + } + if (base.isEmpty()) + return {}; + if (!base.endsWith('/')) + base += '/'; + + auto* dl = ModelDownloader::instance(); + if (!dl) + return {}; + + QDir().mkpath(QFileInfo(dest).absolutePath()); + const QString url = base + QString::fromLatin1(kModelFile); + const QString label = QStringLiteral("ARKit face template"); + + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QStringLiteral("ARKit template download start")); + + QEventLoop loop; + bool ok = false, timedOut = false, done = false; + auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = true; done = true; loop.quit(); } + }); + auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = false; done = true; loop.quit(); } + }); + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&timeout, &QTimer::timeout, &loop, + [&]() { timedOut = true; loop.quit(); }); + timeout.start(300000); // 5 min — the template is ~17 MB + + dl->startDownload(url, dest, label); + // `done` guards the synchronous-failure case: startDownload can emit + // downloadError DURING the call (another download active, .part file + // unopenable) — entering the loop then would block for the full timeout. + if (!done) + loop.exec(); + + QObject::disconnect(onDone); + QObject::disconnect(onErr); + if (timedOut && dl) + dl->cancelDownload(); + + const bool success = ok && !timedOut && QFileInfo::exists(dest); + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + success ? QStringLiteral("ARKit template download ok") + : QStringLiteral("ARKit template download failed%1") + .arg(timedOut ? QStringLiteral(" (timeout)") : QString())); + return success ? dest : QString(); +} + +} // namespace FaceRig diff --git a/src/FaceRig/ArkitTemplate.h b/src/FaceRig/ArkitTemplate.h new file mode 100644 index 000000000..bba7c1519 --- /dev/null +++ b/src/FaceRig/ArkitTemplate.h @@ -0,0 +1,63 @@ +#ifndef ARKITTEMPLATE_H +#define ARKITTEMPLATE_H + +// The ARKit blendshape TEMPLATE that face auto-rig (#889) transfers onto a +// user mesh. It is the ICT-FaceKit generic-neutral head (MIT, USC-ICT) plus +// its 52 ARKit-named expression deltas, packed by scripts/export-arkit- +// template.py into one binary (arkit_template.bin) and downloaded on first +// use to AppData/ai_models/facerig/. +// +// Ogre-free, pure data — the NonRigidICP (#891) / DeformationTransfer (#892) +// stages consume it; headless-unit-tested. Shape names are the canonical +// FaceCap::kBlendshapeNames (the mocap-52 vocabulary), so the generated +// morph targets match what face capture drives. + +#include +#include + +#include +#include + +namespace FaceRig { + +struct ArkitShape { + QString name; // a FaceCap::kBlendshapeNames entry + std::vector deltas; // vertexCount*3, (expr - neutral) +}; + +class ArkitTemplate { +public: + bool valid() const { return m_vertexCount > 0 && !m_shapes.empty(); } + int vertexCount() const { return m_vertexCount; } + int faceCount() const { return m_faceCount; } + int shapeCount() const { return static_cast(m_shapes.size()); } + + // neutral positions, vertexCount*3 (x,y,z interleaved) + const std::vector& neutral() const { return m_neutral; } + // triangle vertex indices, faceCount*3 + const std::vector& faces() const { return m_faces; } + const std::vector& shapes() const { return m_shapes; } + QStringList shapeNames() const; + + // Load from an explicit arkit_template.bin path. + bool load(const QString& path, QString* error = nullptr); + + // ---- model management (the house pattern) ---------------------------- + static QString modelPath(); // AppData/ai_models/facerig/arkit_template.bin + static bool present(); + // Blocking first-use download; returns the path or empty + // (offline guard QTMESH_FACERIG_NO_DOWNLOAD; base URL override + // QTMESH_FACERIG_MODEL_BASE_URL / QSettings ai/facerigModelBaseUrl). + static QString ensureModelBlocking(); + +private: + int m_vertexCount = 0; + int m_faceCount = 0; + std::vector m_neutral; + std::vector m_faces; + std::vector m_shapes; +}; + +} // namespace FaceRig + +#endif // ARKITTEMPLATE_H diff --git a/src/FaceRig/ArkitTemplate_test.cpp b/src/FaceRig/ArkitTemplate_test.cpp new file mode 100644 index 000000000..ebab4e467 --- /dev/null +++ b/src/FaceRig/ArkitTemplate_test.cpp @@ -0,0 +1,146 @@ +#include + +#include +#include +#include +#include +#include + +#include "FaceRig/ArkitTemplate.h" + +#include + +namespace { + +// Write a minimal valid arkit_template.bin: V verts, F faces, S named shapes. +QString writeTemplate(const QString& dir, int V, int F, + const QStringList& shapeNames) +{ + QByteArray b; + auto putI32 = [&](int32_t v) { + int32_t le = qToLittleEndian(v); + b.append(reinterpret_cast(&le), 4); + }; + auto putF32 = [&](float f) { + quint32 raw; + std::memcpy(&raw, &f, 4); + raw = qToLittleEndian(raw); + b.append(reinterpret_cast(&raw), 4); + }; + b.append("QMFRT1\0\0", 8); + putI32(V); + putI32(F); + putI32(shapeNames.size()); + for (int i = 0; i < V * 3; ++i) + putF32(0.1f * i); // deterministic neutral + for (int i = 0; i < F * 3; ++i) + putI32(i % V); // dummy faces + for (int s = 0; s < shapeNames.size(); ++s) { + QByteArray nm = shapeNames[s].toLatin1().left(31); + b.append(nm); + b.append(QByteArray(32 - nm.size(), '\0')); + for (int i = 0; i < V * 3; ++i) + putF32(s == 0 && i == 1 ? -0.5f : 0.0f); // shape0 moves vert0.y + } + const QString path = dir + QStringLiteral("/arkit_template.bin"); + QFile f(path); + f.open(QIODevice::WriteOnly); + f.write(b); + f.close(); + return path; +} + +} // namespace + +TEST(ArkitTemplate, LoadsHeaderNeutralFacesShapes) +{ + QTemporaryDir tmp; + ASSERT_TRUE(tmp.isValid()); + const QStringList names{"jawOpen", "mouthSmileLeft", "eyeBlinkRight"}; + const QString path = writeTemplate(tmp.path(), 4, 2, names); + + FaceRig::ArkitTemplate t; + QString err; + ASSERT_TRUE(t.load(path, &err)) << err.toStdString(); + EXPECT_TRUE(t.valid()); + EXPECT_EQ(t.vertexCount(), 4); + EXPECT_EQ(t.faceCount(), 2); + EXPECT_EQ(t.shapeCount(), 3); + EXPECT_EQ(t.neutral().size(), 12u); + EXPECT_FLOAT_EQ(t.neutral()[3], 0.3f); // vert1.x = 0.1*3 + EXPECT_EQ(t.faces().size(), 6u); + EXPECT_EQ(t.shapeNames(), names); + // shape 0 ("jawOpen") moves vert0.y by -0.5 + EXPECT_FLOAT_EQ(t.shapes()[0].deltas[1], -0.5f); + EXPECT_FLOAT_EQ(t.shapes()[1].deltas[1], 0.0f); +} + +TEST(ArkitTemplate, RejectsBadMagic) +{ + QTemporaryDir tmp; + const QString path = tmp.path() + "/bad.bin"; + QFile f(path); + f.open(QIODevice::WriteOnly); + f.write(QByteArray("NOTMAGIC", 8) + QByteArray(64, '\0')); + f.close(); + FaceRig::ArkitTemplate t; + QString err; + EXPECT_FALSE(t.load(path, &err)); + EXPECT_FALSE(err.isEmpty()); +} + +TEST(ArkitTemplate, RejectsTruncated) +{ + QTemporaryDir tmp; + const QString good = writeTemplate(tmp.path(), 4, 2, {"jawOpen"}); + QFile f(good); + f.open(QIODevice::ReadOnly); + QByteArray full = f.readAll(); + f.close(); + const QString path = tmp.path() + "/trunc.bin"; + QFile o(path); + o.open(QIODevice::WriteOnly); + o.write(full.left(full.size() - 20)); // chop the last shape's deltas + o.close(); + FaceRig::ArkitTemplate t; + QString err; + EXPECT_FALSE(t.load(path, &err)); + EXPECT_TRUE(err.contains("truncat", Qt::CaseInsensitive)); +} + +TEST(ArkitTemplate, MissingFileFails) +{ + FaceRig::ArkitTemplate t; + QString err; + EXPECT_FALSE(t.load("/nonexistent/arkit_template.bin", &err)); + EXPECT_FALSE(err.isEmpty()); + EXPECT_FALSE(t.valid()); +} + +// Env-gated: run against the REAL bundle (set QTMESH_FACERIG_TEMPLATE to the +// exported arkit_template.bin) — verifies the 51 shapes + ARKit names. +TEST(ArkitTemplate, EnvGatedRealBundle) +{ + const QByteArray p = qgetenv("QTMESH_FACERIG_TEMPLATE"); + if (p.isEmpty()) { + // pass as a no-op — the CI harness treats ANY skipped test as a suite + // failure (same convention as SkinEvaluate's env-gated reference test) + SUCCEED() << "QTMESH_FACERIG_TEMPLATE not set — real bundle not exercised"; + return; + } + FaceRig::ArkitTemplate t; + QString err; + ASSERT_TRUE(t.load(QString::fromUtf8(p), &err)) << err.toStdString(); + EXPECT_GT(t.vertexCount(), 10000); // ICT is ~26.7k + EXPECT_GE(t.shapeCount(), 51); + EXPECT_TRUE(t.shapeNames().contains("jawOpen")); + EXPECT_TRUE(t.shapeNames().contains("mouthSmileLeft")); + EXPECT_TRUE(t.shapeNames().contains("eyeBlinkLeft")); + // jawOpen should actually deform (nonzero deltas) + const auto& d = t.shapes()[t.shapeNames().indexOf("jawOpen")].deltas; + float maxMag = 0.f; + for (size_t i = 0; i + 2 < d.size(); i += 3) + maxMag = std::max(maxMag, + std::abs(d[i]) + std::abs(d[i + 1]) + std::abs(d[i + 2])); + EXPECT_GT(maxMag, 0.f); +} diff --git a/src/FaceRig/DeformationTransfer.cpp b/src/FaceRig/DeformationTransfer.cpp new file mode 100644 index 000000000..176c6a106 --- /dev/null +++ b/src/FaceRig/DeformationTransfer.cpp @@ -0,0 +1,294 @@ +#include "DeformationTransfer.h" + +#include +#include +#include + +namespace FaceRig { + +namespace { + +using Vec3 = std::array; +using Mat3 = std::array; // row-major + +Vec3 sub(const Vec3& a, const Vec3& b) { return {a[0]-b[0], a[1]-b[1], a[2]-b[2]}; } +Vec3 cross(const Vec3& a, const Vec3& b) +{ + return {a[1]*b[2]-a[2]*b[1], a[2]*b[0]-a[0]*b[2], a[0]*b[1]-a[1]*b[0]}; +} +double norm(const Vec3& a) { return std::sqrt(a[0]*a[0]+a[1]*a[1]+a[2]*a[2]); } + +Vec3 vert(const std::vector& v, int i) +{ + return {v[size_t(i)*3], v[size_t(i)*3+1], v[size_t(i)*3+2]}; +} + +// 4th "normal" vertex: v4 = v1 + n/√|n|, n = (v2-v1)×(v3-v1) +Vec3 normalV4(const Vec3& a, const Vec3& b, const Vec3& c) +{ + const Vec3 n = cross(sub(b, a), sub(c, a)); + const double ln = norm(n); + const double s = ln > 1e-12 ? 1.0 / std::sqrt(ln) : 0.0; + return {a[0]+n[0]*s, a[1]+n[1]*s, a[2]+n[2]*s}; +} + +// frame V = [v2-v1, v3-v1, v4-v1] as columns (row-major 3x3) +Mat3 frame(const Vec3& a, const Vec3& b, const Vec3& c, const Vec3& d) +{ + const Vec3 e1 = sub(b,a), e2 = sub(c,a), e3 = sub(d,a); + return {e1[0], e2[0], e3[0], + e1[1], e2[1], e3[1], + e1[2], e2[2], e3[2]}; +} + +bool invert3(const Mat3& m, Mat3& out) +{ + const double det = + m[0]*(m[4]*m[8]-m[5]*m[7]) - m[1]*(m[3]*m[8]-m[5]*m[6]) + + m[2]*(m[3]*m[7]-m[4]*m[6]); + if (std::abs(det) < 1e-18) + return false; + const double id = 1.0/det; + out[0] = (m[4]*m[8]-m[5]*m[7])*id; + out[1] = (m[2]*m[7]-m[1]*m[8])*id; + out[2] = (m[1]*m[5]-m[2]*m[4])*id; + out[3] = (m[5]*m[6]-m[3]*m[8])*id; + out[4] = (m[0]*m[8]-m[2]*m[6])*id; + out[5] = (m[2]*m[3]-m[0]*m[5])*id; + out[6] = (m[3]*m[7]-m[4]*m[6])*id; + out[7] = (m[1]*m[6]-m[0]*m[7])*id; + out[8] = (m[0]*m[4]-m[1]*m[3])*id; + return true; +} + +Mat3 matmul(const Mat3& a, const Mat3& b) +{ + Mat3 r{}; + for (int i = 0; i < 3; ++i) + for (int j = 0; j < 3; ++j) + for (int k = 0; k < 3; ++k) + r[size_t(i*3+j)] += a[size_t(i*3+k)] * b[size_t(k*3+j)]; + return r; +} + +} // namespace + +bool DeformationTransfer::init(const std::vector& tmplNeutral, + const std::vector& faces, + const std::vector& fitted) +{ + m_valid = false; + const int N = int(tmplNeutral.size() / 3); + const int F = int(faces.size() / 3); + if (N < 3 || F < 1 || int(fitted.size() / 3) != N) + return false; + // Reject malformed buffers outright: trailing floats/indices that don't + // form whole vertices/triangles, and face indices outside [0, N) — those + // would be dereferenced by vert() below. + if (tmplNeutral.size() % 3 != 0 || faces.size() % 3 != 0 + || fitted.size() % 3 != 0) + return false; + for (int idx : faces) + if (idx < 0 || idx >= N) + return false; + + m_n = N; + m_tmplNeutral = tmplNeutral; + m_faces = faces; + m_fitted = fitted; + // global template->fitted size ratio, used to scale island gauge-anchor + // deltas into fitted space (see the anchor rhs in transfer()). + { + auto diagOf = [](const std::vector& v) { + float lo[3] = {1e30f, 1e30f, 1e30f}, hi[3] = {-1e30f, -1e30f, -1e30f}; + for (size_t i = 0; i + 2 < v.size(); i += 3) + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], v[i + size_t(a)]); + hi[a] = std::max(hi[a], v[i + size_t(a)]); + } + double s2 = 0; + for (int a = 0; a < 3; ++a) + s2 += double(hi[a] - lo[a]) * (hi[a] - lo[a]); + return std::sqrt(s2); + }; + const double td = diagOf(tmplNeutral); + m_fitScale = td > 1e-12 ? diagOf(fitted) / td : 1.0; + } + m_srcRestInv.assign(size_t(F), {}); + m_tgtRestInv.assign(size_t(F), {}); + m_tgtNormalV4.assign(size_t(F), {}); + + // Unknowns (per axis, solved independently): the N deformed FITTED vertices + // PLUS one free "4th vertex" per triangle (columns N..N+F-1). The 4th vertex + // is Sumner's normal-direction trick: it only exists to make each triangle + // frame V = [x1-x0, x2-x0, x4-x0] a full-rank 3x3, and in the DEFORMED solve + // it is left UNCONSTRAINED (no normal equation pins it) — it simply absorbs + // the out-of-plane component the two edges can't represent. + // + // Per triangle the deformation gradient w.r.t. the target REST frame is + // A_f = Vtgt_deformed · Vtgt_rest⁻¹, and we require A_f == S_f (source). + // Each of the 3 gradient columns g gives one row (per axis): the deformed + // edge combination Σ_col Vinv[col][g]·(x_{col}-x0) must equal S column g. + // Anchor rows (small weight x_i == rest_i) fix the translation gauge. + + std::vector> trip; + trip.reserve(size_t(F) * 3 * 4 + size_t(N)); + + auto pushGrad = [&](int rowBase, int f) { + const int i0 = m_faces[size_t(f)*3]; + const int i1 = m_faces[size_t(f)*3+1]; + const int i2 = m_faces[size_t(f)*3+2]; + const int i3 = N + f; // the free 4th-vertex column + const Vec3 a = vert(m_fitted, i0), b = vert(m_fitted, i1), c = vert(m_fitted, i2); + const Vec3 d = normalV4(a, b, c); + m_tgtNormalV4[size_t(f)] = d; + Mat3 Vt = frame(a, b, c, d), Vinv; + if (!invert3(Vt, Vinv)) { + // degenerate target triangle — skip its gradient rows (anchors keep it) + m_tgtRestInv[size_t(f)] = {}; + return; + } + m_tgtRestInv[size_t(f)] = Vinv; + // edges columns are [x1-x0, x2-x0, x4-x0]; gradient column g combines + // them by Vinv[col][g]. Distribute onto the four real unknowns: + // x1 → Vinv[0][g], x2 → Vinv[1][g], x4 → Vinv[2][g], + // x0 → -(sum of the three) (from every -x0 term) + for (int g = 0; g < 3; ++g) { + const double w1 = Vinv[size_t(0*3+g)]; + const double w2 = Vinv[size_t(1*3+g)]; + const double w4 = Vinv[size_t(2*3+g)]; + const double w0 = -(w1 + w2 + w4); + const int row = rowBase + g; + trip.push_back({double(row), double(i0), w0}); + trip.push_back({double(row), double(i1), w1}); + trip.push_back({double(row), double(i2), w2}); + trip.push_back({double(row), double(i3), w4}); + } + }; + + // rows: 3 per triangle (gradient columns) + ONE anchor row PER CONNECTED + // COMPONENT. columns: N real verts + F free 4th-vertices. + // + // Deformation gradients are translation-invariant, so the system is only + // determined up to a translation PER TRIANGLE ISLAND — the ICT template + // has dozens of them (eyeballs, corneas, teeth, mouth interior). One + // anchor per island fixes each gauge; anchoring EVERY vertex would fight + // the shape (the gradients want expr, the anchor wants neutral) and pull + // the solution back toward the neutral, corrupting the transfer. + for (int f = 0; f < F; ++f) + pushGrad(3 * f, f); + const double anchorW = 1.0; + std::vector comp(size_t(N), 0); + for (int i = 0; i < N; ++i) comp[size_t(i)] = i; + auto findRoot = [&comp](int a) { + while (comp[size_t(a)] != a) { + comp[size_t(a)] = comp[size_t(comp[size_t(a)])]; + a = comp[size_t(a)]; + } + return a; + }; + for (int f = 0; f < F; ++f) { + const int a = findRoot(m_faces[size_t(f)*3]); + comp[size_t(findRoot(m_faces[size_t(f)*3+1]))] = a; + comp[size_t(findRoot(m_faces[size_t(f)*3+2]))] = a; + } + m_anchors.clear(); + std::set anchoredRoots; + for (int i = 0; i < N; ++i) { + if (anchoredRoots.insert(findRoot(i)).second) { + trip.push_back({double(3 * F + int(m_anchors.size())), + double(i), anchorW}); + m_anchors.push_back(i); + } + } + + m_A.fromTriplets(3 * F + int(m_anchors.size()), N + F, trip); + + // source (template) rest inverse frames — for building S_f per shape + for (int f = 0; f < F; ++f) { + const int i0 = m_faces[size_t(f)*3]; + const int i1 = m_faces[size_t(f)*3+1]; + const int i2 = m_faces[size_t(f)*3+2]; + const Vec3 a = vert(m_tmplNeutral, i0), b = vert(m_tmplNeutral, i1), + c = vert(m_tmplNeutral, i2); + const Vec3 d = normalV4(a, b, c); + Mat3 Vs = frame(a, b, c, d), Vinv; + m_srcRestInv[size_t(f)] = invert3(Vs, Vinv) ? Vinv : Mat3{}; + } + + m_valid = true; + return true; +} + +std::vector DeformationTransfer::transfer( + const std::vector& tmplExprDelta) const +{ + if (!m_valid || int(tmplExprDelta.size() / 3) != m_n) + return {}; + const int F = int(m_faces.size() / 3); + + // template EXPRESSION verts = neutral + delta + std::vector expr(m_tmplNeutral.size()); + for (size_t i = 0; i < expr.size(); ++i) + expr[i] = m_tmplNeutral[i] + tmplExprDelta[i]; + + // Build rhs for each of the 3 output axes. For triangle f: + // S_f = Vsrc_expr · Vsrc_rest⁻¹ (3x3 source deformation gradient). + // The gradient constraint rows require the target gradient column g == S + // column g; the free 4th vertex is an unknown (its normal equation exists + // in the matrix), so the rhs is just S — no rest-approximation term. + const int nCols = m_n + F; + std::vector> rhs( + 3, std::vector(size_t(3 * F) + m_anchors.size(), 0.0)); + + for (int f = 0; f < F; ++f) { + const int i0 = m_faces[size_t(f)*3]; + const int i1 = m_faces[size_t(f)*3+1]; + const int i2 = m_faces[size_t(f)*3+2]; + // source expression frame + Vec3 a{expr[size_t(i0)*3], expr[size_t(i0)*3+1], expr[size_t(i0)*3+2]}; + Vec3 b{expr[size_t(i1)*3], expr[size_t(i1)*3+1], expr[size_t(i1)*3+2]}; + Vec3 c{expr[size_t(i2)*3], expr[size_t(i2)*3+1], expr[size_t(i2)*3+2]}; + Vec3 d = normalV4(a, b, c); + Mat3 Vexpr = frame(a, b, c, d); + const Mat3& VsInv = m_srcRestInv[size_t(f)]; + const Mat3 S = matmul(Vexpr, VsInv); // 3x3 source gradient + for (int axis = 0; axis < 3; ++axis) + for (int g = 0; g < 3; ++g) + rhs[size_t(axis)][size_t(3*f+g)] = S[size_t(axis*3+g)]; + } + // anchor rhs: pin one vertex PER ISLAND to (fitted rest + the template's + // displacement of that vertex, SCALED into fitted space) — fixes each + // island's translation gauge while letting the shape move it the way the + // source moved it. The gradient system cannot represent rigid + // translation, and the raw template-space delta has the wrong amplitude + // when the fit changed scale; the global size ratio corrects that (a + // rotationless fit is the pipeline contract — the similarity prealign + // has no rotation term). + const double anchorW = 1.0; + for (size_t ai = 0; ai < m_anchors.size(); ++ai) { + const size_t v = size_t(m_anchors[ai]); + for (int axis = 0; axis < 3; ++axis) + rhs[size_t(axis)][size_t(3*F) + ai] = + anchorW * (m_fitted[v*3 + size_t(axis)] + + m_fitScale * tmplExprDelta[v*3 + size_t(axis)]); + } + + // solve per axis (warm-start real verts at the fitted rest, 4th verts at + // their rest normal offset). Read back only the N real vertices as a delta. + std::vector out(size_t(m_n) * 3, 0.0f); + for (int axis = 0; axis < 3; ++axis) { + std::vector x(size_t(nCols), 0.0); + for (int i = 0; i < m_n; ++i) + x[size_t(i)] = m_fitted[size_t(i)*3+axis]; + for (int f = 0; f < F; ++f) + x[size_t(m_n + f)] = m_tgtNormalV4[size_t(f)][size_t(axis)]; + solveLeastSquaresCG(m_A, rhs[size_t(axis)], x, 800, 1e-8); + for (int i = 0; i < m_n; ++i) + out[size_t(i)*3+axis] = + float(x[size_t(i)] - m_fitted[size_t(i)*3+axis]); // delta + } + return out; +} + +} // namespace FaceRig diff --git a/src/FaceRig/DeformationTransfer.h b/src/FaceRig/DeformationTransfer.h new file mode 100644 index 000000000..da4cd73c7 --- /dev/null +++ b/src/FaceRig/DeformationTransfer.h @@ -0,0 +1,71 @@ +#ifndef DEFORMATIONTRANSFER_H +#define DEFORMATIONTRANSFER_H + +// Deformation transfer (Sumner & Popović 2004) for face auto-rig (#889, +// Slice D #892). Pure data — no Ogre, reuses FaceRig::SparseMatrix — and +// headless-tested. +// +// Given the NRICP correspondence (template verts fitted onto the USER +// identity, in TEMPLATE topology — from NonRigidICP #891) and the template's +// per-shape neutral→expression delta, transfer each expression's per-triangle +// deformation onto the fitted (user-identity) mesh. The output is the +// expression realized on the user's identity, still in template topology, as +// a per-template-vertex delta from the fitted neutral. FaceRigger (#893) +// resamples that to the real user vertices via the correspondence. +// +// Method: per source triangle build the deformation gradient +// S = [e1' e2' n'] · [e1 e2 n]⁻¹ +// (Sumner's 4th "normal" vertex trick: n = (e1×e2)/√|e1×e2|), where the +// unprimed frame is the template NEUTRAL triangle and the primed is the +// template EXPRESSION triangle. Then solve, over the FITTED mesh, for vertex +// positions whose per-triangle deformation gradient matches S — one sparse +// least-squares (the Sumner-Popović matrix), with the fitted neutral as the +// rest state, plus a small anchor term to pin the solution (deformation +// gradients are translation-free). + +#include "SparseSolve.h" + +#include + +namespace FaceRig { + +// Precomputes the per-triangle rest frames of the FITTED mesh once, so all 52 +// shapes transfer without rebuilding the (topology-fixed) system. +class DeformationTransfer { +public: + // tmplNeutral / faces: the TEMPLATE neutral verts (N*3) + tris (F*3). + // fitted: template verts fitted onto the user identity (N*3) — the NRICP + // correspondence; SAME topology as the template. + bool init(const std::vector& tmplNeutral, + const std::vector& faces, + const std::vector& fitted); + + bool valid() const { return m_valid; } + int vertexCount() const { return m_n; } + + // Transfer one template shape (tmplExprDelta = expr - tmplNeutral, N*3) → + // a per-vertex delta on the FITTED mesh (N*3, added to `fitted` gives the + // expression on the user identity). Returns empty on failure. + std::vector transfer(const std::vector& tmplExprDelta) const; + +private: + bool m_valid = false; + int m_n = 0; + std::vector m_tmplNeutral; // template rest (for source gradients) + std::vector m_faces; + std::vector m_fitted; // target rest (user identity) + // The transfer system A x = c is fixed by topology + fitted rest; only c + // (built from each shape's source deformation gradient) changes per shape. + SparseMatrix m_A; + // per-triangle inverse rest frames of the TEMPLATE neutral (source) and of + // the FITTED mesh (target), cached for building c. + std::vector> m_srcRestInv; // 3x3 per tri + std::vector> m_tgtRestInv; + std::vector> m_tgtNormalV4; // fitted 4th vertex/tri + std::vector m_anchors; // one gauge-anchor vertex per island + double m_fitScale = 1.0; // template->fitted global size ratio +}; + +} // namespace FaceRig + +#endif // DEFORMATIONTRANSFER_H diff --git a/src/FaceRig/DeformationTransfer_test.cpp b/src/FaceRig/DeformationTransfer_test.cpp new file mode 100644 index 000000000..6a0155112 --- /dev/null +++ b/src/FaceRig/DeformationTransfer_test.cpp @@ -0,0 +1,149 @@ +#include + +#include "FaceRig/DeformationTransfer.h" + +#include +#include + +namespace { + +struct Grid { + std::vector V; + std::vector F; +}; + +// A bumpy plane grid — real triangles for the per-face deformation gradients. +Grid makeGrid(int n, float extent, float bump = 0.0f) +{ + Grid g; + for (int y = 0; y < n; ++y) + for (int x = 0; x < n; ++x) { + const float fx = (float(x)/(n-1) - 0.5f) * extent; + const float fy = (float(y)/(n-1) - 0.5f) * extent; + const float fz = bump * std::sin(float(x)) * std::cos(float(y)); + g.V.insert(g.V.end(), {fx, fy, fz}); + } + for (int y = 0; y < n-1; ++y) + for (int x = 0; x < n-1; ++x) { + const int a = y*n+x, b = y*n+x+1, c = (y+1)*n+x, d = (y+1)*n+x+1; + g.F.insert(g.F.end(), {a, b, c}); + g.F.insert(g.F.end(), {b, d, c}); + } + return g; +} + +double diag(const std::vector& v) +{ + float lo[3] = {1e30f,1e30f,1e30f}, hi[3] = {-1e30f,-1e30f,-1e30f}; + for (size_t i = 0; i < v.size(); i += 3) + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], v[i+a]); + hi[a] = std::max(hi[a], v[i+a]); + } + double s = 0; + for (int a = 0; a < 3; ++a) s += double(hi[a]-lo[a])*double(hi[a]-lo[a]); + return std::sqrt(s); +} + +double maxAbs(const std::vector& v) +{ + double m = 0; + for (float x : v) m = std::max(m, double(std::abs(x))); + return m; +} + +} // namespace + +TEST(DeformationTransfer, RejectsBadInput) +{ + FaceRig::DeformationTransfer dt; + const Grid g = makeGrid(6, 2.0f); + EXPECT_FALSE(dt.init({}, {}, {})); // empty + EXPECT_FALSE(dt.init(g.V, g.F, {})); // fitted count mismatch + EXPECT_FALSE(dt.valid()); + // valid init + EXPECT_TRUE(dt.init(g.V, g.F, g.V)); + EXPECT_TRUE(dt.valid()); + EXPECT_EQ(dt.vertexCount(), int(g.V.size()/3)); +} + +// Identity: fitted == template neutral. Transferring a shape's delta should +// reproduce (approximately) that same delta — the source and target rest +// frames are identical, so the deformation gradients map back to the input. +TEST(DeformationTransfer, IdentityFitReproducesShapeDelta) +{ + const Grid g = makeGrid(10, 2.0f, 0.1f); + FaceRig::DeformationTransfer dt; + ASSERT_TRUE(dt.init(g.V, g.F, g.V)); + + // a shape: push the centre region up in +Z (like a smile bump) + const int n = int(g.V.size()/3); + std::vector delta(g.V.size(), 0.0f); + for (int i = 0; i < n; ++i) { + const float x = g.V[size_t(i)*3], y = g.V[size_t(i)*3+1]; + const float bump = 0.15f * std::exp(-(x*x + y*y) * 4.0f); + delta[size_t(i)*3+2] = bump; + } + const auto out = dt.transfer(delta); + ASSERT_EQ(out.size(), delta.size()); + for (float v : out) ASSERT_TRUE(std::isfinite(v)); + + // out should track delta closely (identity transfer). Compare RMS error. + double num = 0, den = 0; + for (size_t i = 0; i < delta.size(); ++i) { + const double d = double(out[i]) - double(delta[i]); + num += d*d; + den += double(delta[i])*double(delta[i]); + } + const double relRms = std::sqrt(num / std::max(den, 1e-12)); + EXPECT_LT(relRms, 0.15); // within 15% of the original shape +} + +// Winding / orientation preserved: transferring onto a UNIFORMLY SCALED user +// identity should scale the shape delta by the same factor (deformation +// gradients are scale-covariant), and keep signs (no inside-out flip). +TEST(DeformationTransfer, ScaledIdentityScalesTheDelta) +{ + const Grid tmpl = makeGrid(10, 2.0f, 0.1f); + // fitted = template scaled 2x (a "bigger head") + std::vector fitted(tmpl.V.size()); + for (size_t i = 0; i < fitted.size(); ++i) fitted[i] = tmpl.V[i] * 2.0f; + + FaceRig::DeformationTransfer dt; + ASSERT_TRUE(dt.init(tmpl.V, tmpl.F, fitted)); + + const int n = int(tmpl.V.size()/3); + std::vector delta(tmpl.V.size(), 0.0f); + for (int i = 0; i < n; ++i) { + const float x = tmpl.V[size_t(i)*3], y = tmpl.V[size_t(i)*3+1]; + delta[size_t(i)*3+2] = 0.12f * std::exp(-(x*x + y*y) * 4.0f); + } + const auto out = dt.transfer(delta); + ASSERT_EQ(out.size(), delta.size()); + for (float v : out) ASSERT_TRUE(std::isfinite(v)); + + // on a 2x mesh the same relative deformation should be ~2x the amplitude + const double dIn = maxAbs(delta); + const double dOut = maxAbs(out); + EXPECT_GT(dOut, dIn * 1.3); // clearly scaled up + EXPECT_LT(dOut, dIn * 3.0); // but bounded + + // sign preserved: peak stays +Z (no flip) + double peak = 0; + for (int i = 0; i < n; ++i) + if (std::abs(out[size_t(i)*3+2]) > std::abs(peak)) + peak = out[size_t(i)*3+2]; + EXPECT_GT(peak, 0.0); +} + +// Zero shape → zero delta (the neutral maps to the neutral). +TEST(DeformationTransfer, ZeroDeltaProducesZero) +{ + const Grid g = makeGrid(8, 2.0f, 0.1f); + FaceRig::DeformationTransfer dt; + ASSERT_TRUE(dt.init(g.V, g.F, g.V)); + std::vector zero(g.V.size(), 0.0f); + const auto out = dt.transfer(zero); + ASSERT_EQ(out.size(), zero.size()); + EXPECT_LT(maxAbs(out) / diag(g.V), 1e-3); +} diff --git a/src/FaceRig/FaceLandmarkDetector.cpp b/src/FaceRig/FaceLandmarkDetector.cpp new file mode 100644 index 000000000..1ba874f20 --- /dev/null +++ b/src/FaceRig/FaceLandmarkDetector.cpp @@ -0,0 +1,375 @@ +#include "FaceLandmarkDetector.h" + +#include "ArkitTemplate.h" // reuse its model dir + base-url convention +#include "../ModelDownloader.h" +#include "../SentryReporter.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef ENABLE_ONNX +#include +#endif + +namespace FaceRig { + +namespace { +constexpr const char* kModelFile = "face_landmarks.onnx"; +constexpr int kInputSize = 256; // MediaPipe FaceMesh V2 crop +constexpr int kMinLandmarkFloats = 468 * 3; // 468/478 landmarks × xyz +} // namespace + +struct FaceLandmarkDetector::Impl { +#ifdef ENABLE_ONNX + std::unique_ptr env; + std::unique_ptr session; + std::vector inputNames; + std::vector outputNames; + std::vector inputNamesC; + std::vector outputNamesC; +#endif + bool loaded = false; +}; + +FaceLandmarkDetector::FaceLandmarkDetector() : d(std::make_unique()) {} +FaceLandmarkDetector::~FaceLandmarkDetector() = default; + +QString FaceLandmarkDetector::modelPath() +{ + const QString dataPath = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + return QDir(dataPath).filePath(QStringLiteral("ai_models/facerig/") + + QString::fromLatin1(kModelFile)); +} + +bool FaceLandmarkDetector::present() { return QFileInfo::exists(modelPath()); } + +bool FaceLandmarkDetector::backendAvailable() +{ +#ifdef ENABLE_ONNX + return true; +#else + return false; +#endif +} + +QString FaceLandmarkDetector::ensureModelBlocking() +{ + const QString dest = modelPath(); + if (QFileInfo::exists(dest)) + return dest; + if (!qEnvironmentVariableIsEmpty("QTMESH_FACERIG_NO_DOWNLOAD")) + return {}; + + // Same base URL as the ARKit template (they live together in facerig/). + QString base; + { + QSettings s; + base = s.value(QStringLiteral("ai/facerigModelBaseUrl")).toString(); + if (base.isEmpty()) { + const QByteArray env = qgetenv("QTMESH_FACERIG_MODEL_BASE_URL"); + base = env.isEmpty() + ? QStringLiteral("https://huggingface.co/fernandotonon/" + "QtMeshEditor-models/resolve/main/facerig/") + : QString::fromUtf8(env); + } + } + if (base.isEmpty()) + return {}; + if (!base.endsWith('/')) + base += '/'; + + auto* dl = ModelDownloader::instance(); + if (!dl) + return {}; + + QDir().mkpath(QFileInfo(dest).absolutePath()); + const QString url = base + QString::fromLatin1(kModelFile); + const QString label = QStringLiteral("face landmark model"); + + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QStringLiteral("face landmark model download start")); + + QEventLoop loop; + bool ok = false, timedOut = false, done = false; + auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = true; done = true; loop.quit(); } + }); + auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = false; done = true; loop.quit(); } + }); + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&timeout, &QTimer::timeout, &loop, + [&]() { timedOut = true; loop.quit(); }); + timeout.start(120000); // 2 min — the model is small (~3 MB) + + dl->startDownload(url, dest, label); + // done-guard: a synchronous downloadError would otherwise block the + // loop for the full timeout (same pattern as ArkitTemplate). + if (!done) + loop.exec(); + + QObject::disconnect(onDone); + QObject::disconnect(onErr); + if (timedOut && dl) + dl->cancelDownload(); + + const bool success = ok && !timedOut && QFileInfo::exists(dest); + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + success ? QStringLiteral("face landmark model download ok") + : QStringLiteral("face landmark model download failed%1") + .arg(timedOut ? QStringLiteral(" (timeout)") : QString())); + return success ? dest : QString(); +} + +bool FaceLandmarkDetector::isAvailable() const { return d && d->loaded; } + +#ifdef ENABLE_ONNX + +bool FaceLandmarkDetector::load(const QString& path) +{ + m_error.clear(); + d->loaded = false; + const QString p = path.isEmpty() ? modelPath() : path; + if (!QFileInfo::exists(p)) { + m_error = QStringLiteral("face landmark model not found at %1").arg(p); + return false; + } + try { + d->env = std::make_unique(ORT_LOGGING_LEVEL_WARNING, + "qtmesh_facelmk"); + Ort::SessionOptions so; + so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); +#if defined(__APPLE__) + try { + std::unordered_map coreml; + so.AppendExecutionProvider("CoreML", coreml); + } catch (...) { /* CPU fallback */ } +#endif +#ifdef _WIN32 + const std::wstring wp = p.toStdWString(); + d->session = std::make_unique(*d->env, wp.c_str(), so); +#else + const std::string sp = p.toStdString(); + d->session = std::make_unique(*d->env, sp.c_str(), so); +#endif + Ort::AllocatorWithDefaultOptions alloc; + const size_t ni = d->session->GetInputCount(); + for (size_t i = 0; i < ni; ++i) { + auto n = d->session->GetInputNameAllocated(i, alloc); + d->inputNames.emplace_back(n.get()); + } + const size_t no = d->session->GetOutputCount(); + for (size_t i = 0; i < no; ++i) { + auto n = d->session->GetOutputNameAllocated(i, alloc); + d->outputNames.emplace_back(n.get()); + } + for (auto& s : d->inputNames) d->inputNamesC.push_back(s.c_str()); + for (auto& s : d->outputNames) d->outputNamesC.push_back(s.c_str()); + d->loaded = !d->inputNames.empty() && !d->outputNames.empty(); + if (!d->loaded) + m_error = QStringLiteral("model has no inputs/outputs"); + return d->loaded; + } catch (const std::exception& e) { + m_error = QStringLiteral("failed to load face landmark model: %1") + .arg(QString::fromUtf8(e.what())); + d->loaded = false; + return false; + } +} + +LandmarkResult FaceLandmarkDetector::detect(const QImage& image) +{ + LandmarkResult r; + if (!isAvailable() || image.isNull()) return r; + + const int W = image.width(), H = image.height(); + const int side0 = std::min(W, H); + + // The landmark graph is trained on tight, detector-cropped FACES. Measured + // on our own renders: presence logit -27 full-frame, -8.6 with the head + // filling the frame, +19.8 on a chin-to-forehead crop — so candidate crops + // must reach face-tightness before the model reports a face at all. + // + // Our renders put the subject on a black background, so a silhouette scan + // finds the head deterministically: bbox of non-black pixels, head width + // measured over the TOP part of the blob (shoulders are wider), then a few + // face-square candidates around it. A non-black background degrades to the + // centre-square fallback candidate. + struct Cand { int ox, oy, side; }; + std::vector cands; + { + const QImage gray = image.convertToFormat(QImage::Format_Grayscale8); + int top = H, bot = -1; + std::vector rowMin(size_t(H), W), rowMax(size_t(H), -1); + for (int y = 0; y < H; ++y) { + const uchar* ln = gray.constScanLine(y); + for (int x = 0; x < W; ++x) { + if (ln[x] > 12) { + top = std::min(top, y); bot = std::max(bot, y); + rowMin[size_t(y)] = std::min(rowMin[size_t(y)], x); + rowMax[size_t(y)] = std::max(rowMax[size_t(y)], x); + } + } + } + if (bot > top + 16) { + // head width = median row span over the top 35% of the blob + const int headRows = std::max(8, (bot - top) * 35 / 100); + int wMin = W, wMax = -1; + long cxSum = 0; int cxN = 0; + for (int y = top + headRows / 4; y < top + headRows; ++y) { + if (rowMax[size_t(y)] < 0) continue; + wMin = std::min(wMin, rowMin[size_t(y)]); + wMax = std::max(wMax, rowMax[size_t(y)]); + cxSum += (rowMin[size_t(y)] + rowMax[size_t(y)]) / 2; ++cxN; + } + if (wMax > wMin + 16 && cxN > 0) { + const int headW = wMax - wMin; + const int cx = int(cxSum / cxN); + // face square candidates: the face sits in the lower-middle of + // the head — try a few sizes/centres around it. + for (float s : {0.9f, 1.15f, 1.45f}) { + int side = std::min(int(headW * s), std::min(W, H)); + if (side < 32) continue; + const int cyFace = top + int(side * 0.62f); + cands.push_back({ + std::clamp(cx - side / 2, 0, W - side), + std::clamp(cyFace - side / 2, 0, H - side), + side }); + } + } + } + } + cands.push_back({ (W - side0) / 2, (H - side0) / 2, side0 }); // fallback + + LandmarkResult best; + for (const Cand& c : cands) { + LandmarkResult pr = runPass(image, c.ox, c.oy, c.side); + if (pr.ok && !pr.points.empty() + && pr.presenceLogit > best.presenceLogit) { + best = std::move(pr); + } + } + if (!best.ok || best.points.empty()) return best; + + // Refine: tight re-crop around the winning landmark bbox (×1.6, the + // expansion MediaPipe's own face detector applies). + float mnx = best.points[0][0], mxx = mnx; + float mny = best.points[0][1], mxy = mny; + for (const auto& p : best.points) { + mnx = std::min(mnx, p[0]); mxx = std::max(mxx, p[0]); + mny = std::min(mny, p[1]); mxy = std::max(mxy, p[1]); + } + const float cx = (mnx + mxx) * 0.5f, cy = (mny + mxy) * 0.5f; + int side1 = int(std::max(mxx - mnx, mxy - mny) * 1.6f); + if (side1 >= 32) { + side1 = std::min(side1, std::min(W, H)); + const int ox1 = std::clamp(int(cx - side1 * 0.5f), 0, W - side1); + const int oy1 = std::clamp(int(cy - side1 * 0.5f), 0, H - side1); + LandmarkResult pass2 = runPass(image, ox1, oy1, side1); + if (pass2.ok && !pass2.points.empty() + && pass2.presenceLogit >= best.presenceLogit) + return pass2; + } + return best; +} + +LandmarkResult FaceLandmarkDetector::runPass(const QImage& image, + int ox, int oy, int side) +{ + LandmarkResult r; + if (!isAvailable() || image.isNull() || side <= 0) return r; + QImage crop = image.copy(ox, oy, side, side) + .convertToFormat(QImage::Format_RGB888) + .scaled(kInputSize, kInputSize, Qt::IgnoreAspectRatio, + Qt::SmoothTransformation); + + // NHWC float [0,1] + std::vector input(size_t(kInputSize) * kInputSize * 3); + for (int y = 0; y < kInputSize; ++y) { + const uchar* line = crop.constScanLine(y); + for (int x = 0; x < kInputSize; ++x) { + const uchar* px = line + x * 3; + const size_t o = (size_t(y) * kInputSize + x) * 3; + input[o + 0] = px[0] / 255.0f; + input[o + 1] = px[1] / 255.0f; + input[o + 2] = px[2] / 255.0f; + } + } + + try { + Ort::MemoryInfo mem = Ort::MemoryInfo::CreateCpu( + OrtArenaAllocator, OrtMemTypeDefault); + const std::array shape{1, kInputSize, kInputSize, 3}; + Ort::Value in = Ort::Value::CreateTensor( + mem, input.data(), input.size(), shape.data(), shape.size()); + auto outs = d->session->Run( + Ort::RunOptions{nullptr}, d->inputNamesC.data(), &in, 1, + d->outputNamesC.data(), d->outputNamesC.size()); + + const float* rawLandmarks = nullptr; + size_t landmarkFloats = 0; + float presenceLogit = 0.f; + bool presenceFound = false; + for (auto& o : outs) { + const auto info = o.GetTensorTypeAndShapeInfo(); + const size_t count = info.GetElementCount(); + if (count >= size_t(kMinLandmarkFloats)) { + rawLandmarks = o.GetTensorData(); + landmarkFloats = count; + } else if (count == 1 && !presenceFound) { + presenceLogit = o.GetTensorData()[0]; + presenceFound = true; + } + } + if (!rawLandmarks) { + m_error = QStringLiteral("unexpected face landmark outputs"); + return r; + } + r.confidence = presenceFound + ? 1.f / (1.f + std::exp(-std::clamp(presenceLogit, -50.f, 50.f))) + : 1.f; + r.presenceLogit = presenceFound ? presenceLogit : 0.f; + + const int n = int(landmarkFloats / 3); + r.points.reserve(size_t(n)); + const float scale = float(side) / float(kInputSize); + for (int i = 0; i < n; ++i) { + // model outputs 256-space px (x,y) + relative z; map back to the + // ORIGINAL image pixel space via the crop offset + scale. + const float lx = rawLandmarks[i * 3 + 0] * scale + ox; + const float ly = rawLandmarks[i * 3 + 1] * scale + oy; + const float lz = rawLandmarks[i * 3 + 2] * scale; + r.points.push_back({lx, ly, lz}); + } + r.ok = true; + return r; + } catch (const std::exception& e) { + m_error = QStringLiteral("face landmark inference failed: %1") + .arg(QString::fromUtf8(e.what())); + return r; + } +} + +#else // !ENABLE_ONNX + +bool FaceLandmarkDetector::load(const QString&) +{ + m_error = QStringLiteral("built without ONNX (ENABLE_ONNX off)"); + return false; +} +LandmarkResult FaceLandmarkDetector::detect(const QImage&) { return {}; } + +#endif // ENABLE_ONNX + +} // namespace FaceRig diff --git a/src/FaceRig/FaceLandmarkDetector.h b/src/FaceRig/FaceLandmarkDetector.h new file mode 100644 index 000000000..6f69f0804 --- /dev/null +++ b/src/FaceRig/FaceLandmarkDetector.h @@ -0,0 +1,82 @@ +#ifndef FACELANDMARKDETECTOR_H +#define FACELANDMARKDETECTOR_H + +// Facial-landmark detection for the face auto-rig (#889). Runs the MediaPipe +// Face Mesh V2 landmark graph (face_landmarks.onnx — Apache-2.0, the same model +// the mocap face-capture uses, #869) on a rendered head image to get the 478 +// canonical face landmarks. Those anchor the non-rigid ICP fit so the ARKit +// template lands on the ACTUAL face features (eyes/nose/mouth) instead of a +// low-residual-but-mis-oriented drape — the fix for wrong shape placement. +// +// We render the head ourselves (centred, front-facing, evenly lit), so the +// upstream face DETECTOR is unnecessary — we feed a plain centred 256×256 crop +// straight to the landmark graph. Ogre-free + ENABLE_ONNX-guarded; the model +// downloads on first use to ai_models/facerig/ (same dir/base-url as the ARKit +// template). Without ONNX or the model, isAvailable() stays false and the +// caller falls back to the landmark-free fit. + +#include +#include + +#include +#include +#include + +namespace FaceRig { + +struct LandmarkResult { + // 478 landmarks as (x,y,z), x/y in the INPUT IMAGE's pixel space (0..W/H), + // z a relative depth (unused for back-projection). Empty when no face. + std::vector> points; + float confidence = 0.0f; // presence sigmoid, 0 = no face + // RAW presence logit. The sigmoid saturates (~1.0) for both a true face + // (logit ~+20) and a convincing false positive like the smooth back of a + // head (low positive logit) — rank candidate views/crops by THIS, never + // by `confidence`. + float presenceLogit = -1e9f; + bool ok = false; +}; + +class FaceLandmarkDetector { +public: + FaceLandmarkDetector(); + ~FaceLandmarkDetector(); + + // ---- model management (mirrors ArkitTemplate) ---- + static QString modelPath(); // AppData/ai_models/facerig/face_landmarks.onnx + static bool present(); + // Blocking first-use download; returns the path or empty (offline guard + // QTMESH_FACERIG_NO_DOWNLOAD; base URL QTMESH_FACERIG_MODEL_BASE_URL / + // QSettings ai/facerigModelBaseUrl — same as the ARKit template). + static QString ensureModelBlocking(); + + // True only when built with ENABLE_ONNX. + static bool backendAvailable(); + + // Load the ONNX session from `path` (default: modelPath()). Returns false + // if ONNX is off / the file is missing / the session can't be created. + bool load(const QString& path = {}); + bool isAvailable() const; + QString lastError() const { return m_error; } + + // Detect landmarks on `image` (any format; the head should fill the frame, + // centred and front-facing). Points come back in `image`'s pixel space. + // Runs TWO passes: a loose full-frame pass to locate the face, then a + // tight re-crop around the pass-1 landmark bbox — FaceMesh is trained on + // detector-cropped faces, so a loose frame depresses both the presence + // score and landmark accuracy. + LandmarkResult detect(const QImage& image); + +private: + // One inference on the square crop (ox, oy, side) of `image`; landmark + // x/y mapped back to `image` pixel space. + LandmarkResult runPass(const QImage& image, int ox, int oy, int side); + + struct Impl; + std::unique_ptr d; + QString m_error; +}; + +} // namespace FaceRig + +#endif // FACELANDMARKDETECTOR_H diff --git a/src/FaceRig/FaceRigAttach.cpp b/src/FaceRig/FaceRigAttach.cpp new file mode 100644 index 000000000..70b37ab2b --- /dev/null +++ b/src/FaceRig/FaceRigAttach.cpp @@ -0,0 +1,339 @@ +#include "FaceRigAttach.h" + +#include "ArkitTemplate.h" +#include "FaceRigLandmarks.h" + +#include "../AutoRig.h" +#include "../MeshSegmenter.h" +#include "../commands/MorphCommands.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +namespace FaceRig { + +namespace { + +// Read tight xyz floats from a VertexData POSITION element (mirrors +// SkinWeights.cpp's extractor). Returns false if there's no position stream. +bool extractPositions(Ogre::VertexData* vd, std::vector& out) +{ + if (!vd) return false; + const auto* posElem = + vd->vertexDeclaration->findElementBySemantic(Ogre::VES_POSITION); + if (!posElem) return false; + auto vbuf = vd->vertexBufferBinding->getBuffer(posElem->getSource()); + if (!vbuf || vd->vertexCount == 0) return false; + out.resize(size_t(vd->vertexCount) * 3); + const size_t stride = vbuf->getVertexSize(); + auto* base = static_cast( + vbuf->lock(Ogre::HardwareBuffer::HBL_READ_ONLY)); + for (size_t i = 0; i < vd->vertexCount; ++i) { + float* fp = nullptr; + posElem->baseVertexPointerToElement(base + i * stride, &fp); + out[3*i+0] = fp[0]; out[3*i+1] = fp[1]; out[3*i+2] = fp[2]; + } + vbuf->unlock(); + return true; +} + +void appendIndices(Ogre::IndexData* id, std::uint32_t offset, + std::vector& out) +{ + if (!id || !id->indexBuffer || id->indexCount == 0) return; + auto ibuf = id->indexBuffer; + const auto* base = static_cast( + ibuf->lock(Ogre::HardwareBuffer::HBL_READ_ONLY)); + base += id->indexStart * ibuf->getIndexSize(); + out.reserve(out.size() + id->indexCount); + if (ibuf->getType() == Ogre::HardwareIndexBuffer::IT_32BIT) { + const auto* ip = reinterpret_cast(base); + for (size_t i = 0; i < id->indexCount; ++i) out.push_back(int(ip[i] + offset)); + } else { + const auto* ip = reinterpret_cast(base); + for (size_t i = 0; i < id->indexCount; ++i) out.push_back(int(ip[i] + offset)); + } + ibuf->unlock(); +} + +} // namespace + +FaceRigGeometry extractGeometry(Ogre::Entity* entity) +{ + FaceRigGeometry geo; + if (!entity) return geo; + Ogre::MeshPtr mesh = entity->getMesh(); + if (!mesh) return geo; + + // Collect every geometry owner into ONE combined vertex/index set so the + // fit sees the whole face (a single-submesh head is the common case, but a + // face split across submeshes still fits as one surface). Track each owner + // so we can split the per-vertex deltas back onto the right pose handle. + auto addOwner = [&](unsigned short handle, Ogre::VertexData* vd, + Ogre::IndexData* id) { + std::vector pos; + if (!extractPositions(vd, pos)) return; + const std::uint32_t base = std::uint32_t(geo.userV.size() / 3); + geo.owners.push_back({handle, base, int(pos.size() / 3)}); + geo.userV.insert(geo.userV.end(), pos.begin(), pos.end()); + appendIndices(id, base, geo.userF); + }; + + // Only extract the shared pool when a submesh actually references it — + // some importers allocate sharedVertexData that no submesh uses, and the + // orphan vertices would join the fit (and the head mask / bounding + // computations) without any triangles. + bool anyShared = false; + for (unsigned short si = 0; si < mesh->getNumSubMeshes(); ++si) { + Ogre::SubMesh* sm = mesh->getSubMesh(si); + if (sm && sm->useSharedVertices) { anyShared = true; break; } + } + if (mesh->sharedVertexData && anyShared) { + // shared pool → handle 0; its indices live per-submesh. + std::vector pos; + if (extractPositions(mesh->sharedVertexData, pos)) { + geo.owners.push_back({0, 0, int(pos.size() / 3)}); + geo.userV.insert(geo.userV.end(), pos.begin(), pos.end()); + for (unsigned short si = 0; si < mesh->getNumSubMeshes(); ++si) { + Ogre::SubMesh* sm = mesh->getSubMesh(si); + if (sm && sm->useSharedVertices) + appendIndices(sm->indexData, 0, geo.userF); + } + } + } + for (unsigned short si = 0; si < mesh->getNumSubMeshes(); ++si) { + Ogre::SubMesh* sm = mesh->getSubMesh(si); + if (!sm || sm->useSharedVertices) continue; + addOwner(static_cast(si + 1), sm->vertexData, sm->indexData); + } + + // ── HEAD ISOLATION (the fix for full-body characters) ──────────────────── + // The ARKit template is a FACE. Fitting it against a whole dancing body + // smears the face over the body (mouth shapes end up on an arm). So we + // isolate the head region and only fit / deform those vertices. Preference: + // 1) rig-prior — if the mesh is SKINNED, label each vertex by the body + // region of the bone it's most weighted to (EXACT; handles the fox's + // snout/ears and Mixamo's exaggerated proportions the coordinate model + // can't). This is AutoRig::rigPriorPartLabels, ordered to match our + // combined gather. + // 2) geometric fallback — MeshSegmenter's spatial head/torso/limb split. + // If neither finds a plausible head (e.g. the mesh really IS just a face), + // leave headMask empty and fit the whole thing (previous behaviour). + const int nv = int(geo.userV.size() / 3); + const int headPart = int(MeshSegmenter::Part::Head); + std::vector labels; + if (mesh->hasSkeleton()) { + int resolved = 0; + labels = AutoRig::rigPriorPartLabels(entity, nv, &resolved); + // Require the rig prior to resolve a decent share; else fall through. + if (resolved < nv / 2) labels.clear(); + } + if (labels.empty() && geo.userF.size() >= 3) { + std::vector idx(geo.userF.begin(), geo.userF.end()); + MeshSegmenter::Result seg = MeshSegmenter::segmentGeometric( + geo.userV.data(), nv, idx.data(), int(idx.size())); + if (seg.ok) labels = seg.vertexLabels; + } + if (int(labels.size()) == nv) { + int headCount = 0; + geo.headMask.assign(size_t(nv), 0); + for (int v = 0; v < nv; ++v) + if (labels[size_t(v)] == headPart) { geo.headMask[size_t(v)] = 1; ++headCount; } + + // GEOMETRIC expansion: eyes / teeth / tongue / lashes are often + // separate submeshes skinned to non-body-region bones (eye bones), + // which the rig-prior can't label — they'd be silently EXCLUDED from + // the face rig and never blink or look around (field-reported: 414-vert + // eye submesh with 0 masked verts). Anything inside the labeled head's + // slightly-expanded AABB belongs to the face. + if (headCount >= 50) { + float lo[3] = {1e30f, 1e30f, 1e30f}, hi[3] = {-1e30f, -1e30f, -1e30f}; + for (int v = 0; v < nv; ++v) { + if (!geo.headMask[size_t(v)]) continue; + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], geo.userV[size_t(v)*3 + a]); + hi[a] = std::max(hi[a], geo.userV[size_t(v)*3 + a]); + } + } + float pad[3]; + for (int a = 0; a < 3; ++a) pad[a] = 0.05f * (hi[a] - lo[a]); + for (int v = 0; v < nv; ++v) { + if (geo.headMask[size_t(v)]) continue; + bool inside = true; + for (int a = 0; a < 3; ++a) { + const float p = geo.userV[size_t(v)*3 + a]; + if (p < lo[a] - pad[a] || p > hi[a] + pad[a]) { inside = false; break; } + } + if (inside) { geo.headMask[size_t(v)] = 1; ++headCount; } + } + } + + // Only isolate when the head is a real, minority region of the mesh — + // i.e. this looks like a full body, not a bare face. A head that IS + // most of the mesh means it's already a face crop; fit it whole. + if (headCount >= 50 && headCount < nv * 3 / 4) + geo.headVertexCount = headCount; + else + geo.headMask.clear(); // treat as a face crop + } + return geo; +} + +void headSubmesh(const FaceRigGeometry& geo, + std::vector& outV, std::vector& outF) +{ + outV.clear(); + outF.clear(); + const int nv = int(geo.userV.size() / 3); + if (int(geo.headMask.size()) != nv) { + // no head isolation — the whole mesh IS the face crop. + outV = geo.userV; + outF = geo.userF; + return; + } + std::vector fullToSub(size_t(nv), -1); + for (int v = 0; v < nv; ++v) { + if (!geo.headMask[size_t(v)]) continue; + fullToSub[size_t(v)] = int(outV.size() / 3); + outV.insert(outV.end(), {geo.userV[size_t(v)*3], + geo.userV[size_t(v)*3+1], + geo.userV[size_t(v)*3+2]}); + } + for (size_t f = 0; f + 2 < geo.userF.size(); f += 3) { + const int a = geo.userF[f], b = geo.userF[f+1], c = geo.userF[f+2]; + const int nv = int(fullToSub.size()); + if (a < 0 || b < 0 || c < 0 || a >= nv || b >= nv || c >= nv) continue; + const int sa = fullToSub[size_t(a)], sb = fullToSub[size_t(b)], + sc = fullToSub[size_t(c)]; + if (sa >= 0 && sb >= 0 && sc >= 0) + outF.insert(outF.end(), {sa, sb, sc}); + } + if (outV.size() < 9 || outF.size() < 3) { outV = geo.userV; outF = geo.userF; } +} + +void attachShapes(Ogre::Entity* entity, const FaceRigGeometry& geo, + const FaceRigResult& result, AttachReport& report) +{ + // Attach each shape as a pose + VAT_POSE clip, splitting the combined + // deltas back onto each owner's handle. Reuse AddMorphTargetCommand's + // redo() (the exact MorphCommands pose-build) so face capture drives these + // with no new playback code; call redo() directly (headless-safe — no undo + // stack required, GUI callers can wrap in UndoManager separately). + for (const FaceRigShape& shape : result.shapes) { + std::vector slices; + for (const GeometryOwner& o : geo.owners) { + MorphPoseSlice slice; + slice.submeshHandle = o.handle; + for (int i = 0; i < o.count; ++i) { + const std::uint32_t gv = o.base + std::uint32_t(i); + if (size_t(gv) * 3 + 2 >= shape.userDeltas.size()) break; + const float* d = &shape.userDeltas[size_t(gv) * 3]; + if (d[0] == 0.0f && d[1] == 0.0f && d[2] == 0.0f) continue; + slice.offsets[static_cast(i)] = + Ogre::Vector3f(d[0], d[1], d[2]); + } + if (!slice.offsets.empty()) slices.push_back(std::move(slice)); + } + if (slices.empty()) continue; // shape moved nothing on this mesh + AddMorphTargetCommand cmd(entity, shape.name, slices); + cmd.redo(); + report.shapesAttached++; + report.shapeNames.push_back(shape.name); + } + report.ok = report.shapesAttached > 0; + if (!report.ok) + report.error = QStringLiteral("no blendshapes produced any vertex movement"); +} + +AttachReport attachFaceRig(Ogre::Entity* entity, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts) +{ + AttachReport rep; + if (!entity) { rep.error = QStringLiteral("no entity"); return rep; } + if (!entity->getMesh()) { rep.error = QStringLiteral("entity has no mesh"); return rep; } + if (!tmpl.valid()) { rep.error = QStringLiteral("ARKit template not loaded"); return rep; } + + const FaceRigGeometry geo = extractGeometry(entity); + if (!geo.valid()) { + rep.error = QStringLiteral("could not read mesh geometry"); + return rep; + } + + // Facial-landmark anchors (render + detect on template AND user, pair by + // MediaPipe index). Frame/raycast the HEAD sub-mesh so the face fills the + // detector's frame. Empty when ONNX/model/face-detection unavailable — the + // fit then runs unanchored (previous behaviour). + std::vector headV; std::vector headF; + headSubmesh(geo, headV, headF); + const std::vector anchors = + buildLandmarkAnchors(entity, headV, headF, tmpl); + + const FaceRigResult res = buildFaceRig(geo.userV, geo.userF, tmpl, opts, + geo.headMask, anchors); + rep.userVertexCount = res.userVertexCount; + rep.fitMeanResidualPct = res.fitMeanResidualPct; + rep.fitMaxResidualPct = res.fitMaxResidualPct; + if (!res.ok) { + rep.error = QString::fromStdString(res.error); + return rep; + } + + attachShapes(entity, geo, res, rep); + return rep; +} + +AttachReport attachFaceRigWithBundledTemplate(Ogre::Entity* entity, + const FaceRigOptions& opts) +{ + AttachReport rep; + const QString path = ArkitTemplate::ensureModelBlocking(); + if (path.isEmpty()) { + rep.error = QStringLiteral( + "ARKit template unavailable (offline and not downloaded, or the " + "build has no face-rig model). Set QTMESH_FACERIG_MODEL_BASE_URL or " + "download arkit_template.bin to ai_models/facerig/."); + return rep; + } + ArkitTemplate tmpl; + QString err; + if (!tmpl.load(path, &err)) { + rep.error = QStringLiteral("failed to load ARKit template: %1").arg(err); + return rep; + } + return attachFaceRig(entity, tmpl, opts); +} + +bool writeArkitSidecar(const QString& meshPath, + const std::vector& shapeNames) +{ + if (meshPath.isEmpty() || shapeNames.empty()) return false; + QJsonArray names; + for (const QString& n : shapeNames) names.append(n); + QJsonObject root; + root["schema"] = QStringLiteral("qtmesh-arkit-blendshapes-v1"); + root["count"] = int(shapeNames.size()); + root["names"] = names; // ordered, matches the mesh's morph targets + + // .arkit.json alongside the exported mesh — recovers the ARKit names + // that Assimp 6.0's glTF exporter drops (it doesn't emit targetNames even + // though we set aiAnimMesh::mName). Face capture (#869) and re-import can + // read this to rebind the mocap-52 vocabulary by index. + QFile f(meshPath + QStringLiteral(".arkit.json")); + if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) return false; + f.write(QJsonDocument(root).toJson(QJsonDocument::Indented)); + f.close(); + return true; +} + +} // namespace FaceRig diff --git a/src/FaceRig/FaceRigAttach.h b/src/FaceRig/FaceRigAttach.h new file mode 100644 index 000000000..8cb645558 --- /dev/null +++ b/src/FaceRig/FaceRigAttach.h @@ -0,0 +1,101 @@ +#ifndef FACERIGATTACH_H +#define FACERIGATTACH_H + +// FaceRigAttach — the Ogre-touching bridge for the face auto-rig (#889, +// Slice E #893). Extracts a user entity's mesh geometry, runs the Ogre-free +// FaceRigger (NRICP → DeformationTransfer → resample), and attaches the +// resulting ARKit blendshapes as Ogre::Pose + VAT_POSE morph targets on the +// entity — reusing the exact pose-build mechanic MorphCommands uses so face +// capture (#869) drives them with no new playback code. +// +// Shared by the CLI (`qtmesh facerig`), MCP (`add_arkit_blendshapes`), and the +// GUI "Add ARKit Blendshapes" button. + +#include "FaceRigger.h" + +#include + +#include +#include + +namespace Ogre { class Entity; } + +namespace FaceRig { + +struct AttachReport { + bool ok = false; + QString error; + int shapesAttached = 0; + int userVertexCount = 0; + double fitMeanResidualPct = 0.0; + double fitMaxResidualPct = 0.0; + QString templateFallback; // set when the template had to be downloaded/etc. + std::vector shapeNames; // attached shape names, in order +}; + +// One geometry owner (shared vertex pool, or a per-submesh vertex data) with +// the morph pose target handle it maps to and its base offset into the combined +// vertex set. Ogre's 1-based convention: 0 = shared, 1..N = submesh index+1. +struct GeometryOwner { + unsigned short handle; + std::uint32_t base; + int count; +}; + +// The user entity's geometry read out of Ogre once, so the heavy Ogre-free +// buildFaceRig() can run OFF the main thread (the GUI path) while extraction + +// attach stay on the main thread. Pure data — no Ogre handles retained. +struct FaceRigGeometry { + std::vector userV; // combined positions (Nu*3) + std::vector userF; // combined triangle indices + std::vector owners; + // Per-combined-vertex head flag (size Nu). Empty = no head isolation (the + // whole mesh is treated as the face). When populated, only head vertices + // participate in the fit + receive blendshape deltas — so a full-body + // character rigs correctly instead of smearing the face over the body. + std::vector headMask; + int headVertexCount = 0; + bool valid() const { return userV.size() >= 9 && userF.size() >= 3; } +}; + +// MAIN-thread: read the entity's combined geometry (locks Ogre hardware +// buffers — milliseconds). Empty/invalid result on failure. +FaceRigGeometry extractGeometry(Ogre::Entity* entity); + +// Extract the HEAD sub-mesh (local V + remapped F) from a geometry + its +// headMask — the region the landmark detector should frame/raycast. Returns the +// full mesh when there's no head mask (a bare-face crop). Pure data. +void headSubmesh(const FaceRigGeometry& geo, + std::vector& outV, std::vector& outF); + +// MAIN-thread: attach a computed FaceRigResult's shapes to `entity` as +// Ogre::Pose + VAT_POSE morph targets (via AddMorphTargetCommand), splitting +// the combined per-vertex deltas back onto each owner's handle. Fills the +// report's shapesAttached / ok. `geo.owners` must match the geometry the +// result was computed from. +void attachShapes(Ogre::Entity* entity, const FaceRigGeometry& geo, + const FaceRigResult& result, AttachReport& report); + +// Runs the whole pipeline on `entity` using the given (already-loaded) template +// and attaches the shapes as poses + a per-target VAT_POSE clip. Re-initialises +// the live entity so the pose buffers exist (mirrors the morph/auto-rig path). +// On failure nothing is attached and report.error explains why. +AttachReport attachFaceRig(Ogre::Entity* entity, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts = {}); + +// Convenience: ensure the bundled template is available (download-on-first-use), +// load it, then attachFaceRig. Returns a clear error if the template can't be +// obtained (offline + not present, or the build lacks the model). +AttachReport attachFaceRigWithBundledTemplate(Ogre::Entity* entity, + const FaceRigOptions& opts = {}); + +// Write a `.arkit.json` sidecar with the ordered ARKit shape names, +// so downstream tools recover the names even though Assimp 6.0's glTF exporter +// drops mesh.extras.targetNames. Returns false on write failure. +bool writeArkitSidecar(const QString& meshPath, + const std::vector& shapeNames); + +} // namespace FaceRig + +#endif // FACERIGATTACH_H diff --git a/src/FaceRig/FaceRigLandmarks.cpp b/src/FaceRig/FaceRigLandmarks.cpp new file mode 100644 index 000000000..effefcecc --- /dev/null +++ b/src/FaceRig/FaceRigLandmarks.cpp @@ -0,0 +1,1180 @@ +#include "FaceRigLandmarks.h" + +#include "ArkitTemplate.h" +#include "FaceLandmarkDetector.h" +#include "FaceRigAttach.h" + +#include "../Manager.h" +#include "../MeshDepthRenderer.h" +#include "../TransformOperator.h" +#include "../OgreWidget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace FaceRig { + +namespace { + +constexpr int kRenderSize = 512; // render + detect resolution + +// Möller–Trumbore ray/triangle intersection. Returns t (ray param) > 0 on hit. +bool rayTri(const Ogre::Vector3& o, const Ogre::Vector3& d, + const Ogre::Vector3& a, const Ogre::Vector3& b, + const Ogre::Vector3& c, float& tOut) +{ + const Ogre::Vector3 e1 = b - a, e2 = c - a; + const Ogre::Vector3 p = d.crossProduct(e2); + const float det = e1.dotProduct(p); + if (std::fabs(det) < 1e-9f) return false; + const float inv = 1.0f / det; + const Ogre::Vector3 tv = o - a; + const float u = tv.dotProduct(p) * inv; + if (u < -1e-4f || u > 1.0001f) return false; + const Ogre::Vector3 q = tv.crossProduct(e1); + const float v = d.dotProduct(q) * inv; + if (v < -1e-4f || u + v > 1.0001f) return false; + const float t = e2.dotProduct(q) * inv; + if (t <= 1e-5f) return false; + tOut = t; + return true; +} + +} // namespace + +MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, + const std::vector& localV, + const std::vector& localF) +{ + MeshLandmarks out; + if (!entity || localV.size() < 9 || localF.size() < 3) return out; + + FaceLandmarkDetector det; + if (!det.load()) return out; // ONNX off / model missing → caller falls back + + // World transform for building the head focus box + later ray/tri tests. + Ogre::Node* node = entity->getParentNode(); + const Ogre::Matrix4 world = node ? node->_getFullTransform() + : Ogre::Matrix4::IDENTITY; + + // Head focus box (WORLD space) from the local head verts, so the render + // frames tightly on the FACE — a full-body character's face would be a few + // pixels if we framed the whole entity, and MediaPipe wouldn't detect it. + Ogre::AxisAlignedBox focus; + { const int fnv = int(localV.size() / 3); + for (int i = 0; i < fnv; ++i) + focus.merge(world * Ogre::Vector3(localV[size_t(i)*3], + localV[size_t(i)*3+1], + localV[size_t(i)*3+2])); } + + // 1+2) render the head + detect landmarks — MULTI-VIEW. Nothing guarantees + // the mesh faces the renderer's "front" (glTF assets commonly face +Z + // while MeshDepthRenderer::front() places the camera on -Z; the LH-flip + // asymmetry between import and glTF export also flips facing on + // round-trips). Render the four horizontal views and keep the one + // MediaPipe is most confident about; a true face scores high (~0.9+), the + // back of a head scores low or fails outright. Early-out on a confident + // hit so the common facing stays one render. + const MeshDepthRenderer::View views[] = { + MeshDepthRenderer::front(), MeshDepthRenderer::back(), + MeshDepthRenderer::left(), MeshDepthRenderer::right(), + }; + MeshDepthRenderer::RenderResult rr; + LandmarkResult lr; + float bestLogit = -1e9f; + // FACING may only be claimed by SHADED-mode detections: the fog depth + // statue has no texture and MediaPipe false-positives on smooth domes + // (measured: Rumba's occiput depth render scored logit +8 while its true + // stylized face scored -3.5 — the facing flipped to the back of the + // head). Depth-mode detections still feed LANDMARKS (the constellation + // gate protects those), just never the facing decision. + float bestShadedLogit = -1e9f; + Ogre::Vector3 bestShadedCamDir = Ogre::Vector3::ZERO; + // Geometric facing fallback: mean |Laplacian| of the fog depth render + // over subject pixels. The face side of a head (nose, brows, lips, chin) + // carries far more depth detail than the smooth occiput — used to decide + // facing when NO view shows positive face evidence (stylized faces + // MediaPipe can't read return only noise logits). + double bestDetail = -1.0; + Ogre::Vector3 bestDetailCamDir = Ogre::Vector3::ZERO; + auto depthDetail = [](const QImage& img) -> double { + const QImage g = img.convertToFormat(QImage::Format_Grayscale8); + double acc = 0; long n = 0; + for (int y = 1; y + 1 < g.height(); ++y) { + const uchar* lm = g.constScanLine(y - 1); + const uchar* lc = g.constScanLine(y); + const uchar* lp = g.constScanLine(y + 1); + for (int x = 1; x + 1 < g.width(); ++x) { + if (lc[x] <= 12) continue; // background + acc += std::fabs(4.0 * lc[x] - lc[x-1] - lc[x+1] - lm[x] - lp[x]); + ++n; + } + } + return n > 0 ? acc / double(n) : -1.0; + }; + // Two render styles per view: the shaded render (materials intact — + // carries texture contrast MediaPipe likes) and the fog depth-map render + // (pure geometry statue — immune to broken normals / inconsistent winding + // / missing textures, which turn the shaded render into unusable noise). + // + // EVERY view in a mode is evaluated — no first-hit early-out. The winner + // is the highest RAW presence logit: a true face scores ~+20 while a + // false positive (the smooth back of a head) scores far lower, but both + // saturate the sigmoid, so an early-out on `confidence` locked onto the + // back of backwards-facing imports (glb round-trips flip facing). Ranking + // all four views by logit IS the orientation detection. + constexpr float kStrongFaceLogit = 6.0f; // sigmoid ≈ 0.998 + for (int depthMode = 0; + depthMode <= 1 && bestLogit < kStrongFaceLogit; ++depthMode) { + for (const auto& view : views) { + QString err; + MeshDepthRenderer::RenderResult vrr = depthMode + ? MeshDepthRenderer::renderDepthMapView( + entity, kRenderSize, view, &err, + focus.isNull() ? nullptr : &focus) + : MeshDepthRenderer::renderShadedView( + entity, kRenderSize, view, &err, + focus.isNull() ? nullptr : &focus); + if (vrr.depth.isNull()) continue; + if (depthMode) { + const double det = depthDetail(vrr.depth); + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] detail view=%s score=%.2f\n", + view.name, det); + if (det > bestDetail) { + bestDetail = det; + bestDetailCamDir = vrr.camDirection; + } + } + // Flatness sanity: a blown-out / silhouette render (near-zero + // intensity variance inside the subject) carries no facial + // features — MediaPipe false-positives on such blobs with high + // presence, and the garbage landmarks CORRELATE between the + // template and user renders, slipping through the constellation + // gate. A genuinely shaded face has stddev well above this. + { + const QImage g = vrr.depth.convertToFormat(QImage::Format_Grayscale8); + double sum = 0, sum2 = 0; long n = 0; + for (int y = 0; y < g.height(); ++y) { + const uchar* ln = g.constScanLine(y); + for (int x = 0; x < g.width(); ++x) { + if (ln[x] > 12) { sum += ln[x]; sum2 += double(ln[x]) * ln[x]; ++n; } + } + } + const double var = n > 0 ? (sum2 / n - (sum / n) * (sum / n)) : 0.0; + if (n < 64 || var < 36.0) { // stddev < 6 → featureless + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] detect view=%s mode=%s " + "SKIPPED (flat render, var=%.1f)\n", + view.name, depthMode ? "depth" : "shaded", var); + continue; + } + } + LandmarkResult vlr = det.detect(vrr.depth); + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, + "[facerig] detect view=%s mode=%s ok=%d conf=%.2f " + "logit=%.1f\n", + view.name, depthMode ? "depth" : "shaded", + vlr.ok, vlr.confidence, vlr.presenceLogit); + if (const char* dp = std::getenv("QTMESH_FACERIG_DUMP_RENDER")) + vrr.depth.save(QString::fromUtf8(dp) + "." + + QString::fromStdString(entity->getName()) + "." + + (depthMode ? "depth." : "shaded.") + + view.name + ".png"); + if (!vlr.ok || vlr.points.empty()) continue; + if (!depthMode && vlr.presenceLogit > bestShadedLogit) { + bestShadedLogit = vlr.presenceLogit; + bestShadedCamDir = vrr.camDirection; + } + if (vlr.presenceLogit > bestLogit) { + bestLogit = vlr.presenceLogit; + rr = std::move(vrr); + lr = std::move(vlr); + } + } + } + // Facing signal: the face points TOWARD the winning view's camera + // (= against its look direction). When NO view produced positive face + // evidence (all logits negative — MediaPipe can't read stylized faces), + // the logit "winner" is noise; fall back to the depth-DETAIL winner + // instead (the face side out-details the smooth back of the head). + // Exposed in MESH-LOCAL space even when the landmarks themselves are too + // weak to use — the proportional-default marker placement needs only the + // facing. + { + // Facing ladder: (1) a strong SHADED-mode detection; (2) the active + // viewport camera; (3) feet direction; (4) depth-detail winner; (5) + // the overall logit winner as a last resort. + Ogre::Vector3 camDir = Ogre::Vector3::ZERO; + const bool strongShaded = bestShadedLogit >= kStrongFaceLogit + && !bestShadedCamDir.isZeroLength(); + if (strongShaded) { + camDir = bestShadedCamDir; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] facing from SHADED detection " + "(logit=%.1f)\n", bestShadedLogit); + } + // The ACTIVE VIEWPORT camera is the strongest user-intent hint: the + // user orbits to LOOK AT the face before rigging, so the face points + // toward that camera. Trust it over every geometric fallback whenever + // detection itself isn't conclusive. + bool vpResolved = false; + if (!strongShaded) { + if (auto* to = TransformOperator::getSingletonPtr()) { + if (auto* w = to->getActiveWidget()) { + if (w->getViewport() && w->getViewport()->getCamera()) { + const Ogre::Vector3 d = + w->getViewport()->getCamera()->getDerivedDirection(); + if (!d.isZeroLength()) { + camDir = d; // camera looks toward the face + vpResolved = true; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, + "[facerig] facing from VIEWPORT camera " + "(bestLogit=%.1f)\n", bestLogit); + } + } + } + } + } + if (!strongShaded && !vpResolved) { + // No positive face evidence anywhere (stylized / covered faces) — + // the logit "winner" is noise. For a FULL-BODY character the feet + // are the strongest facing cue: toes extend forward of the ankle. + // Compare the horizontal centroid of the feet slab (lowest 8% of + // the body) against the ankle slab above it. + bool feetResolved = false; + const FaceRigGeometry full = extractGeometry(entity); + if (full.valid()) { + const int n = int(full.userV.size() / 3); + float bLoY = 1e30f, bHiY = -1e30f; + for (int i = 0; i < n; ++i) { + bLoY = std::min(bLoY, full.userV[size_t(i)*3+1]); + bHiY = std::max(bHiY, full.userV[size_t(i)*3+1]); + } + // head height from the head verts we render (localV) + float hLoY = 1e30f, hHiY = -1e30f; + for (int i = 0; i < int(localV.size()/3); ++i) { + hLoY = std::min(hLoY, localV[size_t(i)*3+1]); + hHiY = std::max(hHiY, localV[size_t(i)*3+1]); + } + const float bodyH = bHiY - bLoY, headH = hHiY - hLoY; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] feet gate: bodyH=%.2f " + "headH=%.2f ratio=%.2f\n", + bodyH, headH, headH > 0 ? bodyH/headH : 0.f); + if (bodyH > 2.5f * headH && bodyH > 1e-6f) { + const float feetTop = bLoY + 0.08f * bodyH; + const float ankleTop = bLoY + 0.16f * bodyH; + double fx = 0, fz = 0, ax = 0, az = 0; + long nf = 0, na = 0; + for (int i = 0; i < n; ++i) { + const float y = full.userV[size_t(i)*3+1]; + if (y < feetTop) { + fx += full.userV[size_t(i)*3]; fz += full.userV[size_t(i)*3+2]; ++nf; + } else if (y < ankleTop) { + ax += full.userV[size_t(i)*3]; az += full.userV[size_t(i)*3+2]; ++na; + } + } + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] feet slabs: nf=%ld " + "na=%ld\n", nf, na); + if (nf > 8 && na > 8) { + const double dx = fx/nf - ax/na, dz = fz/nf - az/na; + const double len = std::sqrt(dx*dx + dz*dz); + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] feet dir: " + "d=(%.3f,%.3f) len=%.3f min=%.3f\n", + dx, dz, len, 0.005 * bodyH); + // 0.5% of body height: the toe-forward offset is small + // on dance-pose rigs (Rumba: 1.9cm on a 2m body) but + // its DIRECTION is reliable; only reject a truly + // degenerate (near-zero) offset. + if (len > 0.005 * bodyH) { + // LOCAL-space facing; convert to a WORLD camDir + // (the block below converts back) by mapping + // through the entity transform. + const Ogre::Vector3 o = world * Ogre::Vector3::ZERO; + Ogre::Vector3 faceW = + (world * Ogre::Vector3(float(dx), 0, float(dz))) - o; + if (!faceW.isZeroLength()) { + camDir = -faceW; // camera looks against facing + feetResolved = true; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, + "[facerig] facing from FEET dir " + "local=(%.2f,0,%.2f)\n", dx/len, dz/len); + } + } + } + } + } + if (!feetResolved && bestDetail >= 0.0 + && !bestDetailCamDir.isZeroLength()) { + camDir = bestDetailCamDir; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] facing from depth detail " + "(bestLogit=%.1f, detail=%.2f)\n", + bestLogit, bestDetail); + } + } + if (!camDir.isZeroLength()) { + const Ogre::Matrix4 wInv = world.inverse(); + const Ogre::Vector3 faceW = -camDir; + const Ogre::Vector3 o = wInv * Ogre::Vector3::ZERO; + Ogre::Vector3 faceL = (wInv * faceW) - o; + if (!faceL.isZeroLength()) { + faceL.normalise(); + out.faceDirLocal = {faceL.x, faceL.y, faceL.z}; + out.faceDirValid = true; + } + } + } + if (rr.depth.isNull() || !lr.ok || lr.points.empty()) return out; + out.confidence = lr.confidence; +#ifndef NDEBUG + if (const char* dp = std::getenv("QTMESH_FACERIG_DUMP_LANDMARKS")) { + QImage vis = rr.depth.convertToFormat(QImage::Format_RGB888); + for (const auto& p : lr.points) { + const int px = int(p[0]), py = int(p[1]); + for (int dy = -1; dy <= 1; ++dy) + for (int dx = -1; dx <= 1; ++dx) { + const int x = px+dx, y = py+dy; + if (x >= 0 && y >= 0 && x < vis.width() && y < vis.height()) + vis.setPixel(x, y, qRgb(0, 255, 0)); + } + } + vis.save(QString::fromUtf8(dp)); + } +#endif + + // 3) build world-space head triangles (local verts × node world transform). + // The render framed the WORLD bounding box, so rays are in world space; + // we intersect world triangles and transform the hit back to LOCAL (the + // frame the fit uses). `world` was computed above for the focus box. + const Ogre::Matrix4 worldInv = world.inverse(); + const int nv = int(localV.size() / 3); + std::vector wv(size_t(nv), Ogre::Vector3::ZERO); + for (int i = 0; i < nv; ++i) + wv[size_t(i)] = world * Ogre::Vector3(localV[size_t(i)*3], + localV[size_t(i)*3+1], + localV[size_t(i)*3+2]); + + // Inverse of (proj * view) to unproject pixels into world rays. + const Ogre::Matrix4 vp = rr.projMatrix * rr.viewMatrix; + const Ogre::Matrix4 vpInv = vp.inverse(); + const float W = float(rr.depth.width()), H = float(rr.depth.height()); + + const int nl = int(lr.points.size()); + out.points.assign(size_t(nl), {0, 0, 0}); + out.valid.assign(size_t(nl), 0); + + for (int i = 0; i < nl; ++i) { + // pixel → NDC ([-1,1], y up). Landmarks are in image pixels (y down). + const float ndcX = (lr.points[size_t(i)][0] / W) * 2.0f - 1.0f; + const float ndcY = 1.0f - (lr.points[size_t(i)][1] / H) * 2.0f; + // unproject near (z=-1) and far (z=1) → world ray. + const Ogre::Vector3 nearW = vpInv * Ogre::Vector3(ndcX, ndcY, -1.0f); + const Ogre::Vector3 farW = vpInv * Ogre::Vector3(ndcX, ndcY, 1.0f); + const Ogre::Vector3 o = nearW; + Ogre::Vector3 dir = farW - nearW; + if (dir.isZeroLength()) continue; + dir.normalise(); + + // nearest triangle hit along the ray. + float bestT = std::numeric_limits::max(); + Ogre::Vector3 hit; + bool found = false; + for (size_t f = 0; f + 2 < localF.size(); f += 3) { + const int ia = localF[f], ib = localF[f+1], ic = localF[f+2]; + if (ia < 0 || ib < 0 || ic < 0 || ia >= nv || ib >= nv || ic >= nv) + continue; + float t; + if (rayTri(o, dir, wv[size_t(ia)], wv[size_t(ib)], wv[size_t(ic)], t) + && t < bestT) { + bestT = t; hit = o + dir * t; found = true; + } + } + if (!found) continue; + const Ogre::Vector3 local = worldInv * hit; // back to mesh-local + out.points[size_t(i)] = {local.x, local.y, local.z}; + out.valid[size_t(i)] = 1; + } + + // ok only if we anchored a useful number of landmarks. + int good = 0; + for (char v : out.valid) good += v ? 1 : 0; + out.ok = good >= 20; // need a meaningful anchor set + return out; +} + +namespace { + +// Build a throwaway Ogre entity for the template neutral so we can render + +// detect its landmarks. Cached by vertex/face count so repeated rigs in one +// session reuse it (the template is fixed). Returns nullptr on failure. +Ogre::Entity* templateEntity(const std::vector& v, + const std::vector& f) +{ + auto* mgr = Manager::getSingletonPtr(); + if (!mgr) return nullptr; + Ogre::SceneManager* sm = mgr->getSceneMgr(); + if (!sm) return nullptr; + + static Ogre::Entity* cached = nullptr; + static size_t cachedKey = 0; + const size_t key = v.size() * 1000003u + f.size(); + if (cached && cachedKey == key) return cached; + + const std::string meshName = "QtMeshFaceRigTemplateMesh"; + const std::string entName = "QtMeshFaceRigTemplateEnt"; + auto& mm = Ogre::MeshManager::getSingleton(); + if (sm->hasEntity(entName)) sm->destroyEntity(entName); + if (mm.resourceExists(meshName)) mm.remove(meshName); + + Ogre::MeshPtr mesh = mm.createManual( + meshName, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + auto* sub = mesh->createSubMesh(); + sub->useSharedVertices = false; + const int vc = int(v.size() / 3); + sub->vertexData = new Ogre::VertexData(); + sub->vertexData->vertexCount = size_t(vc); + auto* decl = sub->vertexData->vertexDeclaration; + decl->addElement(0, 0, Ogre::VET_FLOAT3, Ogre::VES_POSITION); + decl->addElement(0, sizeof(float) * 3, Ogre::VET_FLOAT3, Ogre::VES_NORMAL); + + // Smooth vertex normals (area-weighted face-normal accumulation). Without + // a NORMAL attribute the render pipeline samples undefined per-vertex data + // and the shaded render degrades to per-triangle noise MediaPipe can't + // detect a face in. + std::vector normals(v.size(), 0.0f); + for (size_t i = 0; i + 2 < f.size(); i += 3) { + const int a = f[i], b = f[i+1], c = f[i+2]; + if (a < 0 || b < 0 || c < 0 || a >= vc || b >= vc || c >= vc) continue; + const Ogre::Vector3 pa(v[size_t(a)*3], v[size_t(a)*3+1], v[size_t(a)*3+2]); + const Ogre::Vector3 pb(v[size_t(b)*3], v[size_t(b)*3+1], v[size_t(b)*3+2]); + const Ogre::Vector3 pc(v[size_t(c)*3], v[size_t(c)*3+1], v[size_t(c)*3+2]); + const Ogre::Vector3 n = (pb - pa).crossProduct(pc - pa); // area-weighted + for (int k : {a, b, c}) { + normals[size_t(k)*3+0] += n.x; + normals[size_t(k)*3+1] += n.y; + normals[size_t(k)*3+2] += n.z; + } + } + for (int i = 0; i < vc; ++i) { + Ogre::Vector3 n(normals[size_t(i)*3], normals[size_t(i)*3+1], + normals[size_t(i)*3+2]); + if (n.isZeroLength()) n = Ogre::Vector3::UNIT_Z; + n.normalise(); + normals[size_t(i)*3+0] = n.x; + normals[size_t(i)*3+1] = n.y; + normals[size_t(i)*3+2] = n.z; + } + + std::vector interleaved(size_t(vc) * 6); + for (int i = 0; i < vc; ++i) { + interleaved[size_t(i)*6+0] = v[size_t(i)*3+0]; + interleaved[size_t(i)*6+1] = v[size_t(i)*3+1]; + interleaved[size_t(i)*6+2] = v[size_t(i)*3+2]; + interleaved[size_t(i)*6+3] = normals[size_t(i)*3+0]; + interleaved[size_t(i)*6+4] = normals[size_t(i)*3+1]; + interleaved[size_t(i)*6+5] = normals[size_t(i)*3+2]; + } + auto vbuf = Ogre::HardwareBufferManager::getSingleton().createVertexBuffer( + decl->getVertexSize(0), vc, Ogre::HardwareBuffer::HBU_STATIC_WRITE_ONLY); + vbuf->writeData(0, interleaved.size() * sizeof(float), interleaved.data()); + sub->vertexData->vertexBufferBinding->setBinding(0, vbuf); + + const bool use32 = vc > 65535; + auto ibuf = Ogre::HardwareBufferManager::getSingleton().createIndexBuffer( + use32 ? Ogre::HardwareIndexBuffer::IT_32BIT + : Ogre::HardwareIndexBuffer::IT_16BIT, + f.size(), Ogre::HardwareBuffer::HBU_STATIC_WRITE_ONLY); + if (use32) { + std::vector i32(f.begin(), f.end()); + ibuf->writeData(0, i32.size() * sizeof(uint32_t), i32.data()); + } else { + std::vector i16(f.size()); + for (size_t i = 0; i < f.size(); ++i) i16[i] = uint16_t(f[i]); + ibuf->writeData(0, i16.size() * sizeof(uint16_t), i16.data()); + } + sub->indexData->indexBuffer = ibuf; + sub->indexData->indexCount = f.size(); + + Ogre::Vector3 mn(1e30f, 1e30f, 1e30f), mx(-1e30f, -1e30f, -1e30f); + for (int i = 0; i < vc; ++i) { + Ogre::Vector3 p(v[size_t(i)*3], v[size_t(i)*3+1], v[size_t(i)*3+2]); + mn.makeFloor(p); mx.makeCeil(p); + } + mesh->_setBounds(Ogre::AxisAlignedBox(mn, mx)); + mesh->_setBoundingSphereRadius(0.5f * (mx - mn).length()); + mesh->load(); + + Ogre::Entity* ent = sm->createEntity(entName, meshName); + // Parent to a detached node so it's in the scene graph for world transform + // but hidden (renderShadedView hides all other entities anyway; this one is + // the render target when we pass it in). + Ogre::SceneNode* node = sm->getRootSceneNode()->createChildSceneNode(); + node->attachObject(ent); + node->setVisible(false); // only shown during its own render pass + + cached = ent; + cachedKey = key; + return ent; +} + +} // namespace + +namespace { +double constellationResidual(const std::vector>& C, + const std::vector>& U); +} // namespace + +std::vector buildLandmarkAnchors( + Ogre::Entity* userEntity, + const std::vector& userLocalV, + const std::vector& userLocalF, + const ArkitTemplate& tmpl) +{ + std::vector anchors; + if (!userEntity || !tmpl.valid()) return anchors; + if (!FaceLandmarkDetector::backendAvailable()) return anchors; + // Cheap pre-check: no model → skip the renders entirely. + { FaceLandmarkDetector probe; if (!probe.load()) return anchors; } + + // template side: build a temp entity, detect, map each landmark → nearest + // template vertex. + Ogre::Entity* tent = templateEntity(tmpl.neutral(), tmpl.faces()); + if (!tent) return anchors; + // render pass needs it visible; renderShadedView hides OTHERS, so show it. + if (auto* tn = tent->getParentSceneNode()) tn->setVisible(true); + const MeshLandmarks tlm = detectMeshLandmarks(tent, tmpl.neutral(), + std::vector(tmpl.faces())); + if (auto* tn = tent->getParentSceneNode()) tn->setVisible(false); + if (!tlm.ok) return anchors; + + // user side. + const MeshLandmarks ulm = detectMeshLandmarks(userEntity, userLocalV, userLocalF); + if (!ulm.ok) return anchors; + + const int nl = int(std::min(tlm.points.size(), ulm.points.size())); + const int tvc = tmpl.vertexCount(); + const auto& tn = tmpl.neutral(); + for (int i = 0; i < nl; ++i) { + if (!tlm.valid[size_t(i)] || !ulm.valid[size_t(i)]) continue; + // nearest template vertex to the template landmark point. + const auto& tp = tlm.points[size_t(i)]; + int best = -1; float bestD = std::numeric_limits::max(); + for (int vtx = 0; vtx < tvc; ++vtx) { + const float dx = tn[size_t(vtx)*3] - tp[0]; + const float dy = tn[size_t(vtx)*3+1] - tp[1]; + const float dz = tn[size_t(vtx)*3+2] - tp[2]; + const float d = dx*dx + dy*dy + dz*dz; + if (d < bestD) { bestD = d; best = vtx; } + } + if (best < 0) continue; + anchors.push_back({best, ulm.points[size_t(i)]}); + } + + // Gate on constellation consistency: MediaPipe returns a scattered garbage + // blob on cartoon/stylized faces, and anchoring the fit to garbage is worse + // than fitting unanchored. Compare the template-vs-user landmark layouts + // under a similarity — a real detection agrees, garbage doesn't. + { + std::vector> C, U; + C.reserve(anchors.size()); U.reserve(anchors.size()); + for (const auto& a : anchors) { + C.push_back({tn[size_t(a.tmplVertex)*3], + tn[size_t(a.tmplVertex)*3+1], + tn[size_t(a.tmplVertex)*3+2]}); + U.push_back(a.target); + } + const double resid = constellationResidual(C, U); + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] landmark anchors: tmpl.ok=%d " + "user.ok=%d anchors=%zu residual=%.3f\n", + tlm.ok, ulm.ok, anchors.size(), resid); + if (resid >= 0.15) anchors.clear(); // garbage → fit unanchored + } + return anchors; +} + +// Canonical MediaPipe FaceMesh indices for the few anatomical anchors the fit +// needs. IMPORTANT side semantics: MediaPipe names sides in IMAGE space, which +// is MIRRORED for a camera-facing subject — MP "left" indices (33/133/61/105) +// are the CHARACTER'S RIGHT (measured on the template: they sit at +X = +// character-right). Labels here are in CHARACTER space (what a user placing +// markers on a model naturally means by left/right). +int canonicalTemplateVertex(int mpIndex) +{ + // ICT-FaceKit topology (26,719 verts). Derived offline from the packed + // template's own blendshape deltas + midline geometry; side assignment + // matches the catalog's measured detector convention (anchorsFromMarkers' + // mirror check absorbs a global L/R flip regardless). + switch (mpIndex) { + case 1: return 4841; // nose tip (front-most midline) + case 152: return 961; // chin / gnathion (lowest front midline) + case 13: return 5829; // upper lip (mouthUpperUp peak, midline) + case 14: return 5945; // lower lip (mouthLowerDown peak, midline) + case 10: return 2138; // forehead (midline above brows) + case 33: return 2798; // eye outer corner (+X lid extreme) + case 133: return 3585; // eye inner corner (+X) + case 61: return 6156; // mouth corner (+X, mouthSmile peak) + case 105: return 2590; // brow (+X, browOuterUp peak) + case 263: return 557; // eye outer corner (-X) + case 362: return 1370; // eye inner corner (-X) + case 291: return 5651; // mouth corner (-X) + case 334: return 349; // brow (-X) + default: return -1; + } +} + +const std::vector>& faceMarkerCatalog() +{ + static const std::vector> kCatalog = { + {QStringLiteral("Nose tip"), 1}, + {QStringLiteral("Chin"), 152}, + {QStringLiteral("Right eye outer"), 33}, + {QStringLiteral("Left eye outer"), 263}, + {QStringLiteral("Right eye inner"), 133}, + {QStringLiteral("Left eye inner"), 362}, + {QStringLiteral("Right mouth corner"), 61}, + {QStringLiteral("Left mouth corner"), 291}, + {QStringLiteral("Upper lip"), 13}, + {QStringLiteral("Lower lip"), 14}, + {QStringLiteral("Right brow"), 105}, + {QStringLiteral("Left brow"), 334}, + {QStringLiteral("Forehead"), 10}, + }; + return kCatalog; +} + +namespace { +// Side-pair / midline structure of the catalog (MediaPipe indices). Used to +// SYMMETRIZE the template-side anchors (the template is x-symmetric with the +// midline at x=0, but MediaPipe drifts on the untextured template render — +// measured: "nose tip" detected 4 units off the midline) and to auto-correct +// a user who placed markers with the opposite left/right convention. +constexpr int kMidlineIdx[] = {1, 152, 13, 14, 10}; +constexpr int kPairIdx[][2] = {{33, 263}, {133, 362}, {61, 291}, {105, 334}}; + +bool isMidline(int mpIdx) +{ + for (int m : kMidlineIdx) if (m == mpIdx) return true; + return false; +} +int pairOf(int mpIdx) +{ + for (const auto& p : kPairIdx) { + if (p[0] == mpIdx) return p[1]; + if (p[1] == mpIdx) return p[0]; + } + return -1; +} +} // namespace + +namespace { +int nearestTemplateVertex(const std::vector& tn, int tvc, + const std::array& p) +{ + int best = -1; float bestD = std::numeric_limits::max(); + for (int vtx = 0; vtx < tvc; ++vtx) { + const float dx = tn[size_t(vtx)*3] - p[0]; + const float dy = tn[size_t(vtx)*3+1] - p[1]; + const float dz = tn[size_t(vtx)*3+2] - p[2]; + const float d = dx*dx + dy*dy + dz*dz; + if (d < bestD) { bestD = d; best = vtx; } + } + return best; +} + +// Does the user point constellation actually LOOK like the template's? Align +// C onto U with a similarity (centroid + RMS scale, no rotation — both faces +// upright/front by contract) and measure the mean residual normalised by U's +// spread. A real face layout agrees (≲0.2); a garbage detection (MediaPipe on +// a cartoon face returns a scattered blob) does not. This is the gate that +// keeps garbage landmarks from silently poisoning the warp/fit. +double constellationResidual(const std::vector>& C, + const std::vector>& U) +{ + const int n = int(std::min(C.size(), U.size())); + if (n < 4) return 1e9; + std::array cC{0,0,0}, cU{0,0,0}; + for (int i = 0; i < n; ++i) + for (int d = 0; d < 3; ++d) { + cC[size_t(d)] += C[size_t(i)][size_t(d)] / n; + cU[size_t(d)] += U[size_t(i)][size_t(d)] / n; + } + double sC = 0, sU = 0; + for (int i = 0; i < n; ++i) { + double dc = 0, du = 0; + for (int d = 0; d < 3; ++d) { + const double a = C[size_t(i)][size_t(d)] - cC[size_t(d)]; + const double b = U[size_t(i)][size_t(d)] - cU[size_t(d)]; + dc += a*a; du += b*b; + } + sC += std::sqrt(dc); sU += std::sqrt(du); + } + if (sC < 1e-12 || sU < 1e-12) return 1e9; + const double scale = sU / sC; + double resid = 0; + for (int i = 0; i < n; ++i) { + double d2 = 0; + for (int d = 0; d < 3; ++d) { + const double m = (C[size_t(i)][size_t(d)] - cC[size_t(d)]) * scale + + cU[size_t(d)]; + const double e = U[size_t(i)][size_t(d)] - m; + d2 += e*e; + } + resid += std::sqrt(d2); + } + resid /= n; + return resid / (sU / n); // normalise by mean spread of U +} +} // namespace + +std::vector seedFaceMarkers( + Ogre::Entity* userEntity, + const std::vector& userLocalV, + const std::vector& userLocalF, + const ArkitTemplate& tmpl, + bool* outConfident) +{ + std::vector markers; + if (outConfident) *outConfident = false; + if (!userEntity || !tmpl.valid()) return markers; + + const auto& cat = faceMarkerCatalog(); + markers.reserve(cat.size()); + for (const auto& [label, idx] : cat) { + FaceMarker m; m.label = label; m.mediapipeIndex = idx; + markers.push_back(std::move(m)); + } + + if (!FaceLandmarkDetector::backendAvailable()) return markers; + { FaceLandmarkDetector probe; if (!probe.load()) return markers; } + + const int tvc = tmpl.vertexCount(); + const auto& tn = tmpl.neutral(); + + // Template-side marker vertices: CANONICAL constants for the ICT topology, + // derived offline from the template's own blendshape deltas (the + // mouthSmile peak IS the mouth corner, the eyeBlink-moved lid's lateral + // extremes ARE the eye corners, gnathion = lowest front midline vertex). + // Detection on the template's untextured render carries a systematic + // detector bias (lower-face landmarks drift UP one anatomical step), and + // that bias used to define the "ground truth" every user marker was + // matched against. Falls back to template detection for a non-ICT + // template (vertex count mismatch). + const bool canonicalOk = (tvc == 26719); + if (canonicalOk) { + for (auto& m : markers) + m.tmplVertex = canonicalTemplateVertex(m.mediapipeIndex); + } + // Template detection still runs even with canonical vertices: the + // template is rendered + detected EXACTLY like the user mesh, so the + // difference between where the detector puts a marker on the template and + // its canonical vertex measures the detector's systematic bias on this + // render style (lower-face landmarks drift up one anatomical step on + // untextured statues) — which we then subtract from the user detections. + MeshLandmarks tlm; + { + Ogre::Entity* tent = templateEntity(tmpl.neutral(), tmpl.faces()); + if (!tent) return markers; + if (auto* tnode = tent->getParentSceneNode()) tnode->setVisible(true); + tlm = detectMeshLandmarks(tent, tmpl.neutral(), + std::vector(tmpl.faces())); + if (auto* tnode = tent->getParentSceneNode()) tnode->setVisible(false); + } + std::vector> tlmSym(markers.size(), {0,0,0}); + std::vector tlmSymOk(markers.size(), 0); + if (tlm.ok) { + // SYMMETRIZE the detected template landmarks before resolving vertices: + // the ICT template is x-symmetric (midline at x=0), but MediaPipe + // drifts on the untextured template render (measured: nose tip 4 units + // off-midline), which mis-anchors EVERYTHING downstream. Midline + // features snap to x=0; side pairs get mirrored positions with the + // pair-mean height/depth and mean |x|. Detection still supplies the + // vertical/depth placement it gets roughly right. + auto detected = [&](int mpIdx, std::array& out) -> bool { + if (mpIdx < 0 || mpIdx >= int(tlm.points.size()) + || !tlm.valid[size_t(mpIdx)]) return false; + out = tlm.points[size_t(mpIdx)]; + return true; + }; + for (size_t k = 0; k < markers.size(); ++k) { + auto& m = markers[k]; + const int i = m.mediapipeIndex; + std::array p; + if (!detected(i, p)) continue; + if (isMidline(i)) { + p[0] = 0.0f; + } else if (const int j = pairOf(i); j >= 0) { + std::array q; + if (detected(j, q)) { + const float xm = 0.5f * (std::fabs(p[0]) + std::fabs(q[0])); + const float ym = 0.5f * (p[1] + q[1]); + const float zm = 0.5f * (p[2] + q[2]); + // keep this marker on the side detection put it (ties → + // MP-left-named index goes to +X = character-right, + // the measured convention on this template). + float sign = p[0] > q[0] ? 1.0f : (p[0] < q[0] ? -1.0f + : ((i == 33 || i == 133 || i == 61 || i == 105) ? 1.0f + : -1.0f)); + p = { sign * xm, ym, zm }; + } // single-sided detection: use as-is + } + tlmSym[k] = p; + tlmSymOk[k] = 1; + if (!canonicalOk) + m.tmplVertex = nearestTemplateVertex(tn, tvc, p); + } + } + + // User detection seeds the editable positions — but ONLY when the detected + // constellation actually looks like a face layout. MediaPipe returns a + // scattered garbage blob on cartoon/stylized faces, and garbage seeds + // silently poison the warp/fit (measured: jawOpen deltas 50x too small). + const MeshLandmarks ulm = detectMeshLandmarks(userEntity, userLocalV, userLocalF); + + // Detector-bias correction: transfer each marker's measured template-side + // bias (symmetrized detection − canonical vertex) into the user's frame + // (scaled by the template→user constellation size ratio) and subtract it + // from the user detection. Template and user go through the same render + + // detector, so the systematic statue bias cancels; what remains is the + // user's actual anatomy. + std::vector> uCorr(markers.size(), {0,0,0}); + std::vector uOk(markers.size(), 0); + if (ulm.ok) { + // constellation size ratio from raw detected pairs (bias-consistent + // on both sides, so the ratio is unaffected by the bias itself) + double sT = 0, sUsr = 0; + { + std::array cT{0,0,0}, cUsr{0,0,0}; + int n = 0; + for (size_t k = 0; k < markers.size(); ++k) { + const int i = markers[k].mediapipeIndex; + if (!tlmSymOk[k] || i < 0 || i >= int(ulm.points.size()) + || !ulm.valid[size_t(i)]) continue; + for (int d = 0; d < 3; ++d) { + cT[size_t(d)] += tlmSym[k][size_t(d)]; + cUsr[size_t(d)] += ulm.points[size_t(i)][size_t(d)]; + } + ++n; + } + if (n >= 4) { + for (int d = 0; d < 3; ++d) { cT[size_t(d)] /= n; cUsr[size_t(d)] /= n; } + for (size_t k = 0; k < markers.size(); ++k) { + const int i = markers[k].mediapipeIndex; + if (!tlmSymOk[k] || i < 0 || i >= int(ulm.points.size()) + || !ulm.valid[size_t(i)]) continue; + double dt = 0, du = 0; + for (int d = 0; d < 3; ++d) { + const double a = tlmSym[k][size_t(d)] - cT[size_t(d)]; + const double b = ulm.points[size_t(i)][size_t(d)] - cUsr[size_t(d)]; + dt += a*a; du += b*b; + } + sT += std::sqrt(dt); sUsr += std::sqrt(du); + } + } + } + const double ratio = (canonicalOk && tlm.ok && sT > 1e-9) + ? sUsr / sT : 0.0; + for (size_t k = 0; k < markers.size(); ++k) { + const int i = markers[k].mediapipeIndex; + if (i < 0 || i >= int(ulm.points.size()) || !ulm.valid[size_t(i)]) + continue; + uCorr[k] = ulm.points[size_t(i)]; + uOk[k] = 1; + const int cv = markers[k].tmplVertex; + if (ratio > 0.0 && tlmSymOk[k] && cv >= 0) { + for (int d = 0; d < 3; ++d) { + const float bias = tlmSym[k][size_t(d)] + - tn[size_t(cv)*3 + size_t(d)]; + uCorr[k][size_t(d)] -= float(ratio * bias); + } + } + } + } + + int seeded = 0; + std::vector> detC, detU; // template/user pairs + for (size_t k = 0; k < markers.size(); ++k) { + if (!uOk[k] || markers[k].tmplVertex < 0) continue; + ++seeded; + const int cv = markers[k].tmplVertex; + detC.push_back({tn[size_t(cv)*3], tn[size_t(cv)*3+1], + tn[size_t(cv)*3+2]}); + detU.push_back(uCorr[k]); + } + // STRICT gate (0.12): the GUI render differs from headless (skybox, + // lighting), so MediaPipe's garbage varies run-to-run and looser gates let + // some of it through (field-reproduced: a "trusted" garbage constellation + // crushed every shape to 0.06% amplitude). Detection must look UNAMBIGUOUSLY + // like a face layout to be trusted; otherwise the proportional defaults are + // measurably good and the user refines from there. + const double resid = constellationResidual(detC, detU); + const bool confident = ulm.ok && seeded >= int(markers.size()) * 3 / 4 + && resid < 0.12; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] seed: ulm.ok=%d conf=%.2f %d/%zu " + "detected, constellation residual %.3f -> %s\n", + ulm.ok, ulm.confidence, seeded, markers.size(), resid, + confident ? "trusted" : "using proportional defaults"); + + if (confident) { + // Consensus outlier correction: fit the similarity model the + // constellation gate already uses (centroid + scale, no rotation) + // and predict each marker from the TEMPLATE layout. Individual + // detections on low-contrast renders drift most at the eye/mouth + // corners (measured up to 0.59 units on the reference vs ~0.07 + // for the nose); a detection that deviates from the consensus + // prediction by more than 2x the median is detector noise — replace + // it with the prediction, snapped to the head surface. + std::array cC{0,0,0}, cU{0,0,0}; + const int np = int(detC.size()); + for (int i = 0; i < np; ++i) + for (int d = 0; d < 3; ++d) { + cC[size_t(d)] += detC[size_t(i)][size_t(d)] / np; + cU[size_t(d)] += detU[size_t(i)][size_t(d)] / np; + } + double sC = 0, sU = 0; + for (int i = 0; i < np; ++i) { + double dc = 0, du = 0; + for (int d = 0; d < 3; ++d) { + const double a = detC[size_t(i)][size_t(d)] - cC[size_t(d)]; + const double b = detU[size_t(i)][size_t(d)] - cU[size_t(d)]; + dc += a*a; du += b*b; + } + sC += std::sqrt(dc); sU += std::sqrt(du); + } + const double scale = (sC > 1e-12) ? sU / sC : 1.0; + + auto predictOf = [&](int tmplVertex) -> std::array { + std::array p; + for (int d = 0; d < 3; ++d) + p[size_t(d)] = float((double(tn[size_t(tmplVertex)*3 + d]) + - cC[size_t(d)]) * scale + cU[size_t(d)]); + return p; + }; + std::vector devs; + for (size_t k = 0; k < markers.size(); ++k) { + const auto& m = markers[k]; + if (!uOk[k] || m.tmplVertex < 0) continue; + const auto pred = predictOf(m.tmplVertex); + const auto& det = uCorr[k]; + const double dx = det[0]-pred[0], dy = det[1]-pred[1], + dz = det[2]-pred[2]; + devs.push_back(std::sqrt(dx*dx + dy*dy + dz*dz)); + } + std::vector sorted = devs; + std::sort(sorted.begin(), sorted.end()); + const double median = sorted.empty() ? 0.0 + : sorted[sorted.size() / 2]; + const double outlierAt = std::max(2.0 * median, 1e-9); + + size_t di = 0; + for (size_t k = 0; k < markers.size(); ++k) { + auto& m = markers[k]; + if (!uOk[k]) continue; + if (m.tmplVertex >= 0 && di < devs.size() + && devs[di] > outlierAt) { + // consensus prediction, snapped onto the head surface + std::array p = predictOf(m.tmplVertex); + float best = 1e30f; + std::array snap = p; + for (size_t v = 0; v + 2 < userLocalV.size(); v += 3) { + const float dx = userLocalV[v] - p[0]; + const float dy = userLocalV[v+1] - p[1]; + const float dz = userLocalV[v+2] - p[2]; + const float d2 = dx*dx + dy*dy + dz*dz; + if (d2 < best) { + best = d2; + snap = {userLocalV[v], userLocalV[v+1], userLocalV[v+2]}; + } + } + m.userPos = snap; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] seed outlier '%s' " + "dev=%.3f (median %.3f) -> consensus\n", + m.label.toUtf8().constData(), devs[di], median); + } else { + m.userPos = uCorr[k]; + } + if (m.tmplVertex >= 0) ++di; + m.placed = true; + } + } else { + // Garbage / weak detection: seed EVERY marker at the head-box-projected + // template position instead. Those proportional defaults measurably + // produce a good rig on their own (the cartoon-face path), and the user + // refines from there. placed=true so they act as anchors even if the + // user rigs without touching them. + // + // FACING-AWARE: the straight box mapping assumes the user's face + // points the template's way (+Z); a backwards-facing import (glb + // round-trips flip facing) landed every default on the BACK of the + // head. Even a too-weak-to-trust detection still tells us which + // render view scored best — yaw the template coordinates to that + // cardinal facing before the box mapping. + int yaw = 0; // 0:+Z (template) 1:-Z 2:+X 3:-X + if (ulm.faceDirValid) { + const float fx = ulm.faceDirLocal[0], fz = ulm.faceDirLocal[2]; + yaw = (std::fabs(fz) >= std::fabs(fx)) ? (fz >= 0.f ? 0 : 1) + : (fx >= 0.f ? 2 : 3); + } + auto yawRot = [yaw](std::array p) -> std::array { + switch (yaw) { + case 1: return {-p[0], p[1], -p[2]}; // 180° + case 2: return { p[2], p[1], -p[0]}; // +Z → +X + case 3: return {-p[2], p[1], p[0]}; // +Z → -X + default: return p; + } + }; + if (std::getenv("QTMESH_FACERIG_DEBUG")) + std::fprintf(stderr, "[facerig] defaults: faceDirValid=%d " + "dir=(%.2f,%.2f,%.2f) yaw=%d\n", + ulm.faceDirValid ? 1 : 0, ulm.faceDirLocal[0], + ulm.faceDirLocal[1], ulm.faceDirLocal[2], yaw); + std::array lo{1e30f,1e30f,1e30f}, hi{-1e30f,-1e30f,-1e30f}; + const int unv = int(userLocalV.size()/3); + for (int i = 0; i < unv; ++i) + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], userLocalV[size_t(i)*3+a]); + hi[a] = std::max(hi[a], userLocalV[size_t(i)*3+a]); + } + std::array tlo{1e30f,1e30f,1e30f}, thi{-1e30f,-1e30f,-1e30f}; + for (int i = 0; i < tvc; ++i) { + const std::array tv = yawRot({tn[size_t(i)*3], + tn[size_t(i)*3+1], + tn[size_t(i)*3+2]}); + for (int a = 0; a < 3; ++a) { + tlo[a] = std::min(tlo[a], tv[size_t(a)]); + thi[a] = std::max(thi[a], tv[size_t(a)]); + } + } + // Depth axis + ray setup for surface snapping: the box mapping puts + // markers at the box's proportional DEPTH, but protrusions (a cigar, + // a long nose, hair) inflate the head box along the facing axis and + // every default then floats off the face. Ray-cast each marker from + // outside the box along the facing direction and take the first + // surface hit as its depth instead. + const int depthAxis = (yaw <= 1) ? 2 : 0; + const float depthSign = (yaw == 0 || yaw == 2) ? 1.0f : -1.0f; + const float margin = 0.25f * (hi[size_t(depthAxis)] - lo[size_t(depthAxis)]); + Ogre::Vector3 rayDir = Ogre::Vector3::ZERO; + rayDir[depthAxis] = -depthSign; // from the face side into the head + const int unvTot = int(userLocalV.size() / 3); + + for (auto& m : markers) { + if (m.tmplVertex < 0) continue; + const std::array tv = yawRot({tn[size_t(m.tmplVertex)*3], + tn[size_t(m.tmplVertex)*3+1], + tn[size_t(m.tmplVertex)*3+2]}); + for (int a = 0; a < 3; ++a) { + const float f = (thi[a]-tlo[a]) > 1e-6f + ? (tv[size_t(a)] - tlo[a]) / (thi[a]-tlo[a]) : 0.5f; + m.userPos[size_t(a)] = lo[a] + f * (hi[a]-lo[a]); + } + // Snap to the head surface along the facing axis. + Ogre::Vector3 o(m.userPos[0], m.userPos[1], m.userPos[2]); + o[depthAxis] = depthSign > 0 + ? hi[size_t(depthAxis)] + margin + : lo[size_t(depthAxis)] - margin; + float bestT = std::numeric_limits::max(); + bool hitAny = false; + Ogre::Vector3 hit; + for (size_t fI = 0; fI + 2 < userLocalF.size(); fI += 3) { + const int ia = userLocalF[fI], ib = userLocalF[fI+1], + ic = userLocalF[fI+2]; + if (ia < 0 || ib < 0 || ic < 0 + || ia >= unvTot || ib >= unvTot || ic >= unvTot) continue; + auto vAt = [&](int k) { + return Ogre::Vector3(userLocalV[size_t(k)*3], + userLocalV[size_t(k)*3+1], + userLocalV[size_t(k)*3+2]); + }; + float t; + if (rayTri(o, rayDir, vAt(ia), vAt(ib), vAt(ic), t) && t < bestT) { + bestT = t; hit = o + rayDir * t; hitAny = true; + } + } + if (hitAny) + m.userPos = {hit.x, hit.y, hit.z}; + m.placed = true; + } + } + + if (outConfident) *outConfident = confident; + return markers; +} + +std::vector anchorsFromMarkers(const std::vector& markers, + const ArkitTemplate& tmpl) +{ + // Two candidate pairings: as placed, and with the left/right PAIR targets + // swapped — a user may reasonably use either the character's or the + // screen's left/right. Score both against the template constellation and + // keep the one that agrees; a mirrored anchor set would ask the warp to + // fold the template through itself and crush every shape. + auto build = [&](bool swapped) { + std::vector anchors; + for (const auto& m : markers) { + if (!m.placed || m.tmplVertex < 0) continue; + std::array target = m.userPos; + if (swapped) { + const int j = pairOf(m.mediapipeIndex); + if (j >= 0) { + // use the PAIRED marker's position instead + for (const auto& o : markers) + if (o.mediapipeIndex == j && o.placed) { + target = o.userPos; + break; + } + } + } + anchors.push_back({m.tmplVertex, target}); + } + return anchors; + }; + auto residualOf = [&](const std::vector& anchors) { + std::vector> C, U; + const auto& tn = tmpl.neutral(); + const int tvc = tmpl.vertexCount(); + for (const auto& a : anchors) { + if (a.tmplVertex < 0 || a.tmplVertex >= tvc) continue; + C.push_back({tn[size_t(a.tmplVertex)*3], + tn[size_t(a.tmplVertex)*3+1], + tn[size_t(a.tmplVertex)*3+2]}); + U.push_back(a.target); + } + return constellationResidual(C, U); + }; + + std::vector normal = build(false); + if (!tmpl.valid()) return normal; + std::vector swapped = build(true); + const double rn = residualOf(normal); + const double rs = residualOf(swapped); + if (rs < rn) { + std::fprintf(stderr, "[facerig] markers look MIRRORED (residual %.3f " + "vs %.3f) — auto-swapping left/right pairs\n", rn, rs); + return swapped; + } + return normal; +} + +} // namespace FaceRig diff --git a/src/FaceRig/FaceRigLandmarks.h b/src/FaceRig/FaceRigLandmarks.h new file mode 100644 index 000000000..677e60626 --- /dev/null +++ b/src/FaceRig/FaceRigLandmarks.h @@ -0,0 +1,106 @@ +#ifndef FACERIGLANDMARKS_H +#define FACERIGLANDMARKS_H + +// Landmark acquisition for the face auto-rig (#889): render a head mesh +// front-on, detect the MediaPipe 478 face landmarks (FaceLandmarkDetector), and +// back-project each 2D landmark through the render camera onto the mesh surface +// to get a 3D landmark ON THE MESH. Running this on BOTH the ARKit template and +// the user head, then pairing by MediaPipe index, yields the correspondences +// that anchor NRICP so the fit lands on the real face features. +// +// Ogre-touching (renders via MeshDepthRenderer::renderShadedView) — main thread. + +#include "NonRigidICP.h" // NricpLandmark + +#include + +#include +#include +#include + +namespace Ogre { class Entity; } + +namespace FaceRig { + +class ArkitTemplate; + +// A user-adjustable face marker: a canonical facial point (eye/nose/mouth/…) +// with the TEMPLATE vertex it anchors and the current USER-mesh position (seeded +// from auto-detection, then draggable). label is a short guidance string. +struct FaceMarker { + QString label; + int mediapipeIndex = -1; // canonical MediaPipe FaceMesh index + int tmplVertex = -1; // template vertex it pins (from tmpl detect) + std::array userPos{0, 0, 0}; // user-mesh position (editable) + bool placed = false; // seeded-or-user-set (vs unresolved) +}; + +// The canonical marker set (label + MediaPipe index), fixed order. These are +// the anatomical anchors the fit needs to lock orientation/scale; the user only +// has to get these few roughly right, not all 478. +const std::vector>& faceMarkerCatalog(); + +struct MeshLandmarks { + // 3D landmark positions in MESH-LOCAL space (same frame as the geometry the + // fit uses), one per MediaPipe index; `valid[i]` is false when landmark i + // didn't hit the surface (skipped as a correspondence). + std::vector> points; + std::vector valid; + float confidence = 0.0f; + // Direction the FACE points, in MESH-LOCAL space — derived from the view + // whose render won the presence-logit ranking (the face points toward that + // camera). Valid even when `ok` is false: a weak detection is still a + // usable FACING signal, and the proportional-default marker placement + // needs it so defaults land on the face instead of the back of the head + // for meshes that don't face the template's +Z. + std::array faceDirLocal{0, 0, 1}; + bool faceDirValid = false; + bool ok = false; +}; + +// Render `entity` front-on, detect face landmarks, back-project to the surface. +// `localV`/`localF` are the MESH-LOCAL vertices/indices the fit operates on +// (the head sub-mesh in local space) — the ray cast intersects these so the +// returned points live in the same frame as the fit. Returns ok=false when the +// detector is unavailable / no face is found. +MeshLandmarks detectMeshLandmarks(Ogre::Entity* entity, + const std::vector& localV, + const std::vector& localF); + +// Build NRICP landmark anchors (template vertex → user position) by detecting +// face landmarks on BOTH the ARKit template and the user head and pairing them +// by MediaPipe index. `userEntity` is rendered for the user side; `userLocalV/F` +// are the head sub-mesh (local frame) the fit uses, both for the user raycast +// and the frame the returned targets live in. Returns empty when the detector +// is unavailable / either face isn't found (caller then fits without anchors). +std::vector buildLandmarkAnchors( + Ogre::Entity* userEntity, + const std::vector& userLocalV, + const std::vector& userLocalF, + const ArkitTemplate& tmpl); + +// Seed the editable face-marker set: detect on the template (reliable — a real +// human face) to resolve each marker's template vertex, then AUTO-DETECT on the +// user head to seed userPos when it works (marker.placed=true) or fall back to +// the template landmark position mapped into the user's head box when it +// doesn't (cartoon faces — the user then drags to correct). Always returns the +// full catalog with template verts resolved; `outConfident` reports whether the +// user auto-detect looked trustworthy. +std::vector seedFaceMarkers( + Ogre::Entity* userEntity, + const std::vector& userLocalV, + const std::vector& userLocalF, + const ArkitTemplate& tmpl, + bool* outConfident = nullptr); + +// Build NRICP anchors from the (possibly user-edited) markers — one per placed +// marker with a resolved template vertex. Auto-corrects a MIRRORED placement: +// if swapping the left/right pair targets matches the template constellation +// better (the user assumed the opposite left/right convention), the swapped +// pairing is used — so either convention works. +std::vector anchorsFromMarkers(const std::vector& markers, + const ArkitTemplate& tmpl); + +} // namespace FaceRig + +#endif // FACERIGLANDMARKS_H diff --git a/src/FaceRig/FaceRigger.cpp b/src/FaceRig/FaceRigger.cpp new file mode 100644 index 000000000..d6432d457 --- /dev/null +++ b/src/FaceRig/FaceRigger.cpp @@ -0,0 +1,693 @@ +#include "FaceRigger.h" + +#include "ArkitTemplate.h" +#include "DeformationTransfer.h" +#include "NonRigidICP.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace FaceRig { + +namespace { + +// A uniform spatial-hash grid over a point set for nearest-point queries. The +// resample maps every user vertex to its nearest CORRESPONDENCE point (the +// fitted template verts X, template topology). Same map for all 52 shapes, so +// we build it once. Brute force would be Nu*Nt (~350M on a 27k template); +// the grid keeps it near-linear. Dependency-free. +class PointGrid { +public: + void build(const std::vector& pts) + { + m_pts = &pts; + const int n = int(pts.size() / 3); + if (n == 0) return; + for (int a = 0; a < 3; ++a) { m_lo[a] = 1e30f; m_hi[a] = -1e30f; } + for (int i = 0; i < n; ++i) + for (int a = 0; a < 3; ++a) { + m_lo[a] = std::min(m_lo[a], pts[size_t(i)*3+a]); + m_hi[a] = std::max(m_hi[a], pts[size_t(i)*3+a]); + } + // aim ~1 point per cell on average + double vol = 1.0; + for (int a = 0; a < 3; ++a) vol *= std::max(1e-6, double(m_hi[a]-m_lo[a])); + m_cell = float(std::cbrt(vol / std::max(1, n))); + if (m_cell <= 1e-9f) m_cell = 1.0f; + for (int i = 0; i < n; ++i) + m_cells[key(cellOf(&pts[size_t(i)*3]))].push_back(i); + } + + // nearest point index to q (3 floats), searching an expanding shell of + // cells until the nearest is provably found. Returns -1 if empty. + int nearest(const float* q) const + { + if (!m_pts || m_cells.empty()) return -1; + // Clamp the SEARCH ORIGIN into the populated bounds: for a query far + // outside the grid, the shell cap below (grid span) could otherwise + // terminate before any populated cell is reached and silently return + // -1 (dropping that vertex's transferred delta). Distances are still + // measured to the real q, so the nearest result is unchanged. + float qc[3]; + for (int a = 0; a < 3; ++a) + qc[a] = std::min(std::max(q[a], m_lo[a]), m_hi[a]); + const std::array c = cellOf(qc); + // absolute cap on the shell radius = span of the grid in cells + 1, + // guarantees termination even if q is far outside the populated region. + int spanCells = 1; + for (int a = 0; a < 3; ++a) + spanCells = std::max(spanCells, + int(std::ceil((m_hi[a]-m_lo[a]) / m_cell)) + 1); + const int rMax = spanCells + 1; + + int best = -1; + double bestD = std::numeric_limits::max(); + for (int r = 0; r <= rMax; ++r) { + for (int dx = -r; dx <= r; ++dx) + for (int dy = -r; dy <= r; ++dy) + for (int dz = -r; dz <= r; ++dz) { + // only the shell at Chebyshev radius r (interior scanned) + if (std::max({std::abs(dx),std::abs(dy),std::abs(dz)}) != r) continue; + auto it = m_cells.find(key({c[0]+dx, c[1]+dy, c[2]+dz})); + if (it == m_cells.end()) continue; + for (int idx : it->second) { + const float* p = &(*m_pts)[size_t(idx)*3]; + const double d = (double(p[0]-q[0])*(p[0]-q[0]) + + double(p[1]-q[1])*(p[1]-q[1]) + + double(p[2]-q[2])*(p[2]-q[2])); + if (d < bestD) { bestD = d; best = idx; } + } + } + // A point in shell r is at least (r-1)*cell away from q; once the + // best found is closer than the guaranteed reach of the NEXT shell, + // no farther shell can beat it. Scan one extra shell to be safe. + if (best >= 0) { + const double guaranteed = double(r) * m_cell; // min dist of shell r+1 + if (guaranteed * guaranteed >= bestD) return best; + } + } + return best; + } + +private: + std::array cellOf(const float* p) const + { + return {int(std::floor((p[0]-m_lo[0]) / m_cell)), + int(std::floor((p[1]-m_lo[1]) / m_cell)), + int(std::floor((p[2]-m_lo[2]) / m_cell))}; + } + static long long key(const std::array& c) + { + // pack 3 ints into one 64-bit key (21 bits each, offset to positive) + const long long x = (c[0] + (1<<20)) & 0x1FFFFF; + const long long y = (c[1] + (1<<20)) & 0x1FFFFF; + const long long z = (c[2] + (1<<20)) & 0x1FFFFF; + return (x << 42) | (y << 21) | z; + } + + const std::vector* m_pts = nullptr; + float m_lo[3] = {0,0,0}, m_hi[3] = {0,0,0}; + float m_cell = 1.0f; + std::unordered_map> m_cells; +}; + +double bboxDiag(const std::vector& v) +{ + if (v.empty()) return 0.0; + float lo[3] = {1e30f,1e30f,1e30f}, hi[3] = {-1e30f,-1e30f,-1e30f}; + for (size_t i = 0; i + 2 < v.size(); i += 3) + for (int a = 0; a < 3; ++a) { + lo[a] = std::min(lo[a], v[i+a]); + hi[a] = std::max(hi[a], v[i+a]); + } + double s = 0; + for (int a = 0; a < 3; ++a) s += double(hi[a]-lo[a]) * double(hi[a]-lo[a]); + return std::sqrt(s); +} + +} // namespace + +std::vector rbfWarpByAnchors(const std::vector& tmplV, + const std::vector& anchors) +{ + const int nv = int(tmplV.size() / 3); + // Collect valid, de-duplicated centers + targets (two anchors on the same + // template vertex would make the system singular - first one wins). + std::vector cs; + std::vector> C, T; // template center, user target + for (const auto& a : anchors) { + if (a.tmplVertex < 0 || a.tmplVertex >= nv) continue; + bool dup = false; + for (int c : cs) if (c == a.tmplVertex) { dup = true; break; } + if (dup) continue; + cs.push_back(a.tmplVertex); + C.push_back({tmplV[size_t(a.tmplVertex)*3], + tmplV[size_t(a.tmplVertex)*3+1], + tmplV[size_t(a.tmplVertex)*3+2]}); + T.push_back({double(a.target[0]), double(a.target[1]), + double(a.target[2])}); + } + const int N = int(cs.size()); + if (N < 4) return {}; + + // 1) SIMILARITY prealign (centroid + RMS-spread scale, no rotation - both + // faces are upright/front-facing by contract). Face markers are nearly + // COPLANAR, so a full affine/thin-plate warp is ill-conditioned along the + // depth axis and can shear the back of the head into garbage - the + // similarity handles the global part robustly, the Gaussian RBF below only + // carries the local residuals and DECAYS away from the face. + std::array cT{0,0,0}, cU{0,0,0}; + for (int i = 0; i < N; ++i) + for (int d = 0; d < 3; ++d) { + cT[size_t(d)] += C[size_t(i)][size_t(d)] / N; + cU[size_t(d)] += T[size_t(i)][size_t(d)] / N; + } + double sT = 0, sU = 0; + for (int i = 0; i < N; ++i) { + double dt = 0, du = 0; + for (int d = 0; d < 3; ++d) { + const double a = C[size_t(i)][size_t(d)] - cT[size_t(d)]; + const double b = T[size_t(i)][size_t(d)] - cU[size_t(d)]; + dt += a*a; du += b*b; + } + sT += std::sqrt(dt); sU += std::sqrt(du); + } + if (sT < 1e-9) return {}; + const double scale = (sU > 1e-9) ? sU / sT : 1.0; + auto prealign = [&](const std::array& p) { + std::array q; + for (int d = 0; d < 3; ++d) + q[size_t(d)] = (p[size_t(d)] - cT[size_t(d)]) * scale + cU[size_t(d)]; + return q; + }; + + // Prealigned centers + residual displacements the RBF must carry. + std::vector> Cp(size_t(N), {0,0,0}); + std::vector> R(size_t(N), {0,0,0}); + for (int i = 0; i < N; ++i) { + Cp[size_t(i)] = prealign(C[size_t(i)]); + for (int d = 0; d < 3; ++d) + R[size_t(i)][size_t(d)] = + T[size_t(i)][size_t(d)] - Cp[size_t(i)][size_t(d)]; + } + + // 2) Gaussian RBF on the residuals, ridge-regularized. sigma = mean + // nearest-neighbour center spacing (x1.5) so influence blobs overlap + // smoothly; far from the face the displacement decays to the similarity. + double sigma = 0; + for (int i = 0; i < N; ++i) { + double best = 1e30; + for (int j = 0; j < N; ++j) { + if (i == j) continue; + double d2 = 0; + for (int d = 0; d < 3; ++d) { + const double dd = Cp[size_t(i)][size_t(d)] - Cp[size_t(j)][size_t(d)]; + d2 += dd*dd; + } + best = std::min(best, d2); + } + sigma += std::sqrt(best) / N; + } + sigma *= 1.5; + if (sigma < 1e-9) return {}; + const double inv2s2 = 1.0 / (2.0 * sigma * sigma); + const double ridge = 1e-3; + + // Solve (A + ridge*I) w = R for the 3 axes with Gaussian elimination. + std::vector A(size_t(N)*N, 0.0); + for (int i = 0; i < N; ++i) + for (int j = 0; j < N; ++j) { + double d2 = 0; + for (int d = 0; d < 3; ++d) { + const double dd = Cp[size_t(i)][size_t(d)] - Cp[size_t(j)][size_t(d)]; + d2 += dd*dd; + } + A[size_t(i)*N + j] = std::exp(-d2 * inv2s2) + (i == j ? ridge : 0.0); + } + std::vector> rhs = R; + for (int col = 0; col < N; ++col) { + int piv = col; + for (int r = col+1; r < N; ++r) + if (std::abs(A[size_t(r)*N+col]) > std::abs(A[size_t(piv)*N+col])) + piv = r; + if (std::abs(A[size_t(piv)*N+col]) < 1e-12) return {}; + if (piv != col) { + for (int c = 0; c < N; ++c) + std::swap(A[size_t(piv)*N+c], A[size_t(col)*N+c]); + std::swap(rhs[size_t(piv)], rhs[size_t(col)]); + } + const double p = A[size_t(col)*N+col]; + for (int r = col+1; r < N; ++r) { + const double f = A[size_t(r)*N+col] / p; + if (f == 0.0) continue; + for (int c = col; c < N; ++c) + A[size_t(r)*N+c] -= f * A[size_t(col)*N+c]; + for (int d = 0; d < 3; ++d) + rhs[size_t(r)][size_t(d)] -= f * rhs[size_t(col)][size_t(d)]; + } + } + std::vector> w(size_t(N), {0,0,0}); + for (int r = N-1; r >= 0; --r) { + std::array acc = rhs[size_t(r)]; + for (int c = r+1; c < N; ++c) + for (int d = 0; d < 3; ++d) + acc[size_t(d)] -= A[size_t(r)*N+c] * w[size_t(c)][size_t(d)]; + for (int d = 0; d < 3; ++d) + w[size_t(r)][size_t(d)] = acc[size_t(d)] / A[size_t(r)*N+r]; + } + + // Warp every template vertex: similarity, then the decaying residual field. + std::vector out(tmplV.size()); + for (int v = 0; v < nv; ++v) { + const std::array p = prealign( + {tmplV[size_t(v)*3], tmplV[size_t(v)*3+1], tmplV[size_t(v)*3+2]}); + std::array disp{0,0,0}; + for (int i = 0; i < N; ++i) { + double d2 = 0; + for (int d = 0; d < 3; ++d) { + const double dd = p[size_t(d)] - Cp[size_t(i)][size_t(d)]; + d2 += dd*dd; + } + const double phi = std::exp(-d2 * inv2s2); + for (int d = 0; d < 3; ++d) + disp[size_t(d)] += w[size_t(i)][size_t(d)] * phi; + } + out[size_t(v)*3] = float(p[0] + disp[0]); + out[size_t(v)*3+1] = float(p[1] + disp[1]); + out[size_t(v)*3+2] = float(p[2] + disp[2]); + } + return out; +} + +FaceRigResult buildFaceRig(const std::vector& userV, + const std::vector& userF, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts, + const std::vector& headMask, + const std::vector& landmarks, + const FaceRigProgressFn& progress) +{ + FaceRigResult r; + if (userV.size() < 9 || userF.size() < 3) { + r.error = "user mesh has no geometry"; + return r; + } + if (!tmpl.valid()) { + r.error = "ARKit template not loaded"; + return r; + } + const int nuFull = int(userV.size() / 3); + r.userVertexCount = nuFull; + + // Head isolation: if a mask is supplied, build a head-only sub-mesh and fit + // THAT (so the face template lands on the face, not the whole body). We + // keep a fit→full-mesh vertex index map so the resampled deltas scatter + // back to the right original vertices; non-head vertices get zero delta. + // Without a mask, fit the whole mesh (a bare-face crop). + std::vector subV; + std::vector subF; + std::vector subToFull; // fit vertex index → full-mesh index + const bool isolate = int(headMask.size()) == nuFull; + if (isolate) { + std::vector fullToSub(size_t(nuFull), -1); + for (int v = 0; v < nuFull; ++v) { + if (!headMask[size_t(v)]) continue; + fullToSub[size_t(v)] = int(subToFull.size()); + subToFull.push_back(v); + subV.insert(subV.end(), + {userV[size_t(v)*3], userV[size_t(v)*3+1], userV[size_t(v)*3+2]}); + } + // keep faces whose 3 verts are all head; remap to sub indices + for (size_t f = 0; f + 2 < userF.size(); f += 3) { + const int a = userF[f], b = userF[f+1], c = userF[f+2]; + const int nFull = int(fullToSub.size()); + if (a < 0 || b < 0 || c < 0 || a >= nFull || b >= nFull || c >= nFull) + continue; + const int sa = fullToSub[size_t(a)], sb = fullToSub[size_t(b)], + sc = fullToSub[size_t(c)]; + if (sa >= 0 && sb >= 0 && sc >= 0) + subF.insert(subF.end(), {sa, sb, sc}); + } + if (subV.size() < 9 || subF.size() < 3) { + // head region had no usable surface — fall back to whole-mesh fit + subV.clear(); subF.clear(); subToFull.clear(); + } + } + const bool useSub = !subToFull.empty(); + const std::vector& fitV = useSub ? subV : userV; + const std::vector& fitF = useSub ? subF : userF; + const int nu = int(fitV.size() / 3); + + // Progress model: the NRICP fit anneals over N stiffness levels (each a + // step under "Fitting…"), then one step per transferred shape. Total = + // fitLevels + shapeCount so the bar advances through BOTH phases. + NricpOptions fitOpts; + fitOpts.landmarks = landmarks; // anchor the fit to detected face features + const int fitLevels = int(fitOpts.stiffness.size()); + const int shapeTotal = opts.maxShapes > 0 + ? std::min(opts.maxShapes, tmpl.shapeCount()) + : tmpl.shapeCount(); + const int total = fitLevels + shapeTotal; + bool cancelled = false; + auto tick = [&](int done, const char* phase) -> bool { + return progress ? progress(done, total, phase) : true; + }; + + // Marker-driven RBF pre-warp: with a small, CURATED anchor set (the + // user-adjusted markers — ≤ ~16), warp the whole template into the user's + // face proportions before the fit, so the mouth/eyes/chin START on the + // marked positions and the space between interpolates smoothly. Soft + // in-fit constraints alone let un-anchored regions slide on faces far from + // the template (cartoon proportions), smearing the transferred shapes. + // Deliberately NOT applied to bulk auto-detected anchor sets (hundreds of + // points): a garbage detection would fold the template. + std::vector fitTmplV = tmpl.neutral(); + if (!landmarks.empty() && landmarks.size() <= 32) { + std::vector warped = rbfWarpByAnchors(tmpl.neutral(), landmarks); + if (!warped.empty()) fitTmplV = std::move(warped); + } + + // ── TEMPLATE COMPONENT SPLIT (the eyes/teeth fix) ─────────────────────── + // The ICT template is ~191 connected components: the outer face surface + // (14k verts) plus eyeballs, corneas, teeth, mouth interior, lashes… The + // surface fit drags INTERIOR component verts onto the OUTER skin (closest- + // point has no better answer), destroying their correspondence — measured: + // eyeLook/eyeBlink deltas landed nowhere and the eyes never moved, on the + // TEMPLATE ITSELF. Fit ONLY the main component; place each satellite by the + // local AFFINE its surrounding main-surface region underwent, preserving + // the eyeball/teeth structure inside the fitted head. + const int tvc = tmpl.vertexCount(); + std::vector comp(size_t(tvc), 0); + int compCount = 1; + { + std::vector par(size_t(tvc), 0); + for (int i = 0; i < tvc; ++i) par[size_t(i)] = i; + std::function findRoot = [&](int x) { + while (par[size_t(x)] != x) { + par[size_t(x)] = par[size_t(par[size_t(x)])]; + x = par[size_t(x)]; + } + return x; + }; + const auto& tf = tmpl.faces(); + for (size_t f = 0; f + 2 < tf.size(); f += 3) { + int a = findRoot(tf[f]), b = findRoot(tf[f+1]); + if (a != b) par[size_t(a)] = b; + a = findRoot(tf[f+1]); b = findRoot(tf[f+2]); + if (a != b) par[size_t(a)] = b; + } + std::unordered_map remap; + compCount = 0; + for (int i = 0; i < tvc; ++i) { + const int root = findRoot(i); + auto it = remap.find(root); + if (it == remap.end()) { remap.emplace(root, compCount); comp[size_t(i)] = compCount++; } + else comp[size_t(i)] = it->second; + } + } + int mainComp = 0; + { + std::vector cnt(size_t(compCount), 0); + for (int i = 0; i < tvc; ++i) cnt[size_t(comp[size_t(i)])]++; + for (int c = 1; c < compCount; ++c) + if (cnt[size_t(c)] > cnt[size_t(mainComp)]) mainComp = c; + } + + // Extract the main-component sub-template (from the possibly-warped verts) + // and remap anchors onto it (markers sit on the outer surface; any anchor + // that resolved onto a satellite is dropped). + std::vector mainV; std::vector mainF; std::vector mainToFull; + std::vector fullToMainIdx(size_t(tvc), -1); + if (compCount > 1) { + for (int i = 0; i < tvc; ++i) { + if (comp[size_t(i)] != mainComp) continue; + fullToMainIdx[size_t(i)] = int(mainToFull.size()); + mainToFull.push_back(i); + mainV.insert(mainV.end(), {fitTmplV[size_t(i)*3], + fitTmplV[size_t(i)*3+1], + fitTmplV[size_t(i)*3+2]}); + } + const auto& tf = tmpl.faces(); + for (size_t f = 0; f + 2 < tf.size(); f += 3) { + const int a = fullToMainIdx[size_t(tf[f])], + b = fullToMainIdx[size_t(tf[f+1])], + c = fullToMainIdx[size_t(tf[f+2])]; + if (a >= 0 && b >= 0 && c >= 0) mainF.insert(mainF.end(), {a, b, c}); + } + std::vector mainAnchors; + for (auto lm : fitOpts.landmarks) { + if (lm.tmplVertex < 0 || lm.tmplVertex >= tvc) continue; + const int mi = fullToMainIdx[size_t(lm.tmplVertex)]; + if (mi >= 0) { lm.tmplVertex = mi; mainAnchors.push_back(lm); } + } + fitOpts.landmarks = std::move(mainAnchors); + } + const bool splitTmpl = compCount > 1 && mainV.size() >= 9 && mainF.size() >= 3; + const std::vector& fitTV = splitTmpl ? mainV : fitTmplV; + const std::vector& fitTF = splitTmpl ? mainF : tmpl.faces(); + + // 1) NRICP: (pre-warped) template MAIN SURFACE → user neutral. + // Report each annealing level so the (long) fit phase visibly advances. + const NricpResult fit = FaceRig::fit( + fitTV, fitTF, fitV, fitF, fitOpts, + [&](int level, int /*levelCount*/) -> bool { + if (!tick(level, "Fitting face template…")) { cancelled = true; return false; } + return true; + }); + if (cancelled) { r.error = "cancelled"; return r; } + if (!fit.ok || fit.diag <= 0.0) { + r.error = "non-rigid fit failed"; + return r; + } + r.fitMeanResidualPct = 100.0 * fit.meanResidual / fit.diag; + r.fitMaxResidualPct = 100.0 * fit.maxResidual / fit.diag; + if (!tick(fitLevels, "Transferring shapes…")) { r.error = "cancelled"; return r; } + + // humanoid-only gate: a bad fit means this isn't a face — refuse. A NRICP + // fit that couldn't converge onto the surface reports non-finite or huge + // residuals (e.g. a plane template forced onto a sphere), which we treat + // as a hard reject. `maxFitResidualPct` gates the MAX residual directly — + // the knob is advertised as `--max-residual`, so it must mean what it says + // (it previously allowed up to 6x the supplied value). The mean gate at a + // quarter of it catches fits that never blow up locally but drape the + // whole surface badly (healthy fits measure mean <= 0.1%, max <= ~4%). + const bool nonFinite = !std::isfinite(r.fitMeanResidualPct) || + !std::isfinite(r.fitMaxResidualPct); + if (nonFinite || r.fitMaxResidualPct > opts.maxFitResidualPct || + r.fitMeanResidualPct > opts.maxFitResidualPct * 0.25) { + r.error = "mesh does not fit the human face template (mean residual " + + std::to_string(r.fitMeanResidualPct) + "%, max " + + std::to_string(r.fitMaxResidualPct) + + "%); this does not look like a human face mesh"; + return r; + } + + // Assemble the FULL fitted correspondence: main verts from the fit; + // satellites by the local affine their neighbouring main region underwent + // (least-squares over the K nearest FINITE main verts). + const std::vector& tn = tmpl.neutral(); + std::vector fitted; + if (!splitTmpl) { + fitted = fit.fitted; + } else { + fitted.assign(size_t(tvc) * 3, 0.0f); + for (size_t m = 0; m < mainToFull.size(); ++m) + for (int d = 0; d < 3; ++d) + fitted[size_t(mainToFull[m])*3 + d] = fit.fitted[m*3 + d]; + + // group satellite verts per component + std::unordered_map> sats; + for (int i = 0; i < tvc; ++i) + if (comp[size_t(i)] != mainComp) sats[comp[size_t(i)]].push_back(i); + + for (auto& [cid, verts] : sats) { + // centroid in the (warped) template space + std::array ctr{0,0,0}; + for (int v : verts) + for (int d = 0; d < 3; ++d) + ctr[size_t(d)] += fitTmplV[size_t(v)*3+d] / double(verts.size()); + // K nearest FINITE main verts to the centroid + constexpr int K = 60; + std::vector> near; // (dist², main idx) + near.reserve(mainToFull.size()); + for (size_t m = 0; m < mainToFull.size(); ++m) { + bool finite = true; + for (int d = 0; d < 3; ++d) + if (!std::isfinite(fit.fitted[m*3+d])) { finite = false; break; } + if (!finite) continue; + const int fv = mainToFull[m]; + float d2 = 0; + for (int d = 0; d < 3; ++d) { + const float dd = fitTmplV[size_t(fv)*3+d] - float(ctr[size_t(d)]); + d2 += dd*dd; + } + near.push_back({d2, int(m)}); + } + const int k = std::min(K, int(near.size())); + if (k < 4) { + // no usable neighbours — leave the satellite at the template + // rest (it just won't deform meaningfully). + for (int v : verts) + for (int d = 0; d < 3; ++d) + fitted[size_t(v)*3+d] = tn[size_t(v)*3+d]; + continue; + } + std::partial_sort(near.begin(), near.begin()+k, near.end()); + // least-squares affine: (warped rest) → (fitted), normal equations + // per output dim: (SᵀS) w = Sᵀ t, S rows = [x y z 1]. + double StS[4][4] = {{0}}, Stt[3][4] = {{0}}; + for (int n = 0; n < k; ++n) { + const int m = near[size_t(n)].second; + const int fv = mainToFull[size_t(m)]; + const double s[4] = {fitTmplV[size_t(fv)*3], fitTmplV[size_t(fv)*3+1], + fitTmplV[size_t(fv)*3+2], 1.0}; + for (int a = 0; a < 4; ++a) + for (int b = 0; b < 4; ++b) + StS[a][b] += s[a]*s[b]; + for (int d = 0; d < 3; ++d) + for (int a = 0; a < 4; ++a) + Stt[d][a] += double(fit.fitted[size_t(m)*3+d]) * s[a]; + } + // solve 4x4 (Gaussian, shared factorisation for the 3 rhs) + double A[4][7]; + for (int a = 0; a < 4; ++a) { + for (int b = 0; b < 4; ++b) A[a][b] = StS[a][b]; + for (int d = 0; d < 3; ++d) A[a][4+d] = Stt[d][a]; + } + bool singular = false; + for (int col = 0; col < 4 && !singular; ++col) { + int piv = col; + for (int rr = col+1; rr < 4; ++rr) + if (std::abs(A[rr][col]) > std::abs(A[piv][col])) piv = rr; + if (std::abs(A[piv][col]) < 1e-12) { singular = true; break; } + if (piv != col) for (int cc = 0; cc < 7; ++cc) std::swap(A[piv][cc], A[col][cc]); + for (int rr = col+1; rr < 4; ++rr) { + const double f2 = A[rr][col] / A[col][col]; + for (int cc = col; cc < 7; ++cc) A[rr][cc] -= f2 * A[col][cc]; + } + } + double W[3][4]; // affine rows per output dim + if (!singular) { + for (int d = 0; d < 3; ++d) + for (int rr = 3; rr >= 0; --rr) { + double acc = A[rr][4+d]; + for (int cc = rr+1; cc < 4; ++cc) acc -= A[rr][cc] * W[d][cc]; + W[d][rr] = acc / A[rr][rr]; + } + } + for (int v : verts) { + if (singular) { + for (int d = 0; d < 3; ++d) + fitted[size_t(v)*3+d] = tn[size_t(v)*3+d]; + continue; + } + const double p[4] = {fitTmplV[size_t(v)*3], fitTmplV[size_t(v)*3+1], + fitTmplV[size_t(v)*3+2], 1.0}; + for (int d = 0; d < 3; ++d) { + double o = 0; + for (int a = 0; a < 4; ++a) o += W[d][a] * p[a]; + fitted[size_t(v)*3+d] = float(o); + } + } + } + } + + // sanitize the correspondence: a handful of template verts may have + // diverged (NaN/inf) even in an accepted fit (< 5% by the NRICP gate). + // Fall those back to the template neutral so they don't poison the + // deformation-transfer solve — they simply won't deform meaningfully. + for (size_t i = 0; i < fitted.size() && i < tn.size(); ++i) + if (!std::isfinite(fitted[i])) + fitted[i] = tn[i]; + + // 2) DeformationTransfer over the fixed (topology + fit) system. + DeformationTransfer dt; + if (!dt.init(tmpl.neutral(), tmpl.faces(), fitted)) { + r.error = "deformation-transfer setup failed"; + return r; + } + + // 3) resample map: fit vertex → nearest correspondence vertex (built once). + PointGrid grid; + grid.build(fitted); + std::vector userToTmpl(size_t(nu), -1); + for (int i = 0; i < nu; ++i) + userToTmpl[size_t(i)] = grid.nearest(&fitV[size_t(i)*3]); + + // noise floor scaled by the FIT region diagonal (a head is smaller than a + // whole body, so scaling on the full-body diag would swallow real motion). + const double diag = bboxDiag(fitV); + const double eps = opts.deltaEpsPct / 100.0 * diag; + + const auto& shapes = tmpl.shapes(); + const int maxShapes = opts.maxShapes > 0 + ? std::min(opts.maxShapes, int(shapes.size())) + : int(shapes.size()); + + for (int s = 0; s < maxShapes; ++s) { + if (!tick(fitLevels + s, "Transferring shapes…")) { r.error = "cancelled"; return r; } + // per-TEMPLATE-vertex delta on the user identity + const std::vector tmplDelta = dt.transfer(shapes[size_t(s)].deltas); + if (int(tmplDelta.size() / 3) != tmpl.vertexCount()) { + r.error = "transfer produced an unexpected vertex count"; + return r; + } + + FaceRigShape out; + out.name = shapes[size_t(s)].name; + // Deltas are always full-mesh sized; head isolation writes only the + // head vertices (via subToFull), leaving the body at zero. + out.userDeltas.assign(size_t(nuFull) * 3, 0.0f); + const float amp = float(std::clamp(opts.amplitude, 0.1, 5.0)); + for (int i = 0; i < nu; ++i) { + const int t = userToTmpl[size_t(i)]; + if (t < 0) continue; + float dvec[3] = {amp * tmplDelta[size_t(t)*3], + amp * tmplDelta[size_t(t)*3+1], + amp * tmplDelta[size_t(t)*3+2]}; + const double mag = std::sqrt(double(dvec[0])*dvec[0] + + double(dvec[1])*dvec[1] + + double(dvec[2])*dvec[2]); + if (mag < eps) continue; // noise floor → keep sparse + const int dst = useSub ? subToFull[size_t(i)] : i; + out.userDeltas[size_t(dst)*3] = dvec[0]; + out.userDeltas[size_t(dst)*3+1] = dvec[1]; + out.userDeltas[size_t(dst)*3+2] = dvec[2]; + out.nonZeroVerts++; + out.maxDisp = std::max(out.maxDisp, float(mag)); + } + r.shapes.push_back(std::move(out)); + } + + // Amplitude safety net: a poisoned anchor set (garbage landmarks that + // slipped every gate) crushes the fit so the transferred shapes come out + // technically-attached but INVISIBLE (~0.05% of the head, vs ~5% for a + // healthy jawOpen). If the anchored run produced nothing visible, retry + // once WITHOUT anchors — a plain head-isolated fit always beats an + // invisible one. (Field-reproduced failure mode; do not remove.) + if (!landmarks.empty()) { + double maxAmp = 0; + for (const auto& sh : r.shapes) + maxAmp = std::max(maxAmp, double(sh.maxDisp)); + // maxDisp includes the user amplitude — normalise it out so a healthy + // rig at amplitude 0.1 doesn't read as "invisible" and rerun. + maxAmp /= std::clamp(opts.amplitude, 0.1, 5.0); + if (maxAmp < 0.005 * diag) { + std::fprintf(stderr, "[facerig] anchored fit produced invisible " + "shapes (max %.5f on diag %.3f) — retrying " + "unanchored\n", maxAmp, diag); + return buildFaceRig(userV, userF, tmpl, opts, headMask, {}, + progress); + } + } + + r.ok = true; + return r; +} + +} // namespace FaceRig diff --git a/src/FaceRig/FaceRigger.h b/src/FaceRig/FaceRigger.h new file mode 100644 index 000000000..1a5566868 --- /dev/null +++ b/src/FaceRig/FaceRigger.h @@ -0,0 +1,113 @@ +#ifndef FACERIGGER_H +#define FACERIGGER_H + +// FaceRigger — the face auto-rig orchestrator (#889, Slice E #893). Chains the +// pure-data stages into per-USER-vertex ARKit blendshape deltas: +// +// ArkitTemplate (neutral + 52 expression deltas, template topology) +// │ +// ▼ NonRigidICP (#891): fit template neutral → user neutral +// correspondence X (template-topology verts lying on the user surface) +// │ +// ▼ DeformationTransfer (#892): per shape, transfer the template's +// │ per-triangle deformation onto X → per-TEMPLATE-vertex delta +// │ +// ▼ resample template-topology deltas → the real USER vertices +// 52 × per-user-vertex deltas, named per FaceCap::kBlendshapeNames +// +// Ogre-free + headless-tested; the Ogre attach (create Pose + VAT_POSE per +// shape on the user entity) + CLI/MCP surfaces live in the FaceRig CLI/MCP +// layer, reusing MorphCommands' buildPosesFromSlices pattern. +// +// Humanoid-only (risk #1 in docs/FACE_RIG_SPIKE.md): the fit residual is a +// quality gate — a non-face mesh fits poorly and is rejected with a clear +// reason, mirroring the AutoRig/Pinocchio precedent. + +#include "NonRigidICP.h" // NricpLandmark + +#include + +#include +#include +#include + +namespace FaceRig { + +class ArkitTemplate; + +struct FaceRigShape { + QString name; // a FaceCap::kBlendshapeNames entry + std::vector userDeltas; // userVertexCount*3, (expr - neutral) + int nonZeroVerts = 0; // verts this shape actually moves + float maxDisp = 0.0f; // max |delta| (mesh units) +}; + +struct FaceRigResult { + std::vector shapes; + int userVertexCount = 0; + double fitMeanResidualPct = 0.0; // NRICP mean residual / user diag (%) + double fitMaxResidualPct = 0.0; + bool ok = false; + std::string error; // set when !ok +}; + +struct FaceRigOptions { + // Reject the rig when the NRICP fit is worse than this (% of user diag). + // A human template only fits a roughly human face; a prop/creature blows + // past this and is refused rather than emitting garbage shapes. + double maxFitResidualPct = 8.0; + // Drop per-vertex deltas below this fraction of the user diagonal (noise + // floor from the transfer solve) so shapes stay sparse. + double deltaEpsPct = 0.02; + // Cap the shapes generated (0 = all in the template). Names are matched + // against the template's own shape order. + int maxShapes = 0; + // Amplitude multiplier applied to every transferred delta (an + // "exaggeration" control, like Mixamo's arm-space). 1 = as transferred; + // >1 amplifies subtle results on stylized faces whose transfer comes out + // conservative. Clamped by callers to a sane range. + double amplitude = 1.0; +}; + +// Progress callback: (done, total, phase). `phase` is a short static label +// ("Fitting…", "Transferring shapes…"). Return false to request cancellation — +// buildFaceRig then returns ok=false with error "cancelled". May be called from +// a worker thread, so the callee must marshal any UI work itself. +using FaceRigProgressFn = + std::function; + +// userV/userF: the user neutral mesh (Nu*3 verts + Fu*3 tris). tmpl: a loaded +// ArkitTemplate. Returns per-user-vertex delta sets (size Nu*3 each), or +// ok=false + error on failure / a poor fit. +// `headMask` (optional, size Nu): when non-empty, ONLY vertices flagged 1 are +// fitted against the face template and receive blendshape deltas — the fix for +// full-body characters (the face template must not smear over the body). Empty +// = fit the whole mesh (a bare-face crop). +// `landmarks` (optional): facial-landmark correspondences pinning template +// vertices to USER positions (in the SAME frame as userV), computed externally +// by the Ogre landmark layer. When present they anchor the NRICP fit so the +// template lands on the real eyes/nose/mouth instead of a mis-oriented drape. +// `progress` (optional) reports fit + per-shape steps and can cancel. +FaceRigResult buildFaceRig(const std::vector& userV, + const std::vector& userF, + const ArkitTemplate& tmpl, + const FaceRigOptions& opts = {}, + const std::vector& headMask = {}, + const std::vector& landmarks = {}, + const FaceRigProgressFn& progress = {}); + +// RBF (thin-plate, φ(r)=r) space warp of the template driven by the landmark +// anchors: each anchor's template vertex lands EXACTLY on its user target and +// the space between interpolates smoothly. Used to PRE-WARP the template into +// the user's face proportions before NRICP — soft in-fit constraints alone let +// the un-anchored regions slide on faces far from the template (cartoon +// proportions), which smears the transferred shapes. Returns the warped +// positions, or empty when there are too few / degenerate anchors (< 4, or a +// singular system) — the caller then fits the unwarped template as before. +// Pure data + unit-tested. +std::vector rbfWarpByAnchors(const std::vector& tmplV, + const std::vector& anchors); + +} // namespace FaceRig + +#endif // FACERIGGER_H diff --git a/src/FaceRig/FaceRigger_test.cpp b/src/FaceRig/FaceRigger_test.cpp new file mode 100644 index 000000000..3a663fbed --- /dev/null +++ b/src/FaceRig/FaceRigger_test.cpp @@ -0,0 +1,237 @@ +#include + +#include "FaceRig/ArkitTemplate.h" +#include "FaceRig/FaceRigger.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +struct Grid { + std::vector V; + std::vector F; +}; + +// A bumpy plane — a stand-in "face" surface with real triangles. +Grid makeGrid(int n, float extent, float bump) +{ + Grid g; + for (int y = 0; y < n; ++y) + for (int x = 0; x < n; ++x) { + const float fx = (float(x)/(n-1) - 0.5f) * extent; + const float fy = (float(y)/(n-1) - 0.5f) * extent; + const float fz = bump * std::sin(1.5f*float(x)) * std::cos(1.5f*float(y)); + g.V.insert(g.V.end(), {fx, fy, fz}); + } + for (int y = 0; y < n-1; ++y) + for (int x = 0; x < n-1; ++x) { + const int a = y*n+x, b = y*n+x+1, c = (y+1)*n+x, d = (y+1)*n+x+1; + g.F.insert(g.F.end(), {a, b, c}); + g.F.insert(g.F.end(), {b, d, c}); + } + return g; +} + +void putI32(QByteArray& b, int32_t v) { + v = qToLittleEndian(v); + b.append(reinterpret_cast(&v), 4); +} +void putF32(QByteArray& b, float f) { + quint32 raw; std::memcpy(&raw, &f, 4); raw = qToLittleEndian(raw); + b.append(reinterpret_cast(&raw), 4); +} + +// Pack a synthetic arkit_template.bin (magic + V/F/S + neutral + faces + +// S×(name[32] + delta[V*3])) and write it to `path`. +bool writeSyntheticTemplate(const QString& path, const Grid& g, + const std::vector>>& shapes) +{ + QByteArray b; + b.append("QMFRT1\0\0", 8); + putI32(b, int(g.V.size()/3)); + putI32(b, int(g.F.size()/3)); + putI32(b, int(shapes.size())); + for (float v : g.V) putF32(b, v); + for (int i : g.F) putI32(b, i); + for (const auto& [name, delta] : shapes) { + char nm[32] = {0}; + const QByteArray n = name.toLatin1(); + std::memcpy(nm, n.constData(), std::min(31, size_t(n.size()))); + b.append(nm, 32); + for (float d : delta) putF32(b, d); + } + QDir().mkpath(QFileInfo(path).absolutePath()); + QFile f(path); + if (!f.open(QIODevice::WriteOnly)) return false; + f.write(b); + f.close(); + return true; +} + +QString tempTemplatePath() +{ + const QString dir = QStandardPaths::writableLocation(QStandardPaths::TempLocation); + return QDir(dir).filePath("qtmesh_facerig_test_template.bin"); +} + +} // namespace + +TEST(FaceRigger, RejectsBadInput) +{ + FaceRig::ArkitTemplate empty; + const auto r = FaceRig::buildFaceRig({}, {}, empty); + EXPECT_FALSE(r.ok); + EXPECT_FALSE(r.error.empty()); +} + +TEST(FaceRigger, ProducesPerUserVertexShapes) +{ + const Grid tmpl = makeGrid(12, 2.0f, 0.12f); + const int nt = int(tmpl.V.size()/3); + + // shape A: push a centre bump in +Z (a "smile"-ish local deform) + std::vector smile(tmpl.V.size(), 0.0f); + for (int i = 0; i < nt; ++i) { + const float x = tmpl.V[size_t(i)*3], y = tmpl.V[size_t(i)*3+1]; + smile[size_t(i)*3+2] = 0.15f * std::exp(-(x*x + y*y) * 5.0f); + } + // shape B: drop the lower half in -Z (a "jawOpen"-ish region deform) + std::vector jaw(tmpl.V.size(), 0.0f); + for (int i = 0; i < nt; ++i) { + const float y = tmpl.V[size_t(i)*3+1]; + if (y < 0.0f) jaw[size_t(i)*3+2] = -0.2f * (-y); + } + + const QString path = tempTemplatePath(); + ASSERT_TRUE(writeSyntheticTemplate(path, tmpl, + {{"mouthSmileLeft", smile}, {"jawOpen", jaw}})); + + FaceRig::ArkitTemplate at; + QString err; + ASSERT_TRUE(at.load(path, &err)) << err.toStdString(); + ASSERT_EQ(at.shapeCount(), 2); + + // user = the SAME surface at a different tessellation (different topology) + const Grid user = makeGrid(16, 2.0f, 0.12f); + const int nu = int(user.V.size()/3); + + const auto r = FaceRig::buildFaceRig(user.V, user.F, at); + ASSERT_TRUE(r.ok) << r.error; + EXPECT_EQ(r.userVertexCount, nu); + EXPECT_EQ(int(r.shapes.size()), 2); + // same-surface fit should be tight + EXPECT_LT(r.fitMeanResidualPct, 5.0); + + for (const auto& sh : r.shapes) { + EXPECT_EQ(int(sh.userDeltas.size()), nu * 3); + for (float v : sh.userDeltas) EXPECT_TRUE(std::isfinite(v)); + EXPECT_GT(sh.nonZeroVerts, 0); // the shape actually moves verts + EXPECT_GT(sh.maxDisp, 0.0f); + } + + // names preserved, order preserved + EXPECT_EQ(r.shapes[0].name, QStringLiteral("mouthSmileLeft")); + EXPECT_EQ(r.shapes[1].name, QStringLiteral("jawOpen")); + + // semantics: the smile shape moves CENTRE verts most; the jaw shape moves + // LOWER-half verts most. Check the centroid of moved mass. + auto movedCentroidY = [&](const FaceRig::FaceRigShape& sh) { + double sy = 0, w = 0; + for (int i = 0; i < nu; ++i) { + const float* d = &sh.userDeltas[size_t(i)*3]; + const double m = std::sqrt(double(d[0])*d[0]+double(d[1])*d[1]+double(d[2])*d[2]); + sy += m * double(user.V[size_t(i)*3+1]); + w += m; + } + return w > 0 ? sy / w : 0.0; + }; + // jaw deformation lives in the lower half (y<0) → its moved-mass centroid Y + // is clearly below the smile's (which is centred at y≈0). + EXPECT_LT(movedCentroidY(r.shapes[1]), movedCentroidY(r.shapes[0])); +} + +// RBF pre-warp: anchors land exactly on their targets and the space between +// interpolates smoothly (a pure translation of all anchors translates the +// whole mesh). +TEST(FaceRigger, RbfWarpInterpolatesAnchors) +{ + const Grid g = makeGrid(8, 2.0f, 0.1f); + const int n = int(g.V.size()/3); + + // pure translation: 5 anchors all displaced by (0.3, -0.2, 0.1) + std::vector anchors; + const int picks[5] = {0, 7, n/2, n-8, n-1}; + for (int p : picks) { + FaceRig::NricpLandmark a; + a.tmplVertex = p; + a.target = {g.V[size_t(p)*3] + 0.3f, g.V[size_t(p)*3+1] - 0.2f, + g.V[size_t(p)*3+2] + 0.1f}; + anchors.push_back(a); + } + const auto warped = FaceRig::rbfWarpByAnchors(g.V, anchors); + ASSERT_EQ(warped.size(), g.V.size()); + // anchors land exactly (affine part reproduces the translation) + for (int p : picks) { + EXPECT_NEAR(warped[size_t(p)*3], g.V[size_t(p)*3] + 0.3f, 1e-3f); + EXPECT_NEAR(warped[size_t(p)*3+1], g.V[size_t(p)*3+1] - 0.2f, 1e-3f); + EXPECT_NEAR(warped[size_t(p)*3+2], g.V[size_t(p)*3+2] + 0.1f, 1e-3f); + } + // a pure-translation anchor set translates EVERY vertex (thin-plate exact + // for affine displacement fields) + for (int v = 0; v < n; ++v) { + EXPECT_NEAR(warped[size_t(v)*3], g.V[size_t(v)*3] + 0.3f, 1e-2f); + EXPECT_NEAR(warped[size_t(v)*3+1], g.V[size_t(v)*3+1] - 0.2f, 1e-2f); + } + // too few anchors → empty (caller falls back to the unwarped template) + anchors.resize(3); + EXPECT_TRUE(FaceRig::rbfWarpByAnchors(g.V, anchors).empty()); +} + +TEST(FaceRigger, RejectsNonFaceMesh) +{ + // template = bumpy plane with a shape + const Grid tmpl = makeGrid(10, 2.0f, 0.12f); + std::vector shape(tmpl.V.size(), 0.0f); + for (size_t i = 2; i < shape.size(); i += 3) shape[i] = 0.1f; + const QString path = tempTemplatePath(); + ASSERT_TRUE(writeSyntheticTemplate(path, tmpl, {{"jawOpen", shape}})); + FaceRig::ArkitTemplate at; + ASSERT_TRUE(at.load(path)); + + // user = a closed sphere: a fundamentally different topology/shape than the + // open plane template. NRICP cannot wrap a plane around a sphere cleanly, so + // the fit residual stays high and the humanoid-only guard rejects it. + Grid user; + { + const int nlat = 12, nlon = 16; + for (int i = 0; i <= nlat; ++i) { + const float th = 3.14159265358979f * float(i) / nlat; + for (int j = 0; j < nlon; ++j) { + const float ph = 2.0f * 3.14159265358979f * float(j) / nlon; + user.V.insert(user.V.end(), + {std::sin(th)*std::cos(ph), std::cos(th), std::sin(th)*std::sin(ph)}); + } + } + for (int i = 0; i < nlat; ++i) + for (int j = 0; j < nlon; ++j) { + const int a = i*nlon + j, b = i*nlon + (j+1)%nlon; + const int c = (i+1)*nlon + j, d = (i+1)*nlon + (j+1)%nlon; + user.F.insert(user.F.end(), {a, b, c}); + user.F.insert(user.F.end(), {b, d, c}); + } + } + FaceRig::FaceRigOptions opts; + opts.maxFitResidualPct = 3.0; + const auto r = FaceRig::buildFaceRig(user.V, user.F, at, opts); + EXPECT_FALSE(r.ok); + EXPECT_NE(r.error.find("face"), std::string::npos); +} diff --git a/src/FaceRig/NonRigidICP.cpp b/src/FaceRig/NonRigidICP.cpp new file mode 100644 index 000000000..e23af239d --- /dev/null +++ b/src/FaceRig/NonRigidICP.cpp @@ -0,0 +1,458 @@ +#include "NonRigidICP.h" + +#include +#include +#include +#include + +namespace FaceRig { + +namespace { + +using Vec3 = std::array; + +Vec3 vsub(const Vec3& a, const Vec3& b) { return {a[0]-b[0], a[1]-b[1], a[2]-b[2]}; } +double vdot(const Vec3& a, const Vec3& b) { return a[0]*b[0]+a[1]*b[1]+a[2]*b[2]; } +Vec3 vadd(const Vec3& a, const Vec3& b) { return {a[0]+b[0], a[1]+b[1], a[2]+b[2]}; } +Vec3 vscale(const Vec3& a, double s) { return {a[0]*s, a[1]*s, a[2]*s}; } + +Vec3 at(const std::vector& v, int i) +{ + return {v[size_t(i)*3], v[size_t(i)*3+1], v[size_t(i)*3+2]}; +} + +// ---- closest point on a triangle (Ericson, Real-Time Collision Detection) -- +Vec3 closestPointTriangle(const Vec3& p, const Vec3& a, const Vec3& b, const Vec3& c) +{ + const Vec3 ab = vsub(b, a), ac = vsub(c, a), ap = vsub(p, a); + const double d1 = vdot(ab, ap), d2 = vdot(ac, ap); + if (d1 <= 0 && d2 <= 0) return a; + const Vec3 bp = vsub(p, b); + const double d3 = vdot(ab, bp), d4 = vdot(ac, bp); + if (d3 >= 0 && d4 <= d3) return b; + const double vc = d1*d4 - d3*d2; + if (vc <= 0 && d1 >= 0 && d3 <= 0) return vadd(a, vscale(ab, d1/(d1-d3))); + const Vec3 cp = vsub(p, c); + const double d5 = vdot(ab, cp), d6 = vdot(ac, cp); + if (d6 >= 0 && d5 <= d6) return c; + const double vb = d5*d2 - d1*d6; + if (vb <= 0 && d2 >= 0 && d6 <= 0) return vadd(a, vscale(ac, d2/(d2-d6))); + const double va = d3*d6 - d5*d4; + if (va <= 0 && (d4-d3) >= 0 && (d5-d6) >= 0) + return vadd(b, vscale(vsub(c, b), (d4-d3)/((d4-d3)+(d5-d6)))); + const double denom = 1.0/(va+vb+vc); + return vadd(a, vadd(vscale(ab, vb*denom), vscale(ac, vc*denom))); +} + +// ---- a simple median-split KD-tree over triangle centroids ----------------- +struct KDTree { + std::vector pts; + std::vector idx; + struct Node { int axis=-1; double split=0; int lo=-1, hi=-1, start=0, count=0; }; + std::vector nodes; + + void build(const std::vector& centroids) + { + pts = centroids; + idx.resize(pts.size()); + for (size_t i = 0; i < idx.size(); ++i) idx[i] = int(i); + nodes.clear(); + buildRange(0, int(idx.size())); + } + int buildRange(int start, int count) + { + const int self = int(nodes.size()); + nodes.push_back({}); + if (count <= 8) { nodes[self] = {-1, 0, -1, -1, start, count}; return self; } + Vec3 mn = pts[idx[start]], mx = pts[idx[start]]; + for (int k = 0; k < count; ++k) { + const Vec3& p = pts[idx[start+k]]; + for (int a = 0; a < 3; ++a) { mn[a] = std::min(mn[a], p[a]); mx[a] = std::max(mx[a], p[a]); } + } + int axis = 0; double ext = mx[0]-mn[0]; + for (int a = 1; a < 3; ++a) if (mx[a]-mn[a] > ext) { ext = mx[a]-mn[a]; axis = a; } + const int mid = start + count/2; + std::nth_element(idx.begin()+start, idx.begin()+mid, idx.begin()+start+count, + [&](int x, int y){ return pts[x][axis] < pts[y][axis]; }); + const double split = pts[idx[mid]][axis]; + const int lo = buildRange(start, mid-start); + const int hi = buildRange(mid, start+count-mid); + nodes[self] = {axis, split, lo, hi, 0, 0}; + return self; + } + // nearest centroid index (broad phase); refine to real distance by caller + int nearest(const Vec3& q) const + { + int best = -1; double bestD2 = std::numeric_limits::max(); + nearestRec(0, q, best, bestD2); + return best; + } + void nearestRec(int n, const Vec3& q, int& best, double& bestD2) const + { + const Node& nd = nodes[n]; + if (nd.axis < 0) { + for (int k = 0; k < nd.count; ++k) { + const int pi = idx[nd.start+k]; + const Vec3 d = vsub(pts[pi], q); + const double d2 = vdot(d, d); + if (d2 < bestD2) { bestD2 = d2; best = pi; } + } + return; + } + const double diff = q[nd.axis] - nd.split; + const int near = diff < 0 ? nd.lo : nd.hi; + const int far = diff < 0 ? nd.hi : nd.lo; + nearestRec(near, q, best, bestD2); + if (diff*diff < bestD2) nearestRec(far, q, best, bestD2); + } + // K nearest centroid indices (broad phase). The caller runs the exact + // point-triangle test over these — a large sliver triangle's true surface + // can be closer than the triangle whose CENTROID is closest, so a single + // centroid winner picks the wrong correspondence on non-uniform meshes. + void nearestK(const Vec3& q, int K, std::vector& out) const + { + out.clear(); + std::vector> heap; // max-heap by distance + nearestKRec(0, q, K, heap); + out.reserve(heap.size()); + for (const auto& [d2, pi] : heap) out.push_back(pi); + } + void nearestKRec(int n, const Vec3& q, int K, + std::vector>& heap) const + { + const Node& nd = nodes[n]; + if (nd.axis < 0) { + for (int k = 0; k < nd.count; ++k) { + const int pi = idx[nd.start+k]; + const Vec3 d = vsub(pts[pi], q); + const double d2 = vdot(d, d); + if (int(heap.size()) < K) { + heap.emplace_back(d2, pi); + std::push_heap(heap.begin(), heap.end()); + } else if (d2 < heap.front().first) { + std::pop_heap(heap.begin(), heap.end()); + heap.back() = {d2, pi}; + std::push_heap(heap.begin(), heap.end()); + } + } + return; + } + const double diff = q[nd.axis] - nd.split; + const int near = diff < 0 ? nd.lo : nd.hi; + const int far = diff < 0 ? nd.hi : nd.lo; + nearestKRec(near, q, K, heap); + const double worstNow = int(heap.size()) < K + ? std::numeric_limits::max() : heap.front().first; + if (diff*diff < worstNow) + nearestKRec(far, q, K, heap); + } +}; + +// ---- CSR sparse matrix + CG on the normal equations (AᵀA x = Aᵀb) ---------- +// A is (rows x cols); we never form AᵀA — CG multiplies by A then Aᵀ. +struct Sparse { + int rows = 0, cols = 0; + std::vector rowPtr; // size rows+1 + std::vector col; + std::vector val; + + // build from triplets (row-major). triplets need not be unique/sorted. + void fromTriplets(int r, int c, std::vector>& trip) + { + rows = r; cols = c; + std::vector cnt(r+1, 0); + for (auto& t : trip) cnt[int(t[0])+1]++; + for (int i = 0; i < r; ++i) cnt[i+1] += cnt[i]; + rowPtr = cnt; + col.resize(trip.size()); val.resize(trip.size()); + std::vector cur = rowPtr; + for (auto& t : trip) { + const int rr = int(t[0]); + const int dst = cur[rr]++; + col[dst] = int(t[1]); val[dst] = t[2]; + } + } + // y = A x (x size cols, y size rows) + void mul(const std::vector& x, std::vector& y) const + { + y.assign(rows, 0.0); + for (int r = 0; r < rows; ++r) { + double s = 0; + for (int k = rowPtr[r]; k < rowPtr[r+1]; ++k) s += val[k]*x[col[k]]; + y[r] = s; + } + } + // y = Aᵀ x (x size rows, y size cols) + void mulT(const std::vector& x, std::vector& y) const + { + y.assign(cols, 0.0); + for (int r = 0; r < rows; ++r) { + const double xr = x[r]; + for (int k = rowPtr[r]; k < rowPtr[r+1]; ++k) y[col[k]] += val[k]*xr; + } + } +}; + +// solve min ‖A x - b‖² by CG on the normal equations, warm-started at x0. +void cgnr(const Sparse& A, const std::vector& b, + std::vector& x, int maxIters, double tol) +{ + std::vector Ax, r(A.cols), p, Ap, AtAp, tmp; + A.mul(x, Ax); + std::vector resid(A.rows); + for (int i = 0; i < A.rows; ++i) resid[i] = b[i] - Ax[i]; + A.mulT(resid, r); // r = Aᵀ(b - Ax) + p = r; + double rs = 0; for (double v : r) rs += v*v; + const double rs0 = rs; + for (int it = 0; it < maxIters && rs > tol*tol*rs0; ++it) { + A.mul(p, Ap); + A.mulT(Ap, AtAp); // AtAp = AᵀA p + double pAp = 0; for (int i = 0; i < A.cols; ++i) pAp += p[i]*AtAp[i]; + if (pAp <= 1e-30) break; + const double a = rs / pAp; + for (int i = 0; i < A.cols; ++i) { x[i] += a*p[i]; r[i] -= a*AtAp[i]; } + double rsn = 0; for (double v : r) rsn += v*v; + const double beta = rsn / rs; + for (int i = 0; i < A.cols; ++i) p[i] = r[i] + beta*p[i]; + rs = rsn; + } +} + +} // namespace + +NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, + const std::vector& userV, const std::vector& userF, + const NricpOptions& opts, + const NricpProgressFn& progress) +{ + NricpResult res; + const int Nt = int(tmplV.size()/3); + const int Fu = int(userF.size()/3); + if (Nt < 3 || Fu < 1 || userV.size() < 9) + return res; + bool aborted = false; + // Reject malformed face buffers at the public boundary — indices are + // dereferenced when building triangles/edges below. + const int Nu = int(userV.size()/3); + for (int idx : userF) + if (idx < 0 || idx >= Nu) return res; + for (int idx : tmplF) + if (idx < 0 || idx >= Nt) return res; + + // user bbox diagonal + Vec3 mn = at(userV,0), mx = mn; + for (int i = 1; i < int(userV.size()/3); ++i) { + const Vec3 p = at(userV, i); + for (int a = 0; a < 3; ++a) { mn[a] = std::min(mn[a], p[a]); mx[a] = std::max(mx[a], p[a]); } + } + res.diag = std::sqrt(vdot(vsub(mx,mn), vsub(mx,mn))); + + // rigid pre-align: centroid + bbox-scale (correspondence-free) + Vec3 tmn = at(tmplV,0), tmx = tmn, tc{0,0,0}, uc{0,0,0}; + for (int i = 0; i < Nt; ++i) { + const Vec3 p = at(tmplV,i); tc = vadd(tc,p); + for (int a=0;a<3;++a){ tmn[a]=std::min(tmn[a],p[a]); tmx[a]=std::max(tmx[a],p[a]); } + } + tc = vscale(tc, 1.0/Nt); + for (int i = 0; i < Nu; ++i) uc = vadd(uc, at(userV,i)); + uc = vscale(uc, 1.0/Nu); + const double tdiag = std::sqrt(vdot(vsub(tmx,tmn), vsub(tmx,tmn))); + const double s = tdiag > 1e-9 ? res.diag/tdiag : 1.0; + + // template homogeneous verts and current fitted positions X + std::vector vhat(Nt), X(Nt); + for (int i = 0; i < Nt; ++i) { + vhat[i] = at(tmplV, i); + X[i] = vadd(vscale(vsub(vhat[i], tc), s), uc); // pre-aligned start + } + + // user triangle geometry + centroid KD-tree + std::vector> utri(Fu); + std::vector ucent(Fu); + for (int f = 0; f < Fu; ++f) { + const Vec3 a = at(userV, userF[f*3]); + const Vec3 b = at(userV, userF[f*3+1]); + const Vec3 c = at(userV, userF[f*3+2]); + utri[f] = {a,b,c}; + ucent[f] = vscale(vadd(vadd(a,b),c), 1.0/3.0); + } + KDTree tree; tree.build(ucent); + + // template edges (unique) + std::vector> edges; + { + std::vector> e; + const int Ft = int(tmplF.size()/3); + e.reserve(Ft*3); + for (int f = 0; f < Ft; ++f) { + const int a=tmplF[f*3], b=tmplF[f*3+1], c=tmplF[f*3+2]; + for (auto pr : {std::array{a,b}, {b,c}, {c,a}}) { + int lo = std::min(pr[0],pr[1]), hi = std::max(pr[0],pr[1]); + e.push_back({lo,hi}); + } + } + std::sort(e.begin(), e.end()); + e.erase(std::unique(e.begin(), e.end()), e.end()); + for (auto& pr : e) edges.push_back({pr[0], pr[1]}); + } + const int E = int(edges.size()); + + // unknown layout: 12 per vertex (3x4 affine, row-major a00..a03,a10..,a20..) + // X_i = A_i * [vhat_i; 1]. We solve 3 independent systems (one per output + // coordinate), each with 4*Nt unknowns (the 4 affine coeffs mapping to that + // coord for every vertex), sharing the SAME sparse matrix. + const int cols = 4*Nt; + + const int levelCount = int(opts.stiffness.size()); + int levelIdx = 0; + for (double alpha : opts.stiffness) { + for (int iter = 0; iter < opts.itersPerLevel; ++iter) { + // find closest surface point per current X_i: broad-phase K + // nearest centroids, then the EXACT point-triangle distance picks + // among them — the centroid-nearest triangle alone mis-corresponds + // next to large/sliver triangles on non-uniform meshes. + std::vector target(Nt); + std::vector cand; + for (int i = 0; i < Nt; ++i) { + tree.nearestK(X[i], 4, cand); + double bestD2 = std::numeric_limits::max(); + Vec3 bestP{0,0,0}; + for (int cf : cand) { + const Vec3 p = closestPointTriangle( + X[i], utri[cf][0], utri[cf][1], utri[cf][2]); + const Vec3 d = vsub(p, X[i]); + const double d2 = vdot(d, d); + if (d2 < bestD2) { bestD2 = d2; bestP = p; } + } + target[i] = cand.empty() + ? X[i] + : bestP; + } + + // Landmark anchors that reference a valid template vertex. Weight + // rides alpha so it dominates while the fit is still rigid (locking + // orientation/scale), then relaxes as alpha anneals down — AND + // decays linearly to ZERO by the finest level, so anchors act as + // an INITIALIZATION, not a hard constraint. User-placed markers + // are never pixel-perfect; without the decay a slightly-off mark + // outweighs the surface term at the end of the anneal and drags + // its neighbourhood off the mesh (spiky eyelid/lip deltas). The + // final levels are pure surface snapping from the marker-steered + // coarse alignment. + std::vector lms; + lms.reserve(opts.landmarks.size()); + for (const auto& lm : opts.landmarks) + if (lm.tmplVertex >= 0 && lm.tmplVertex < Nt) + lms.push_back(&lm); + const int L = int(lms.size()); + const double anneal = levelCount > 1 + ? double(levelCount - 1 - levelIdx) / double(levelCount - 1) + : 1.0; + const double lw = opts.landmarkWeight * alpha * anneal; + + // assemble A (rows = Nt data + E stiffness + L landmark) once; + // rhs differs per axis + std::vector> trip; + trip.reserve(size_t(Nt)*4 + size_t(E)*8 + size_t(L)*4); + // data rows: row i uses cols [4i..4i+3] with [vx,vy,vz,1] + for (int i = 0; i < Nt; ++i) { + trip.push_back({double(i), double(4*i+0), vhat[i][0]}); + trip.push_back({double(i), double(4*i+1), vhat[i][1]}); + trip.push_back({double(i), double(4*i+2), vhat[i][2]}); + trip.push_back({double(i), double(4*i+3), 1.0}); + } + // stiffness rows: for edge (i,j), 4 rows (one per affine coeff) + // alpha*(A_i[k]-A_j[k]) = 0 + for (int e = 0; e < E; ++e) { + const int i = edges[e].first, j = edges[e].second; + for (int k = 0; k < 4; ++k) { + const int row = Nt + e*4 + k; + trip.push_back({double(row), double(4*i+k), alpha}); + trip.push_back({double(row), double(4*j+k), -alpha}); + } + } + // landmark rows: lw*[vx,vy,vz,1]·A_L = lw*target[axis] (rhs per axis) + for (int l = 0; l < L; ++l) { + const int i = lms[l]->tmplVertex; + const int row = Nt + E*4 + l; + trip.push_back({double(row), double(4*i+0), lw*vhat[i][0]}); + trip.push_back({double(row), double(4*i+1), lw*vhat[i][1]}); + trip.push_back({double(row), double(4*i+2), lw*vhat[i][2]}); + trip.push_back({double(row), double(4*i+3), lw}); + } + Sparse A; + A.fromTriplets(Nt + E*4 + L, cols, trip); + + std::vector b(A.rows, 0.0); + std::vector x(cols, 0.0); + for (int axis = 0; axis < 3; ++axis) { + for (int i = 0; i < Nt; ++i) b[i] = target[i][axis]; + // stiffness rhs stays 0 + for (int l = 0; l < L; ++l) + b[Nt + E*4 + l] = lw * double(lms[l]->target[size_t(axis)]); + // warm start x from the current affine estimate for this axis + for (int i = 0; i < Nt; ++i) { + // recover current A_i row for this axis from X_i & vhat_i is + // non-trivial; a zero start with identity bias works well + x[4*i+0] = 0; x[4*i+1] = 0; x[4*i+2] = 0; x[4*i+3] = X[i][axis]; + x[4*i+axis] = 1.0; // identity-ish seed + } + cgnr(A, b, x, opts.cgIters, opts.cgTol); + for (int i = 0; i < Nt; ++i) { + X[i][axis] = x[4*i+0]*vhat[i][0] + x[4*i+1]*vhat[i][1] + + x[4*i+2]*vhat[i][2] + x[4*i+3]; + } + } + } + ++levelIdx; + if (progress && !progress(levelIdx, levelCount)) { + // caller aborted — bail with the best fit so far, and force + // ok=false regardless of how the residuals turn out (the + // finite-count check alone can read a half-finished fit as ok). + aborted = true; + break; + } + } + + // final residuals to the user surface. A vertex whose affine solve + // diverged (NaN/inf — e.g. degenerate fan triangles at a UV-sphere pole) + // must not poison the mean: count it, skip it from the average, but still + // surface it so callers can gate (a well-behaved face mesh produces none). + res.fitted.resize(size_t(Nt)*3); + res.residual.resize(Nt); + double sum = 0, mx2 = 0; + int finiteCount = 0, divergedCount = 0; + for (int i = 0; i < Nt; ++i) { + const bool finite = std::isfinite(X[i][0]) && std::isfinite(X[i][1]) && + std::isfinite(X[i][2]); + double d = 0.0; + if (finite) { + const int cf = tree.nearest(X[i]); + const Vec3 cp = closestPointTriangle(X[i], utri[cf][0], utri[cf][1], utri[cf][2]); + d = std::sqrt(vdot(vsub(X[i],cp), vsub(X[i],cp))); + } + if (finite && std::isfinite(d)) { + res.residual[i] = float(d); + sum += d; mx2 = std::max(mx2, d); + finiteCount++; + } else { + res.residual[i] = std::numeric_limits::infinity(); + divergedCount++; + } + res.fitted[size_t(i)*3+0] = float(X[i][0]); + res.fitted[size_t(i)*3+1] = float(X[i][1]); + res.fitted[size_t(i)*3+2] = float(X[i][2]); + } + res.meanResidual = finiteCount > 0 ? sum/finiteCount + : std::numeric_limits::infinity(); + // a large diverged fraction means the fit failed — reflect it in maxResidual + // (which callers already gate on) so a mostly-NaN fit can't read as "great". + res.maxResidual = (divergedCount > Nt / 20) // > 5% diverged + ? std::numeric_limits::infinity() + : mx2; + res.ok = !aborted && finiteCount > Nt / 2; // half the verts + not cancelled + return res; +} + +} // namespace FaceRig diff --git a/src/FaceRig/NonRigidICP.h b/src/FaceRig/NonRigidICP.h new file mode 100644 index 000000000..7d63a4beb --- /dev/null +++ b/src/FaceRig/NonRigidICP.h @@ -0,0 +1,83 @@ +#ifndef NONRIGIDICP_H +#define NONRIGIDICP_H + +// Non-rigid ICP (Amberg, Romdhani & Vetter 2007, "Optimal Step Nonrigid ICP") +// for face auto-rig (#889, Slice C #891). Pure data — no Ogre, no ONNX, no +// external linear-algebra dependency (a self-contained sparse CG solver lives +// in the .cpp) — and headless-unit-tested. +// +// Fits a TEMPLATE mesh (the ICT ARKit head, ArkitTemplate) to a USER neutral +// head of arbitrary topology, producing per-template-vertex positions that lie +// on the user surface: the CORRESPONDENCE the deformation transfer (#892) +// needs. Each template vertex gets a 3x4 affine A_i; we minimize +// +// Σ_i ‖A_i·ṽ_i − closest_point_on_user(A_i·ṽ_i)‖² (data) +// + α Σ_(i,j)∈edges ‖A_i − A_j‖² (stiffness) +// +// over an annealed stiffness schedule α (high→low). closest_point is a +// point-to-triangle projection against the user mesh (KD-tree over triangle +// centroids for the broad phase). Contract proven in docs/FACE_RIG_SPIKE.md. + +#include +#include +#include +#include + +namespace FaceRig { + +struct NricpResult { + // fitted template vertex positions on the user surface (templateVertexCount + // * 3, xyz interleaved) — the correspondence. + std::vector fitted; + // per-template-vertex residual distance to the user surface (in mesh units) + std::vector residual; + double meanResidual = 0.0; // over all template verts + double maxResidual = 0.0; + double diag = 0.0; // user-mesh bounding-box diagonal (for %) + bool ok = false; +}; + +// A landmark correspondence: template vertex `tmplVertex` should map onto the +// user-space position `target`. Feeding a handful of these (from facial-landmark +// detection) ANCHORS the fit so it can't converge to a low-residual but +// mis-oriented/mis-scaled drape — the fix for the ARKit template landing on the +// wrong face features. (#889) +struct NricpLandmark { + int tmplVertex = -1; + std::array target{0, 0, 0}; +}; + +struct NricpOptions { + // annealed stiffness weights (high = rigid, low = free); ~3 inner iters each + std::vector stiffness = {50, 20, 8, 3, 1, 0.5}; + int itersPerLevel = 3; + int cgIters = 400; // CG cap per axis solve + double cgTol = 1e-6; + // Optional landmark anchors (template vertex → user position). Added as + // high-weight data rows; the weight is strongest at the rigid (high-alpha) + // levels so orientation/scale lock first, then relaxes as the fit refines. + std::vector landmarks; + // Base anchor weight (× alpha at each level). High enough that the marked + // features stay PINNED through the fine anneal levels — at 10 the anchors + // loosened once alpha dropped below 1 and the fitted lip line drifted a few + // mm below the marked lips (field-observed offset). + double landmarkWeight = 30.0; +}; + +// Progress callback for the annealing loop: (level, levelCount). Return false +// to abort the fit early (fit() then returns the best-so-far with ok=false). +using NricpProgressFn = std::function; + +// tmplV/tmplF: template neutral verts (Nt*3) + tris (Ft*3). +// userV/userF: user neutral verts (Nu*3) + tris (Fu*3). +// Both assumed roughly aligned in orientation (+Y up); the fit does a +// centroid+bbox-scale rigid pre-align, then the non-rigid warp. +// `progress` (optional) fires once per completed stiffness level. +NricpResult fit(const std::vector& tmplV, const std::vector& tmplF, + const std::vector& userV, const std::vector& userF, + const NricpOptions& opts = {}, + const NricpProgressFn& progress = {}); + +} // namespace FaceRig + +#endif // NONRIGIDICP_H diff --git a/src/FaceRig/NonRigidICP_test.cpp b/src/FaceRig/NonRigidICP_test.cpp new file mode 100644 index 000000000..800c0c11c --- /dev/null +++ b/src/FaceRig/NonRigidICP_test.cpp @@ -0,0 +1,127 @@ +#include + +#include "FaceRig/NonRigidICP.h" + +#include +#include +#include +#include + +namespace { + +// A small closed-ish grid surface (a bumpy plane) as a stand-in mesh: a +// (n x n) vertex grid triangulated, so it has real edges/faces for NRICP. +struct Grid { + std::vector V; + std::vector F; +}; + +Grid makeGrid(int n, float extent, float bump = 0.0f, float dx = 0.0f) +{ + Grid g; + for (int y = 0; y < n; ++y) + for (int x = 0; x < n; ++x) { + const float fx = (float(x)/(n-1) - 0.5f) * extent + dx; + const float fy = (float(y)/(n-1) - 0.5f) * extent; + const float fz = bump * std::sin(float(x)) * std::cos(float(y)); + g.V.insert(g.V.end(), {fx, fy, fz}); + } + for (int y = 0; y < n-1; ++y) + for (int x = 0; x < n-1; ++x) { + const int a = y*n+x, b = y*n+x+1, c = (y+1)*n+x, d = (y+1)*n+x+1; + g.F.insert(g.F.end(), {a, b, c}); + g.F.insert(g.F.end(), {b, d, c}); + } + return g; +} + +double maxAbs(const std::vector& v) +{ + double m = 0; + for (float x : v) m = std::max(m, double(std::abs(x))); + return m; +} + +} // namespace + +TEST(NonRigidICP, FitToIdenticalMeshIsNearIdentity) +{ + const Grid g = makeGrid(10, 2.0f); + const auto res = FaceRig::fit(g.V, g.F, g.V, g.F); + ASSERT_TRUE(res.ok); + ASSERT_EQ(res.fitted.size(), g.V.size()); + // fitting a mesh onto ITSELF: fitted verts land on the surface (residual ~0) + EXPECT_LT(res.meanResidual / res.diag, 0.02); // < 2% of diag + // and stay close to their original positions + double maxMove = 0; + for (size_t i = 0; i < g.V.size(); ++i) + maxMove = std::max(maxMove, double(std::abs(res.fitted[i] - g.V[i]))); + EXPECT_LT(maxMove / res.diag, 0.1); +} + +TEST(NonRigidICP, FitsOntoTranslatedTarget) +{ + const Grid tmpl = makeGrid(10, 2.0f); + // user = same surface shifted +0.5 in X (a different "identity") + Grid user = makeGrid(10, 2.0f, 0.0f, 0.5f); + const auto res = FaceRig::fit(tmpl.V, tmpl.F, user.V, user.F); + ASSERT_TRUE(res.ok); + // the fitted template should end up on the shifted plane (low residual) + EXPECT_LT(res.meanResidual / res.diag, 0.05); + // and its mean X should have moved toward the user's (+0.5) + double mx = 0; int n = int(res.fitted.size()/3); + for (int i = 0; i < n; ++i) mx += res.fitted[i*3]; + mx /= n; + EXPECT_GT(mx, 0.2); // shifted from ~0 toward +0.5 +} + +TEST(NonRigidICP, FitsOntoDifferentTopologyTarget) +{ + // template 10x10, user 14x14 of the same surface — different vert counts + const Grid tmpl = makeGrid(10, 2.0f, 0.15f); + const Grid user = makeGrid(14, 2.0f, 0.15f); + const auto res = FaceRig::fit(tmpl.V, tmpl.F, user.V, user.F); + ASSERT_TRUE(res.ok); + EXPECT_EQ(int(res.fitted.size()/3), int(tmpl.V.size()/3)); + EXPECT_LT(res.meanResidual / res.diag, 0.05); // fits despite topology gap + for (float x : res.fitted) EXPECT_TRUE(std::isfinite(x)); +} + +TEST(NonRigidICP, BoundedOnPerturbedTarget) +{ + const Grid tmpl = makeGrid(10, 2.0f); + Grid user = makeGrid(10, 2.0f); + std::mt19937 rng(7); + std::normal_distribution jitter(0.f, 0.03f); + for (auto& v : user.V) v += jitter(rng); + const auto res = FaceRig::fit(tmpl.V, tmpl.F, user.V, user.F); + ASSERT_TRUE(res.ok); + // stiffness keeps the fit from chasing every jitter spike + EXPECT_LT(res.maxResidual / res.diag, 0.2); + for (float x : res.fitted) EXPECT_TRUE(std::isfinite(x)); +} + +TEST(NonRigidICP, DegenerateInputDoesNotCrashOrNaN) +{ + const Grid g = makeGrid(6, 1.0f); + // empty template + EXPECT_FALSE(FaceRig::fit({}, {}, g.V, g.F).ok); + // empty user + EXPECT_FALSE(FaceRig::fit(g.V, g.F, {}, {}).ok); + // user with a single degenerate (zero-area) triangle + std::vector uv = {0,0,0, 0,0,0, 0,0,0}; + std::vector uf = {0,1,2}; + const auto res = FaceRig::fit(g.V, g.F, uv, uf); + // may or may not be "ok" but must never produce NaN + for (float x : res.fitted) EXPECT_TRUE(std::isfinite(x)); +} + +TEST(NonRigidICP, ReportsResidualAndDiag) +{ + const Grid g = makeGrid(8, 3.0f); + const auto res = FaceRig::fit(g.V, g.F, g.V, g.F); + ASSERT_TRUE(res.ok); + EXPECT_GT(res.diag, 0.0); + EXPECT_EQ(int(res.residual.size()), int(g.V.size()/3)); + EXPECT_GE(res.maxResidual, res.meanResidual); +} diff --git a/src/FaceRig/SparseSolve.cpp b/src/FaceRig/SparseSolve.cpp new file mode 100644 index 000000000..aba1df4af --- /dev/null +++ b/src/FaceRig/SparseSolve.cpp @@ -0,0 +1,87 @@ +#include "SparseSolve.h" + +namespace FaceRig { + +void SparseMatrix::fromTriplets(int r, int c, + const std::vector>& trip) +{ + rows = r; + cols = c; + std::vector cnt(size_t(r) + 1, 0); + for (const auto& t : trip) + cnt[size_t(t[0]) + 1]++; + for (int i = 0; i < r; ++i) + cnt[size_t(i) + 1] += cnt[size_t(i)]; + m_rowPtr = cnt; + m_col.resize(trip.size()); + m_val.resize(trip.size()); + std::vector cur = m_rowPtr; + for (const auto& t : trip) { + const int rr = int(t[0]); + const int dst = cur[size_t(rr)]++; + m_col[size_t(dst)] = int(t[1]); + m_val[size_t(dst)] = t[2]; + } +} + +void SparseMatrix::mul(const std::vector& x, std::vector& y) const +{ + y.assign(size_t(rows), 0.0); + for (int r = 0; r < rows; ++r) { + double s = 0; + for (int k = m_rowPtr[size_t(r)]; k < m_rowPtr[size_t(r) + 1]; ++k) + s += m_val[size_t(k)] * x[size_t(m_col[size_t(k)])]; + y[size_t(r)] = s; + } +} + +void SparseMatrix::mulT(const std::vector& x, std::vector& y) const +{ + y.assign(size_t(cols), 0.0); + for (int r = 0; r < rows; ++r) { + const double xr = x[size_t(r)]; + for (int k = m_rowPtr[size_t(r)]; k < m_rowPtr[size_t(r) + 1]; ++k) + y[size_t(m_col[size_t(k)])] += m_val[size_t(k)] * xr; + } +} + +void solveLeastSquaresCG(const SparseMatrix& A, const std::vector& b, + std::vector& x, int maxIters, double tol) +{ + std::vector Ax, r, p, Ap, AtAp; + A.mul(x, Ax); + std::vector resid(size_t(A.rows)); + for (int i = 0; i < A.rows; ++i) + resid[size_t(i)] = b[size_t(i)] - Ax[size_t(i)]; + A.mulT(resid, r); // r = Aᵀ(b - Ax) + p = r; + double rs = 0; + for (double v : r) + rs += v * v; + const double rs0 = rs; + if (rs0 <= 0.0) + return; + for (int it = 0; it < maxIters && rs > tol * tol * rs0; ++it) { + A.mul(p, Ap); + A.mulT(Ap, AtAp); // AtAp = AᵀA p + double pAp = 0; + for (int i = 0; i < A.cols; ++i) + pAp += p[size_t(i)] * AtAp[size_t(i)]; + if (pAp <= 1e-30) + break; + const double a = rs / pAp; + for (int i = 0; i < A.cols; ++i) { + x[size_t(i)] += a * p[size_t(i)]; + r[size_t(i)] -= a * AtAp[size_t(i)]; + } + double rsn = 0; + for (double v : r) + rsn += v * v; + const double beta = rsn / rs; + for (int i = 0; i < A.cols; ++i) + p[size_t(i)] = r[size_t(i)] + beta * p[size_t(i)]; + rs = rsn; + } +} + +} // namespace FaceRig diff --git a/src/FaceRig/SparseSolve.h b/src/FaceRig/SparseSolve.h new file mode 100644 index 000000000..ef8f6b1f7 --- /dev/null +++ b/src/FaceRig/SparseSolve.h @@ -0,0 +1,39 @@ +#ifndef FACERIG_SPARSESOLVE_H +#define FACERIG_SPARSESOLVE_H + +// Minimal dependency-free sparse linear algebra for the face-rig native +// solvers. A CSR matrix built from triplets + conjugate-gradient on the normal +// equations (AᵀA x = Aᵀb) — no Eigen, no external solver. Pure data, +// headless-tested. Used by DeformationTransfer (#892); NonRigidICP (#891) still +// carries its own equivalent copy and can be migrated onto this in a follow-up. + +#include +#include + +namespace FaceRig { + +class SparseMatrix { +public: + int rows = 0, cols = 0; + + // Build from triplets {row, col, value}; duplicates are summed on multiply + // (CG only needs the products, so we keep them un-coalesced). + void fromTriplets(int r, int c, const std::vector>& trip); + + void mul(const std::vector& x, std::vector& y) const; // y = A x + void mulT(const std::vector& x, std::vector& y) const; // y = Aᵀ x + +private: + std::vector m_rowPtr, m_col; + std::vector m_val; +}; + +// Solve min ‖A x - b‖² by CG on the normal equations, warm-started at x +// (in/out). b has A.rows entries, x has A.cols. +void solveLeastSquaresCG(const SparseMatrix& A, const std::vector& b, + std::vector& x, int maxIters = 400, + double tol = 1e-6); + +} // namespace FaceRig + +#endif // FACERIG_SPARSESOLVE_H diff --git a/src/FaceRigController.cpp b/src/FaceRigController.cpp new file mode 100644 index 000000000..8d6874f23 --- /dev/null +++ b/src/FaceRigController.cpp @@ -0,0 +1,632 @@ +#include "FaceRigController.h" + +#include "FaceRig/ArkitTemplate.h" +#include "FaceRig/FaceRigAttach.h" +#include "FaceRig/FaceRigLandmarks.h" +#include "GamificationManager.h" +#include "Manager.h" +#include "OgreWidget.h" +#include "SelectionSet.h" +#include "SpaceCamera.h" +#include "SentryReporter.h" +#include "UndoManager.h" +#include "commands/MorphCommands.h" + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +FaceRigController* FaceRigController::m_pSingleton = nullptr; + +FaceRigController* FaceRigController::instance() +{ + if (!m_pSingleton) + m_pSingleton = new FaceRigController(); + return m_pSingleton; +} + +FaceRigController* FaceRigController::qmlInstance(QQmlEngine* engine, QJSEngine*) +{ + Q_UNUSED(engine); + auto* inst = instance(); + QQmlEngine::setObjectOwnership(inst, QQmlEngine::CppOwnership); + return inst; +} + +void FaceRigController::kill() +{ + delete m_pSingleton; + m_pSingleton = nullptr; +} + +FaceRigController::FaceRigController() : QObject(nullptr) +{ + connect(SelectionSet::getSingleton(), &SelectionSet::selectionChanged, + this, [this]() { + // Selecting a DIFFERENT model must reset the face-rig controls: an + // active marker session (chips, overlays, Rig-from-markers state) + // belongs to the entity it was started on, and a stale status line + // from the previous rig reads as if it applied to the new selection. + if (m_markerMode) { + auto* sel = SelectionSet::getSingleton(); + const auto entities = sel ? sel->getResolvedEntities() + : QList{}; + Ogre::Entity* first = entities.isEmpty() ? nullptr : entities.first(); + if (!first || first->getName() != m_markerEntityName) + cancelFaceMarkers(); + } else if (!m_busy) { + setStatus(QString()); + } + emit selectionChanged(); + }); +} + +void FaceRigController::setStatus(const QString& s) +{ + if (m_status == s) return; + m_status = s; + emit statusChanged(); +} + +bool FaceRigController::hasMeshSelection() const +{ + auto* sel = SelectionSet::getSingleton(); + if (!sel) return false; + const auto entities = sel->getResolvedEntities(); + if (entities.isEmpty()) return false; + Ogre::Entity* first = entities.first(); + return first && first->getMesh(); +} + +bool FaceRigController::addArkitBlendshapesAsync(int maxShapes, double maxResidualPct, + double amplitude) +{ + if (m_busy) { + emit error(QStringLiteral("A face-rig is already running.")); + return false; + } + SentryReporter::addBreadcrumb(QStringLiteral("ui.action"), + QStringLiteral("Add ARKit Blendshapes requested")); + + auto* sel = SelectionSet::getSingleton(); + const auto entities = sel ? sel->getResolvedEntities() + : QList{}; + Ogre::Entity* entity = entities.isEmpty() ? nullptr : entities.first(); + if (!entity || !entity->getMesh()) { + emit error(QStringLiteral("No mesh selected.")); + return false; + } + + // MAIN thread: read the entity geometry (locks Ogre buffers — ms). + auto geo = std::make_shared( + FaceRig::extractGeometry(entity)); + if (!geo->valid()) { + emit error(QStringLiteral("Could not read the mesh geometry.")); + return false; + } + + // MAIN thread: ensure + load the bundled template (may download on first + // use — surface that to the UI). ensureModelBlocking() can take a while on + // a first-run download, so show "Downloading…" first. + m_downloading = !FaceRig::ArkitTemplate::present(); + setStatus(m_downloading ? QStringLiteral("Downloading face template…") + : QStringLiteral("Preparing…")); + const QString path = FaceRig::ArkitTemplate::ensureModelBlocking(); + m_downloading = false; + if (path.isEmpty()) { + setStatus(QString()); + emit error(QStringLiteral( + "ARKit template unavailable (offline and not yet downloaded, or " + "this build has no face-rig model).")); + return false; + } + auto tmpl = std::make_shared(); + QString loadErr; + if (!tmpl->load(path, &loadErr)) { + setStatus(QString()); + emit error(QStringLiteral("Failed to load ARKit template: %1").arg(loadErr)); + return false; + } + + // MAIN thread: facial-landmark anchors (renders template + user — Ogre) so + // the worker's fit lands on the real face features. Empty when ONNX/model/ + // face-detection unavailable → the fit runs unanchored (previous behaviour). + setStatus(QStringLiteral("Detecting face landmarks…")); + std::vector headV; std::vector headF; + FaceRig::headSubmesh(*geo, headV, headF); + const std::vector anchors = + FaceRig::buildLandmarkAnchors(entity, headV, headF, *tmpl); + + m_geo = geo; + return runRigAsync(tmpl, maxShapes, maxResidualPct, amplitude, anchors); +} + +bool FaceRigController::runRigAsync( + const std::shared_ptr& tmpl, + int maxShapes, double maxResidualPct, double amplitude, + const std::vector& anchorsIn) +{ + auto geo = m_geo; + if (!geo || !geo->valid()) { + emit error(QStringLiteral("Could not read the mesh geometry.")); + return false; + } + auto* sel = SelectionSet::getSingleton(); + const auto entities = sel ? sel->getResolvedEntities() + : QList{}; + Ogre::Entity* entity = entities.isEmpty() ? nullptr : entities.first(); + if (!entity) { emit error(QStringLiteral("No mesh selected.")); return false; } + + FaceRig::FaceRigOptions opts; + opts.maxShapes = maxShapes; + opts.maxFitResidualPct = maxResidualPct; + opts.amplitude = amplitude; + auto anchors = + std::make_shared>(anchorsIn); + + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.face_rig"), + QStringLiteral("face_rig entity=%1 verts=%2 template=%3v anchors=%4") + .arg(QString::fromStdString(entity->getName())) + .arg(geo->userV.size() / 3).arg(tmpl->vertexCount()) + .arg(anchors->size())); + + m_busy = true; + emit busyChanged(); + m_progress = 0; + m_progressTotal = 0; + emit progressChanged(); + setStatus(QStringLiteral("Fitting face template…")); + + m_cancel = std::make_shared(false); + auto cancel = m_cancel; + const std::string entName = entity->getName(); + QPointer self(this); + + // WORKER thread: the heavy Ogre-free fit + transfer over all 52 shapes. + std::thread([self, geo, tmpl, opts, entName, cancel, anchors]() { + // Progress callback: marshal the counters to the main thread for the + // progress bar; return false to stop the worker when cancel is set. + auto progress = [self, cancel](int done, int total, + const char* phase) -> bool { + if (cancel->load()) return false; + const QString ph = QString::fromUtf8(phase); + QMetaObject::invokeMethod(qApp, [self, done, total, ph]() { + if (!self) return; + self->m_progress = done; + self->m_progressTotal = total; + emit self->progressChanged(); + self->setStatus(ph); + }, Qt::QueuedConnection); + return true; + }; + auto result = std::make_shared( + FaceRig::buildFaceRig(geo->userV, geo->userF, *tmpl, opts, + geo->headMask, *anchors, progress)); + + // MAIN thread: attach (touches Ogre + the undo stack). + QMetaObject::invokeMethod(qApp, [self, geo, result, entName]() { + if (!self) return; + self->m_busy = false; + emit self->busyChanged(); + self->m_progress = 0; + self->m_progressTotal = 0; + emit self->progressChanged(); + self->setStatus(QString()); + + if (!result->ok) { + emit self->error(result->error == "cancelled" + ? QStringLiteral("Face-rig cancelled.") + : QString::fromStdString(result->error)); + return; + } + + // Resolve the entity again by name — the selection may have changed + // while the worker ran; only attach if it's still around. + Ogre::Entity* entity = nullptr; + for (Ogre::Entity* e : Manager::getSingleton()->getEntities()) { + if (e && e->getMovableType() == "Entity" + && e->getName() == entName) { entity = e; break; } + } + if (!entity) { + emit self->error(QStringLiteral( + "The mesh went away before the face-rig could be applied.")); + return; + } + + // Undoable group: one macro so Ctrl+Z removes all shapes at once. + FaceRig::AttachReport rep; + rep.userVertexCount = result->userVertexCount; + rep.fitMeanResidualPct = result->fitMeanResidualPct; + rep.fitMaxResidualPct = result->fitMaxResidualPct; + + // The attach adds poses + a VAT_POSE clip to a LIVE entity and + // re-initialises it per shape. If the entity is mid-skeletal- + // animation, the render loop's _updateAnimation can run against the + // half-rebuilt pose buffers between our steps and crash. Disable + // every enabled animation state for the batch, then restore them — + // so the frame loop leaves the entity static while we mutate it. + std::vector reEnable; + if (auto* ass = entity->getAllAnimationStates()) { + for (const auto& kv : ass->getAnimationStates()) { + if (kv.second && kv.second->getEnabled()) { + reEnable.push_back(QString::fromStdString(kv.first)); + kv.second->setEnabled(false); + } + } + } + // Hide the entity for the batch so the render loop doesn't touch its + // pose/skin buffers while we rebuild them — restored after the single + // _initialise. (Belt-and-braces with the animation-state disable.) + Ogre::SceneNode* enode = entity->getParentSceneNode(); + const bool wasVisible = enode ? entity->getVisible() : true; + if (enode) entity->setVisible(false); + + // Build the per-shape commands first so we know which is LAST — only + // the last re-initialises the entity (deferInit=false on it), the + // rest defer. This makes redo (Ctrl+Shift+Z) rebuild the pose buffers + // exactly once at the end too, not just the initial attach; a + // per-shape re-init would freeze the UI on a multi-submesh mesh. + std::vector cmds; + for (const FaceRig::FaceRigShape& shape : result->shapes) { + std::vector slices; + for (const FaceRig::GeometryOwner& o : geo->owners) { + MorphPoseSlice slice; + slice.submeshHandle = o.handle; + for (int i = 0; i < o.count; ++i) { + const std::uint32_t gv = o.base + std::uint32_t(i); + if (size_t(gv) * 3 + 2 >= shape.userDeltas.size()) break; + const float* d = &shape.userDeltas[size_t(gv) * 3]; + if (d[0] == 0.0f && d[1] == 0.0f && d[2] == 0.0f) continue; + slice.offsets[static_cast(i)] = + Ogre::Vector3f(d[0], d[1], d[2]); + } + if (!slice.offsets.empty()) slices.push_back(std::move(slice)); + } + if (slices.empty()) continue; + cmds.push_back(new AddMorphTargetCommand(entity, shape.name, slices)); + } + for (size_t ci = 0; ci + 1 < cmds.size(); ++ci) + cmds[ci]->setDeferInit(true); // all but the last defer re-init + + // Re-rig = REPLACE, not stack: VAT_POSE keyframes reference poses + // BY INDEX, so attaching a second same-named set on an already- + // rigged mesh both duplicates poses AND corrupts the existing + // keyframe references — the "sliders do nothing" failure. Delete + // the old same-named targets first, inside the same macro so + // undo/redo stays atomic. + std::set existing; + if (auto mesh = entity->getMesh()) + for (const Ogre::Pose* p : mesh->getPoseList()) + if (p) existing.insert(p->getName()); + std::vector dels; + for (const FaceRig::FaceRigShape& shape : result->shapes) + if (existing.count(shape.name.toStdString())) + dels.push_back(new DeleteMorphTargetCommand(entity, shape.name)); + + auto* undo = UndoManager::getSingleton(); + auto* stack = undo ? undo->stack() : nullptr; + if (stack) stack->beginMacro(QStringLiteral("Add ARKit Blendshapes")); + for (auto* del : dels) undo->push(del); + for (auto* cmd : cmds) { undo->push(cmd); rep.shapesAttached++; } + if (stack) stack->endMacro(); + qWarning("[facerig] rig: replaced %zu existing, attached %d shapes " + "(fit mean %.3f%% max %.3f%%)", + dels.size(), rep.shapesAttached, + rep.fitMeanResidualPct, rep.fitMaxResidualPct); + if (enode) entity->setVisible(wasVisible); + + // Restore the animation states we disabled (refreshAvailable... in + // the attach may have recreated the state set, so re-resolve). + if (auto* ass = entity->getAllAnimationStates()) { + for (const QString& n : reEnable) { + const std::string sn = n.toStdString(); + if (ass->hasAnimationState(sn)) + ass->getAnimationState(sn)->setEnabled(true); + } + } + rep.ok = rep.shapesAttached > 0; + + if (!rep.ok) { + emit self->error(QStringLiteral( + "No blendshapes produced any vertex movement.")); + return; + } + + GamificationManager::noteOperation( + QStringLiteral("morph"), + {{QStringLiteral("blendshapes_attached"), rep.shapesAttached}}, + GamificationManager::Surface::Gui); + + QVariantMap map; + map["shapesAttached"] = rep.shapesAttached; + map["userVertexCount"] = rep.userVertexCount; + map["fitMeanResidualPct"] = rep.fitMeanResidualPct; + map["fitMaxResidualPct"] = rep.fitMaxResidualPct; + // Amplitude diagnostics: without these, "shapes attached but + // invisible" (deltas 50x too small) looks identical to success in + // the UI. jawDisp specifically because jawOpen is the shape users + // test first. + double maxDisp = 0, jawDisp = 0; + for (const auto& sh : result->shapes) { + maxDisp = std::max(maxDisp, double(sh.maxDisp)); + if (sh.name == QLatin1String("jawOpen")) + jawDisp = double(sh.maxDisp); + } + map["maxShapeDisp"] = maxDisp; + map["jawOpenDisp"] = jawDisp; + qWarning("[facerig] attached=%d jawOpenDisp=%.5f maxDisp=%.5f", + rep.shapesAttached, jawDisp, maxDisp); + emit self->completed(map); + }, Qt::QueuedConnection); + }).detach(); + + return true; +} + +void FaceRigController::cancel() +{ + if (m_cancel) m_cancel->store(true); +} + +// ─────────────────── Face-marker editing session ──────────────────── + +QStringList FaceRigController::markerLabels() const +{ + QStringList out; + for (const auto& m : m_markers) out << m.label; + return out; +} + +bool FaceRigController::markerPlaced(int index) const +{ + return index >= 0 && index < int(m_markers.size()) + && m_markers[size_t(index)].placed; +} + +bool FaceRigController::beginFaceMarkers() +{ + if (m_busy) { emit error(QStringLiteral("Busy.")); return false; } + auto* sel = SelectionSet::getSingleton(); + const auto entities = sel ? sel->getResolvedEntities() + : QList{}; + Ogre::Entity* entity = entities.isEmpty() ? nullptr : entities.first(); + if (!entity || !entity->getMesh()) { + emit error(QStringLiteral("No mesh selected.")); + return false; + } + + // Geometry + template (same prep as the direct rig). + m_geo = std::make_shared( + FaceRig::extractGeometry(entity)); + if (!m_geo->valid()) { + emit error(QStringLiteral("Could not read the mesh geometry.")); + return false; + } + m_downloading = !FaceRig::ArkitTemplate::present(); + setStatus(m_downloading ? QStringLiteral("Downloading face template…") + : QStringLiteral("Preparing…")); + const QString path = FaceRig::ArkitTemplate::ensureModelBlocking(); + m_downloading = false; + if (path.isEmpty()) { + setStatus(QString()); + emit error(QStringLiteral("ARKit template unavailable.")); + return false; + } + m_markerTmpl = std::make_shared(); + QString loadErr; + if (!m_markerTmpl->load(path, &loadErr)) { + setStatus(QString()); + emit error(QStringLiteral("Failed to load ARKit template: %1").arg(loadErr)); + return false; + } + + // Seed markers: template detection resolves template verts (reliable), + // user detection seeds positions when confident, else sensible defaults. + setStatus(QStringLiteral("Detecting face landmarks…")); + std::vector headV; std::vector headF; + FaceRig::headSubmesh(*m_geo, headV, headF); + m_markers = FaceRig::seedFaceMarkers(entity, headV, headF, *m_markerTmpl, + &m_seededConfident); + setStatus(QString()); + if (m_markers.empty()) { + emit error(QStringLiteral("Could not prepare face markers.")); + return false; + } + + m_markerEntityName = entity->getName(); + m_markerMode = true; + m_selMarker = 0; + refreshMarkerOverlays(); + emit markerModeChanged(); + emit markersChanged(); + return true; +} + +void FaceRigController::selectMarker(int index) +{ + if (!m_markerMode) return; + m_selMarker = (index >= 0 && index < int(m_markers.size())) ? index : -1; + refreshMarkerOverlays(); + emit markersChanged(); +} + +void FaceRigController::cancelFaceMarkers() +{ + if (!m_markerMode) return; + m_markerMode = false; + m_markers.clear(); + m_selMarker = -1; + clearMarkerOverlays(); + setStatus(QString()); + emit markerModeChanged(); + emit markersChanged(); +} + +bool FaceRigController::handleMarkerClick(OgreWidget* widget, const QPoint& screenPos) +{ + if (!m_markerMode || !widget) return false; + + Ogre::Entity* entity = nullptr; + for (Ogre::Entity* e : Manager::getSingleton()->getEntities()) + if (e && e->getMovableType() == "Entity" + && e->getName() == m_markerEntityName) { entity = e; break; } + if (!entity) { cancelFaceMarkers(); return false; } + + auto* spaceCam = widget->getSpaceCamera(); + auto* cam = spaceCam ? spaceCam->getCamera() : nullptr; + if (!cam) return true; + int vw = 0, vh = 0; + widget->pixelSizeForCameraPicking(vw, vh); + if (vw <= 0 || vh <= 0) return true; + const Ogre::Real nx = Ogre::Real(screenPos.x()) / vw; + const Ogre::Real ny = Ogre::Real(screenPos.y()) / vh; + const Ogre::Ray ray = cam->getCameraToViewportRay(nx, ny); + + // Ray-cast to the mesh surface (world-space triangles from the geometry). + Ogre::Node* node = entity->getParentSceneNode(); + const Ogre::Matrix4 world = node ? node->_getFullTransform() + : Ogre::Matrix4::IDENTITY; + const auto& V = m_geo->userV; const auto& F = m_geo->userF; + const int nv = int(V.size()/3); + float bestT = std::numeric_limits::max(); + Ogre::Vector3 hitLocal; bool found = false; + for (size_t f = 0; f + 2 < F.size(); f += 3) { + const int ia=F[f], ib=F[f+1], ic=F[f+2]; + if (ia<0||ib<0||ic<0||ia>=nv||ib>=nv||ic>=nv) continue; + const Ogre::Vector3 a = world*Ogre::Vector3(V[size_t(ia)*3],V[size_t(ia)*3+1],V[size_t(ia)*3+2]); + const Ogre::Vector3 b = world*Ogre::Vector3(V[size_t(ib)*3],V[size_t(ib)*3+1],V[size_t(ib)*3+2]); + const Ogre::Vector3 c = world*Ogre::Vector3(V[size_t(ic)*3],V[size_t(ic)*3+1],V[size_t(ic)*3+2]); + auto res = Ogre::Math::intersects(ray, a, b, c, true, false); + if (res.first && res.second < bestT) { + bestT = res.second; + const Ogre::Vector3 w = ray.getPoint(res.second); + hitLocal = world.inverse() * w; + found = true; + } + } + if (!found) return true; // missed the mesh — consume + + if (m_selMarker < 0 || m_selMarker >= int(m_markers.size())) { + // no selection → pick the nearest marker to the hit, don't move it. + int best = -1; float bd = std::numeric_limits::max(); + for (int i = 0; i < int(m_markers.size()); ++i) { + const auto& p = m_markers[size_t(i)].userPos; + const float d = (Ogre::Vector3(p[0],p[1],p[2]) - hitLocal).squaredLength(); + if (d < bd) { bd = d; best = i; } + } + m_selMarker = best; + } else { + // move the selected marker to the hit point. + auto& m = m_markers[size_t(m_selMarker)]; + m.userPos = { hitLocal.x, hitLocal.y, hitLocal.z }; + m.placed = true; + // auto-advance in CATALOG ORDER so the user can walk the whole set + // click-by-click (defaults mark everything "placed", so advancing to + // the next *unplaced* one would just stick on the same marker). + m_selMarker = (m_selMarker + 1) % int(m_markers.size()); + } + refreshMarkerOverlays(); + emit markersChanged(); + return true; +} + +bool FaceRigController::rigFromMarkers(int maxShapes, double maxResidualPct, + double amplitude) +{ + if (!m_markerMode) { emit error(QStringLiteral("Not in marker mode.")); return false; } + auto tmpl = m_markerTmpl; + if (!tmpl) { emit error(QStringLiteral("Template not loaded.")); return false; } + const auto anchors = FaceRig::anchorsFromMarkers(m_markers, *tmpl); + int placedCount = 0; + for (const auto& m : m_markers) placedCount += m.placed ? 1 : 0; + qWarning("[facerig] rigFromMarkers: %d/%zu markers placed -> %zu anchors", + placedCount, m_markers.size(), anchors.size()); + // Leave marker mode (clears overlays) before the rig runs. + m_markerMode = false; + m_selMarker = -1; + clearMarkerOverlays(); + emit markerModeChanged(); + emit markersChanged(); + if (!tmpl) { emit error(QStringLiteral("Template not loaded.")); return false; } + return runRigAsync(tmpl, maxShapes, maxResidualPct, amplitude, anchors); +} + +void FaceRigController::clearMarkerOverlays() +{ + auto* mgr = Manager::getSingletonPtr(); + Ogre::SceneManager* scene = mgr ? mgr->getSceneMgr() : nullptr; + for (Ogre::SceneNode* n : m_markerNodes) { + if (!n) continue; + if (scene) { + auto objs = n->getAttachedObjects(); + for (auto* o : objs) scene->destroyMovableObject(o); + scene->destroySceneNode(n); + } + } + m_markerNodes.clear(); +} + +void FaceRigController::refreshMarkerOverlays() +{ + clearMarkerOverlays(); + auto* mgr = Manager::getSingletonPtr(); + Ogre::SceneManager* scene = mgr ? mgr->getSceneMgr() : nullptr; + if (!scene) return; + Ogre::Entity* entity = nullptr; + for (Ogre::Entity* e : mgr->getEntities()) + if (e && e->getMovableType() == "Entity" + && e->getName() == m_markerEntityName) { entity = e; break; } + if (!entity) return; + Ogre::Node* node = entity->getParentSceneNode(); + + auto& mm = Ogre::MaterialManager::getSingleton(); + auto ensureMat = [&](const std::string& n, const Ogre::ColourValue& c) { + if (!mm.resourceExists(n)) { + auto mat = mm.create(n, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + auto* pass = mat->getTechnique(0)->getPass(0); + // Lighting ON + self-illumination is what actually colours the + // sphere — with lighting disabled Ogre ignores diffuse/ambient and + // the markers render plain white (indistinguishable states). + pass->setLightingEnabled(true); + pass->setSelfIllumination(c); + pass->setDiffuse(Ogre::ColourValue::Black); + pass->setAmbient(Ogre::ColourValue::Black); + pass->setSpecular(Ogre::ColourValue::Black); + pass->setDepthCheckEnabled(false); + } + }; + ensureMat("__FaceMarkerMat__", Ogre::ColourValue(1.0f, 0.85f, 0.1f, 1.0f)); // placed + ensureMat("__FaceMarkerMatSel__", Ogre::ColourValue(0.2f, 0.9f, 1.0f, 1.0f)); // selected + ensureMat("__FaceMarkerMatUnset__", Ogre::ColourValue(0.6f, 0.6f, 0.6f, 1.0f)); // default/unplaced + + const Ogre::Real r = entity->getBoundingRadius() * 0.02f; + for (int i = 0; i < int(m_markers.size()); ++i) { + const auto& m = m_markers[size_t(i)]; + const Ogre::Vector3 localPos(m.userPos[0], m.userPos[1], m.userPos[2]); + const Ogre::Vector3 worldPos = node ? node->_getFullTransform()*localPos : localPos; + Ogre::SceneNode* sn = scene->getRootSceneNode()->createChildSceneNode(); + Ogre::Entity* sphere = nullptr; + try { sphere = scene->createEntity(Ogre::SceneManager::PT_SPHERE); } catch (...) {} + if (sphere) { + sphere->setMaterialName(i == m_selMarker ? "__FaceMarkerMatSel__" + : m.placed ? "__FaceMarkerMat__" + : "__FaceMarkerMatUnset__"); + sphere->setRenderQueueGroup(Ogre::RENDER_QUEUE_OVERLAY - 1); + sn->attachObject(sphere); + const Ogre::Real s = (r > 1e-4f ? r : 0.02f) / 100.0f + * (i == m_selMarker ? 1.5f : 1.0f); + sn->setScale(s, s, s); + } + sn->setPosition(worldPos); + m_markerNodes.push_back(sn); + } +} diff --git a/src/FaceRigController.h b/src/FaceRigController.h new file mode 100644 index 000000000..6fd2ba890 --- /dev/null +++ b/src/FaceRigController.h @@ -0,0 +1,156 @@ +#ifndef FACE_RIG_CONTROLLER_H +#define FACE_RIG_CONTROLLER_H + +#include "FaceRig/FaceRigLandmarks.h" // FaceMarker, NricpLandmark + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace Ogre { class SceneNode; } +namespace FaceRig { class ArkitTemplate; struct FaceRigGeometry; } + +// QML-facing singleton for the face auto-rig (#889, Slice F #895). Wraps +// FaceRig::attachFaceRig* plus selection state so the Inspector's "Add ARKit +// Blendshapes" button can enable itself only on a mesh selection and run the +// (heavy) fit on a worker thread while the UI stays responsive. +// +// The pipeline is split across threads: geometry extraction + the pose attach +// touch Ogre and run on the MAIN thread; the heavy Ogre-free buildFaceRig() +// (NRICP + deformation transfer over 52 shapes) runs on a WORKER. The attach +// goes through AddMorphTargetCommand so it is undoable, and it lands the +// shapes in the same Ogre::Pose + VAT_POSE form the Vertex Morph section reads, +// so the #869 face-capture panel drives them immediately. +class FaceRigController : public QObject +{ + Q_OBJECT + QML_ELEMENT + QML_SINGLETON + + Q_PROPERTY(bool hasMeshSelection READ hasMeshSelection NOTIFY selectionChanged) + Q_PROPERTY(bool busy READ busy NOTIFY busyChanged) + // True while the bundled ARKit template is downloading on first use. + Q_PROPERTY(bool downloading READ downloading NOTIFY statusChanged) + // Short human-readable status for the button label / tooltip. + Q_PROPERTY(QString status READ status NOTIFY statusChanged) + // Worker progress for a progress bar: done / total steps (0 total = idle). + Q_PROPERTY(int progress READ progress NOTIFY progressChanged) + Q_PROPERTY(int progressTotal READ progressTotal NOTIFY progressChanged) + // Face-marker editing session (auto-seed → user adjusts → rig). + Q_PROPERTY(bool markerMode READ markerMode NOTIFY markerModeChanged) + Q_PROPERTY(QStringList markerLabels READ markerLabels NOTIFY markersChanged) + Q_PROPERTY(int selectedMarker READ selectedMarker NOTIFY markersChanged) + Q_PROPERTY(bool markersSeededFromDetection READ markersSeededFromDetection + NOTIFY markersChanged) + +public: + static FaceRigController* instance(); + static FaceRigController* qmlInstance(QQmlEngine* engine, QJSEngine* scriptEngine); + static void kill(); + + bool hasMeshSelection() const; + bool busy() const { return m_busy; } + bool downloading() const { return m_downloading; } + QString status() const { return m_status; } + int progress() const { return m_progress; } + int progressTotal() const { return m_progressTotal; } + + /// Run the face auto-rig on the first selected entity and attach the ARKit + /// blendshapes. Prepares (extracts geometry + loads the bundled template) + /// on the main thread, runs the fit on a WORKER, commits the undoable + /// attach back on the main thread. Returns false when it could not start + /// (invalid selection / already busy — `error` is emitted); the real + /// outcome arrives via completed(report) or error(msg). + Q_INVOKABLE bool addArkitBlendshapesAsync(int maxShapes = 0, + double maxResidualPct = 8.0, + double amplitude = 1.0); + + /// Request cancellation of an in-flight fit (no-op otherwise). The worker + /// stops at the next progress step; `error("cancelled")` follows. + Q_INVOKABLE void cancel(); + + // ---- Face-marker editing (auto-seed, user adjusts, then rig) -------- + bool markerMode() const { return m_markerMode; } + QStringList markerLabels() const; + int selectedMarker() const { return m_selMarker; } + bool markersSeededFromDetection() const { return m_seededConfident; } + + /// Enter marker mode on the selected face mesh: loads the template, seeds + /// the markers (auto-detect when it works, sensible defaults otherwise), + /// and shows draggable overlays. Emits markersChanged / error. Runs the + /// (main-thread) detection renders inline — quick for a head. + Q_INVOKABLE bool beginFaceMarkers(); + /// Select a marker (index into markerLabels) to reposition on the next + /// mesh click. -1 = none. + Q_INVOKABLE void selectMarker(int index); + /// Leave marker mode, discarding overlays (does NOT rig). + Q_INVOKABLE void cancelFaceMarkers(); + /// Whether a marker is currently "placed" (has a position). QML dims the + /// unplaced ones. + Q_INVOKABLE bool markerPlaced(int index) const; + + /// Commit: build NRICP anchors from the (edited) markers and run the rig + /// (worker thread), same flow as addArkitBlendshapesAsync but anchored to + /// the user-corrected markers. Returns false if it couldn't start. + Q_INVOKABLE bool rigFromMarkers(int maxShapes = 0, double maxResidualPct = 8.0, + double amplitude = 1.0); + + /// Called by TransformOperator on a viewport left-click while markerMode is + /// active: ray-casts to the mesh surface and moves the SELECTED marker + /// there (or picks the nearest marker if none selected). Returns true if + /// the click was consumed. + bool handleMarkerClick(class OgreWidget* widget, const QPoint& screenPos); + +signals: + void selectionChanged(); + void busyChanged(); + void statusChanged(); + void progressChanged(); + void markerModeChanged(); + void markersChanged(); + void completed(const QVariantMap& report); + void error(const QString& message); + +private: + FaceRigController(); + ~FaceRigController() override = default; + + void setStatus(const QString& s); + + // Shared async rig runner: extract geometry + template already loaded on the + // main thread; `anchors` are the landmark constraints (auto or marker-based, + // possibly empty). Runs the fit on a worker, attaches on the main thread. + bool runRigAsync(const std::shared_ptr& tmpl, + int maxShapes, double maxResidualPct, double amplitude, + const std::vector& anchors); + + void clearMarkerOverlays(); + void refreshMarkerOverlays(); + + static FaceRigController* m_pSingleton; + bool m_busy = false; + bool m_downloading = false; + QString m_status; + int m_progress = 0; + int m_progressTotal = 0; + std::shared_ptr m_cancel; + + // Marker session. + bool m_markerMode = false; + int m_selMarker = -1; + bool m_seededConfident = false; + std::string m_markerEntityName; + std::vector m_markers; + std::vector m_markerNodes; + std::shared_ptr m_markerTmpl; + // Geometry extracted on the main thread, handed to the worker rig run. + std::shared_ptr m_geo; +}; + +#endif // FACE_RIG_CONTROLLER_H diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 647764795..0934ec45f 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -48,6 +48,7 @@ #include "SkinWeights.h" #include "SkinningDisplay.h" #include "AutoRig.h" +#include "FaceRig/FaceRigAttach.h" #include "MeshDepthRenderer.h" #include "ModelIsometricRenderer.h" #ifdef ENABLE_STABLE_DIFFUSION @@ -625,6 +626,7 @@ const QMap& MCPServer::toolHandlers() {QStringLiteral("compute_skin_weights"), &MCPServer::toolComputeSkinWeights}, {QStringLiteral("set_skinning_display"), &MCPServer::toolSetSkinningDisplay}, {QStringLiteral("auto_rig"), &MCPServer::toolAutoRig}, + {QStringLiteral("add_arkit_blendshapes"), &MCPServer::toolAddArkitBlendshapes}, {QStringLiteral("generate_mesh_texture"), &MCPServer::toolGenerateMeshTexture}, {QStringLiteral("generate_pbr_maps"), &MCPServer::toolGeneratePbrMaps}, {QStringLiteral("upscale_texture"), &MCPServer::toolUpscaleTexture}, @@ -740,6 +742,7 @@ bool MCPServer::isHeavyTool(const QString &name) QStringLiteral("motion_in_between"), QStringLiteral("generate_motion"), QStringLiteral("segment_mesh"), + QStringLiteral("add_arkit_blendshapes"), QStringLiteral("generate_mesh_from_image"), QStringLiteral("save_scene"), QStringLiteral("open_scene"), @@ -798,6 +801,7 @@ QJsonObject MCPServer::callTool(const QString &name, const QJsonObject &args) {QStringLiteral("uv_set_seams"), QStringLiteral("uv_unwrap")}, {QStringLiteral("compute_skin_weights"), QStringLiteral("skin_weights")}, {QStringLiteral("auto_rig"), QStringLiteral("auto_rig")}, + {QStringLiteral("add_arkit_blendshapes"), QStringLiteral("auto_rig")}, {QStringLiteral("motion_in_between"), QStringLiteral("motion_inbetween")}, {QStringLiteral("generate_motion"), QStringLiteral("animation_blend")}, {QStringLiteral("merge_animations"), QStringLiteral("animation_blend")}, @@ -2291,6 +2295,86 @@ QJsonObject MCPServer::toolAutoRig(const QJsonObject &args) return result; } +QJsonObject MCPServer::toolAddArkitBlendshapes(const QJsonObject &args) +{ + // #889: fit the ARKit blendshape template onto the selected face mesh and + // attach the 52 ARKit morph targets, optionally re-exporting. + if (!hasSelectedEntities()) + return makeErrorResult("No mesh selected. Load a mesh first with load_mesh."); + + FaceRig::FaceRigOptions opts; + if (args.contains("max_shapes")) { + if (!args["max_shapes"].isDouble()) + return makeErrorResult("Error: 'max_shapes' must be a number."); + opts.maxShapes = args["max_shapes"].toInt(); + } + if (args.contains("max_residual_pct")) { + if (!args["max_residual_pct"].isDouble()) + return makeErrorResult("Error: 'max_residual_pct' must be a number."); + opts.maxFitResidualPct = args["max_residual_pct"].toDouble(); + } + if (args.contains("output_path") && !args["output_path"].isString()) + return makeErrorResult("Error: 'output_path' must be a string."); + const QString outputPath = args.value("output_path").toString(); + + SelectionSet* sel = SelectionSet::getSingleton(); + const QList resolved = sel ? sel->getResolvedEntities() + : QList{}; + if (resolved.isEmpty() || !resolved.first()) + return makeErrorResult("No selected entity."); + Ogre::Entity* entity = resolved.first(); + + SentryReporter::addBreadcrumb(QStringLiteral("ai.tool_call"), + QStringLiteral("add_arkit_blendshapes max_shapes=%1").arg(opts.maxShapes)); + + FaceRig::AttachReport rep; + try { + rep = FaceRig::attachFaceRigWithBundledTemplate(entity, opts); + if (!rep.ok) + return makeErrorResult( + QStringLiteral("Face-rig failed: %1").arg(rep.error)); + + if (!outputPath.isEmpty()) { + Ogre::SceneNode* node = entity->getParentSceneNode(); + if (!node) + return makeErrorResult(QStringLiteral( + "Error: rigged, but the entity has no scene node to export from")); + SentryReporter::addBreadcrumb(QStringLiteral("file.export"), + QStringLiteral("add_arkit_blendshapes export requested")); + const int rc = MeshImporterExporter::exporter( + node, outputPath, CLIPipeline::formatForExtension(outputPath)); + if (rc != 0) + return makeErrorResult( + QStringLiteral("Error: blendshapes attached but export to '%1' " + "failed (code %2)").arg(outputPath).arg(rc)); + // NOTE: no sidecar write here — MeshImporterExporter::exporter + // already writes the FULL deduplicated pose-name sidecar; + // overwriting it with only the newly-attached names would drop + // pre-existing morph targets and misalign indices. + } + } catch (const Ogre::Exception& e) { + return makeErrorResult(QStringLiteral("Ogre error: %1") + .arg(QString::fromStdString(e.getFullDescription()))); + } catch (const std::exception& e) { + return makeErrorResult(QStringLiteral("Face-rig error: %1") + .arg(QString::fromUtf8(e.what()))); + } + + QJsonObject result = makeSuccessResult( + QStringLiteral("Attached %1 ARKit blendshape(s) (fit residual mean %2%, " + "max %3%).") + .arg(rep.shapesAttached) + .arg(rep.fitMeanResidualPct, 0, 'f', 3) + .arg(rep.fitMaxResidualPct, 0, 'f', 3)); + QJsonObject j; + j["shapes_attached"] = rep.shapesAttached; + j["user_vertex_count"] = rep.userVertexCount; + j["fit_mean_residual_pct"] = rep.fitMeanResidualPct; + j["fit_max_residual_pct"] = rep.fitMaxResidualPct; + result["facerig"] = j; + return result; +} + QJsonObject MCPServer::toolGenerateMeshTexture(const QJsonObject &args) { #ifndef ENABLE_STABLE_DIFFUSION @@ -8542,6 +8626,35 @@ QJsonArray MCPServer::buildToolsList() ); } + // add_arkit_blendshapes (#889) + { + QJsonObject props; + props["max_shapes"] = QJsonObject{{"type", "integer"}, + {"description", + "Cap the number of ARKit shapes generated (0 = all 52 in the " + "template, default)."}}; + props["max_residual_pct"] = QJsonObject{{"type", "number"}, + {"description", + "Reject the rig when the non-rigid fit is worse than this percent of " + "the mesh diagonal (default 8). A human template only fits a roughly " + "human face; a non-face mesh blows past this and is refused."}}; + props["output_path"] = QJsonObject{{"type", "string"}, + {"description", + "Optional path to re-export the mesh with the attached blendshapes. " + "When omitted, the shapes are added to the in-session scene only."}}; + appendTool( + "add_arkit_blendshapes", + "Fit the ARKit blendshape template onto the currently selected FACE " + "mesh and attach the 52 ARKit morph targets (#889). Native pipeline (no " + "external deps): non-rigid ICP fits the template to the user face, then " + "Sumner-Popovic deformation transfer realizes each expression on the " + "user's identity; the shapes are named per the mocap-52 vocabulary so " + "face capture drives them. Humanoid faces only — a poor fit is rejected. " + "The bundled template downloads on first use.", + props + ); + } + // generate_mesh_texture — only advertised when Stable Diffusion is // compiled in; the handler hard-fails otherwise, so publishing it on // a non-SD build would imply a capability the server can't satisfy. diff --git a/src/MCPServer.h b/src/MCPServer.h index 1555d16fa..82657ff31 100644 --- a/src/MCPServer.h +++ b/src/MCPServer.h @@ -177,6 +177,10 @@ private slots: /// #407: native auto-rig of the selected static mesh (template embedding), /// optional skin chain + re-export. QJsonObject toolAutoRig(const QJsonObject &args); + /// #889: fit the ARKit blendshape template onto the selected face mesh + /// (NRICP + deformation transfer), attach the 52 ARKit morph targets, and + /// optionally re-export. + QJsonObject toolAddArkitBlendshapes(const QJsonObject &args); /// Issue #403: mesh-aware (depth-conditioned) texture /// generation. Renders the selected entity's depth map and /// conditions sd.cpp on it via a ControlNet depth model, then diff --git a/src/MeshDepthRenderer.cpp b/src/MeshDepthRenderer.cpp index 98b855861..709c34685 100644 --- a/src/MeshDepthRenderer.cpp +++ b/src/MeshDepthRenderer.cpp @@ -64,6 +64,12 @@ Ogre::MaterialPtr ensureDepthMaterial() pass->setAmbient(0, 0, 0); // Fog must affect this pass. pass->setFog(false); // false => inherit scene fog + // Double-sided: assets with inconsistent triangle winding (LH round-trips, + // raw scans) would otherwise render half their faces as culled holes and + // the depth map degrades to speckle noise. The depth buffer still keeps + // the nearest surface, so a correctly-wound mesh is unaffected. + pass->setCullingMode(Ogre::CULL_NONE); + pass->setManualCullingMode(Ogre::MANUAL_CULL_NONE); return mat; } @@ -138,7 +144,8 @@ QImage MeshDepthRenderer::renderDepthMap(Ogre::Entity* entity, int size, } MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( - Ogre::Entity* entity, int size, const View& view, QString* errorOut) + Ogre::Entity* entity, int size, const View& view, QString* errorOut, + const Ogre::AxisAlignedBox* focusAabb) { RenderResult result; if (!entity) { @@ -154,8 +161,11 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( // Frame the camera on the entity's bounding sphere so the whole // mesh fills the view (matches how the generated texture is - // projection-baked back). - const Ogre::AxisAlignedBox aabb = entity->getWorldBoundingBox(true); + // projection-baked back). A focus box (e.g. the head of a full-body + // character) overrides the framing when given. + const Ogre::AxisAlignedBox aabb = + (focusAabb && !focusAabb->isNull()) ? *focusAabb + : entity->getWorldBoundingBox(true); const Ogre::Vector3 center = aabb.getCenter(); const Ogre::Real radius = aabb.getHalfSize().length(); if (radius <= 0.0f) { @@ -221,20 +231,31 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( } // Turn off the target entity's bounding box for the capture - // (it's on by default when selected). Remember to restore. + // (it's on when selected). Save the PRIOR state — restoring an + // unconditional `true` used to leave stray debug boxes on (and they + // draw into later captures: the box render ignores visibility). Ogre::SceneNode* targetNode = entity->getParentSceneNode(); + const bool targetBoxWasShown = targetNode && targetNode->getShowBoundingBox(); if (targetNode) targetNode->showBoundingBox(false); // Hide other entities entirely, remembering each node's prior // visibility so we restore exactly what we changed (a node that - // was already hidden must stay hidden on restore). + // was already hidden must stay hidden on restore). Their bounding + // boxes must be turned off too — showBoundingBox draws via the + // scene manager's debug pass even when the node's objects are hidden. std::vector> hiddenNodes; + std::vector hiddenBoxes; if (Manager::getSingletonPtr()) { for (Ogre::Entity* other : Manager::getSingleton()->getEntities()) { if (!other || other == entity) continue; if (other->getMovableType() != "Entity") continue; Ogre::SceneNode* n = other->getParentSceneNode(); - if (n && n->getAttachedObject(0) && n->getAttachedObject(0)->getVisible()) { + if (!n) continue; + if (n->getShowBoundingBox()) { + hiddenBoxes.push_back(n); + n->showBoundingBox(false); + } + if (n->getAttachedObject(0) && n->getAttachedObject(0)->getVisible()) { hiddenNodes.emplace_back(n, true); n->setVisible(false); } @@ -261,7 +282,8 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( sm->setFog(savedFogMode, savedFogColour, 0.0f, savedFogStart, savedFogEnd); sm->setAmbientLight(savedAmbient); if (gridNode) gridNode->setVisible(gridWasVisible); - if (targetNode) targetNode->showBoundingBox(true); + if (targetNode) targetNode->showBoundingBox(targetBoxWasShown); + for (auto* n : hiddenBoxes) n->showBoundingBox(true); for (auto& [n, wasVisible] : hiddenNodes) n->setVisible(wasVisible); }; struct Restorer { @@ -290,6 +312,143 @@ MeshDepthRenderer::RenderResult MeshDepthRenderer::renderDepthMapView( return result; } +MeshDepthRenderer::RenderResult MeshDepthRenderer::renderShadedView( + Ogre::Entity* entity, int size, const View& view, QString* errorOut, + const Ogre::AxisAlignedBox* focusAabb) +{ + RenderResult result; + if (!entity) { + if (errorOut) *errorOut = QStringLiteral("null entity"); + return result; + } + size = std::clamp(size, 64, 2048); + if (!ensureRenderTarget(size, errorOut)) + return result; + + auto* sm = sceneMgr(); + DepthState& st = state(); + + // Frame on the focus box (head) when given, else the whole entity. + const Ogre::AxisAlignedBox aabb = + (focusAabb && !focusAabb->isNull()) ? *focusAabb + : entity->getWorldBoundingBox(true); + const Ogre::Vector3 center = aabb.getCenter(); + const Ogre::Real radius = aabb.getHalfSize().length(); + if (radius <= 0.0f) { + if (errorOut) *errorOut = QStringLiteral("entity has zero-size bounding box"); + return result; + } + const Ogre::Real fovY = st.camera->getFOVy().valueRadians(); + const Ogre::Real dist = radius / std::sin(fovY * 0.5f) * 1.15f; + Ogre::Vector3 dir = view.dir; + if (dir.isZeroLength()) dir = Ogre::Vector3(0, 0, 1); + dir.normalise(); + const Ogre::Vector3 camPos = center - dir * dist; + st.cameraNode->setPosition(camPos); + const Ogre::Vector3 up = view.up.isZeroLength() ? Ogre::Vector3::UNIT_Y : view.up; + st.cameraNode->setFixedYawAxis(true, up); + st.cameraNode->lookAt(center, Ogre::Node::TS_WORLD, + Ogre::Vector3::NEGATIVE_UNIT_Z); + + // Neutral, DETERMINISTIC lighting so MediaPipe sees an evenly-lit, + // photo-like face (no fog, materials intact): moderate ambient + one + // head-on directional. All EXISTING scene lights are disabled for the + // capture — in the live editor the user/default lights stack on top and + // saturate the render to a pure-white silhouette, which the detector + // false-positives on (and the resulting garbage landmarks correlate + // between template and user render, slipping through the constellation + // gate). Headless and GUI captures must produce the same image. + const Ogre::ColourValue savedAmbient = sm->getAmbientLight(); + const Ogre::FogMode savedFogMode = sm->getFogMode(); + const Ogre::ColourValue savedFogColour = sm->getFogColour(); + const Ogre::Real savedFogStart = sm->getFogStart(); + const Ogre::Real savedFogEnd = sm->getFogEnd(); + sm->setFog(Ogre::FOG_NONE); + sm->setAmbientLight(Ogre::ColourValue(0.35f, 0.35f, 0.35f)); + + std::vector disabledLights; + { + auto it = sm->getMovableObjectIterator("Light"); + while (it.hasMoreElements()) { + auto* l = static_cast(it.getNext()); + if (l && l->getVisible()) { + disabledLights.push_back(l); + l->setVisible(false); + } + } + } + + Ogre::Light* light = nullptr; + Ogre::SceneNode* lightNode = nullptr; + try { + light = sm->createLight("QtMeshFaceRigLight"); + light->setType(Ogre::Light::LT_DIRECTIONAL); + light->setDiffuseColour(Ogre::ColourValue(0.65f, 0.65f, 0.65f)); + light->setSpecularColour(Ogre::ColourValue::Black); + lightNode = sm->getRootSceneNode()->createChildSceneNode(); + lightNode->attachObject(light); + lightNode->setDirection(dir, Ogre::Node::TS_WORLD); + } catch (...) { /* lighting is best-effort */ } + + Ogre::SceneNode* gridNode = nullptr; + bool gridWasVisible = false; + if (Manager::getSingletonPtr() + && Manager::getSingleton()->hasSceneNode("GridLine_node")) { + gridNode = Manager::getSingleton()->getSceneNode("GridLine_node"); + if (gridNode) { + gridWasVisible = gridNode->getAttachedObject(0) + ? gridNode->getAttachedObject(0)->getVisible() : true; + gridNode->setVisible(false); + } + } + Ogre::SceneNode* targetNode = entity->getParentSceneNode(); + const bool targetBoxWasShown = targetNode && targetNode->getShowBoundingBox(); + if (targetNode) targetNode->showBoundingBox(false); + std::vector> hiddenNodes; + std::vector hiddenBoxes; + if (Manager::getSingletonPtr()) { + for (Ogre::Entity* other : Manager::getSingleton()->getEntities()) { + if (!other || other == entity) continue; + if (other->getMovableType() != "Entity") continue; + Ogre::SceneNode* n = other->getParentSceneNode(); + if (!n) continue; + if (n->getShowBoundingBox()) { + hiddenBoxes.push_back(n); + n->showBoundingBox(false); + } + if (n->getAttachedObject(0) && n->getAttachedObject(0)->getVisible()) { + hiddenNodes.emplace_back(n, true); + n->setVisible(false); + } + } + } + + auto restore = [&]() { + sm->setAmbientLight(savedAmbient); + sm->setFog(savedFogMode, savedFogColour, 0.0f, savedFogStart, savedFogEnd); + if (lightNode) { lightNode->detachAllObjects(); + sm->getRootSceneNode()->removeAndDestroyChild(lightNode); } + if (light) sm->destroyLight(light); + for (auto* l : disabledLights) l->setVisible(true); + if (gridNode) gridNode->setVisible(gridWasVisible); + if (targetNode) targetNode->showBoundingBox(targetBoxWasShown); + for (auto* n : hiddenBoxes) n->showBoundingBox(true); + for (auto& [n, wasVisible] : hiddenNodes) n->setVisible(wasVisible); + }; + struct Restorer { std::function fn; ~Restorer() { fn(); } } restorer{restore}; + + st.renderTarget->update(); + QImage rgba = readRenderTarget(size); + OgreRenderTargetUtil::restoreEditorRenderTarget(); + + result.viewMatrix = st.camera->getViewMatrix(); + result.projMatrix = st.camera->getProjectionMatrix(); + result.camPosition = camPos; + result.camDirection = dir; + result.depth = rgba.convertToFormat(QImage::Format_RGB888); // RGB, not gray + return result; +} + void MeshDepthRenderer::shutdown() { DepthState& st = state(); diff --git a/src/MeshDepthRenderer.h b/src/MeshDepthRenderer.h index d7bc01341..eb9600e89 100644 --- a/src/MeshDepthRenderer.h +++ b/src/MeshDepthRenderer.h @@ -72,17 +72,36 @@ class MeshDepthRenderer { // Render `entity`'s depth map at `size` x `size` from `view`. Returns a // RenderResult whose `.depth` is null on failure (errorOut populated). // Must be called on the main/render thread — it touches the Ogre scene - // manager. + // manager. `focusAabb` (optional, WORLD space) frames the camera on that + // box instead of the whole entity (same semantics as renderShadedView). static RenderResult renderDepthMapView(Ogre::Entity* entity, int size, const View& view, - QString* errorOut = nullptr); + QString* errorOut = nullptr, + const Ogre::AxisAlignedBox* focusAabb = nullptr); // Back-compat convenience: front-view depth image only (issue #403). static QImage renderDepthMap(Ogre::Entity* entity, int size, QString* errorOut = nullptr); + // Render the entity with its REAL materials (not the flat depth material), + // lit by a temporary head-on light so faces show feature shading — for the + // face auto-rig's landmark detection (#889), which feeds a photo-like image + // to MediaPipe FaceMesh. Same framing / hide-others / RTT reuse as the depth + // path; RenderResult.depth holds the RGB888 image (name kept for reuse) and + // the view/proj/cam fields let the caller back-project 2D landmarks to the + // mesh surface. Main/render thread only. + // `focusAabb` (optional, WORLD space): when non-null, frame the camera on + // THIS box instead of the entity's full bounds — so a full-body character + // can be rendered tightly around the HEAD (the face fills the frame for the + // landmark detector). Pass nullptr to frame the whole entity. + static RenderResult renderShadedView(Ogre::Entity* entity, + int size, + const View& view, + QString* errorOut = nullptr, + const Ogre::AxisAlignedBox* focusAabb = nullptr); + // Release the cached RTT / camera / scene nodes. Safe to call // when Ogre is shutting down. static void shutdown(); diff --git a/src/MeshImporterExporter.cpp b/src/MeshImporterExporter.cpp index 431a6ac70..22ee0fadc 100755 --- a/src/MeshImporterExporter.cpp +++ b/src/MeshImporterExporter.cpp @@ -72,6 +72,7 @@ THE SOFTWARE. #include "SceneLightsIO.h" #include "SelectionSet.h" #include "SentryReporter.h" +#include "FaceRig/FaceRigAttach.h" #include "ExportOptimizer.h" #include "RTShaderHelper.h" @@ -3964,6 +3965,27 @@ int MeshImporterExporter::exporter(const Ogre::SceneNode *_sn, const QString &_u if (formatId == "gltf2" || formatId == "glb2") injectMorphWeightAnimations(file.filePath(), e, /*isBinary=*/formatId == "glb2"); + + // Morph-target NAME sidecar: Assimp's glTF2 exporter also + // drops `targetNames`, so a rigged mesh re-imported from this + // export would degrade to "Shape_N" names. Persist the + // ordered pose names next to the file (the same + // `.arkit.json` the CLI/MCP face-rig paths write); the + // importer restores them on load. + if (e->getMesh() && e->getMesh()->getPoseCount() > 0) + { + std::vector poseNames; + QSet seen; + for (const auto* pose : e->getMesh()->getPoseList()) + { + const QString n = QString::fromStdString(pose->getName()); + if (n.isEmpty() || seen.contains(n)) continue; + seen.insert(n); + poseNames.push_back(n); + } + if (!poseNames.empty()) + FaceRig::writeArkitSidecar(file.filePath(), poseNames); + } } delete scene; diff --git a/src/MorphAnimationManager.cpp b/src/MorphAnimationManager.cpp index 555941f44..2bf38b5a8 100644 --- a/src/MorphAnimationManager.cpp +++ b/src/MorphAnimationManager.cpp @@ -21,6 +21,7 @@ The MIT License #include "commands/MorphCommands.h" #include +#include #include #include @@ -101,11 +102,22 @@ QStringList MorphAnimationManager::morphTargetsFor(Ogre::Entity* entity) const if (!entity) return out; Ogre::MeshPtr mesh = entity->getMesh(); if (!mesh) return out; + // A morph target that spans multiple submeshes has ONE pose per submesh + // handle, all sharing the same name (common on multi-submesh characters — + // e.g. a face split across 11 submeshes yields 11 "jawOpen" poses). The + // UI + every by-name op (weight/key/delete) treat a target as its NAME, so + // list each distinct name once — otherwise the Inspector shows a duplicate + // row per submesh. + QSet seen; const auto& poseList = mesh->getPoseList(); for (const Ogre::Pose* p : poseList) { if (!p) continue; const Ogre::String n = p->getName(); - if (!n.empty()) out << QString::fromStdString(n); + if (n.empty()) continue; + const QString qn = QString::fromStdString(n); + if (seen.contains(qn)) continue; // coalesce same-named poses + seen.insert(qn); + out << qn; } return out; } diff --git a/src/OgreRenderTargetUtil.h b/src/OgreRenderTargetUtil.h index d0702f922..789aae42d 100644 --- a/src/OgreRenderTargetUtil.h +++ b/src/OgreRenderTargetUtil.h @@ -45,8 +45,15 @@ inline void restoreEditorRenderTarget() } } } - if (editorWindow) + if (editorWindow) { rs->_setRenderTarget(editorWindow); + // Clear the render system's cached active viewport. Without this the + // NEXT offscreen RTT update sees its viewport still marked active, + // skips the FBO re-bind, and renders into the editor window instead — + // the RTT then returns the same frozen frame for every later capture + // (multi-view landmark detection got 16 bit-identical images). + rs->_setViewport(nullptr); + } } } // namespace OgreRenderTargetUtil diff --git a/src/TransformOperator.cpp b/src/TransformOperator.cpp index 57c11b76f..afd4ffb4c 100755 --- a/src/TransformOperator.cpp +++ b/src/TransformOperator.cpp @@ -29,6 +29,7 @@ #include "BoneDragRelease.h" #include "EditModeController.h" #include "AutoRigController.h" +#include "FaceRigController.h" #include "TexturePaintController.h" #include "AnimationControlController.h" #include "PropertiesPanelController.h" @@ -1028,6 +1029,13 @@ void TransformOperator::mousePressEvent(QMouseEvent *e) AutoRigController::instance()->handleMarkerClick(m_pActiveWidget, e->pos()); return; } + // Face-rig marker editing (auto-seeded, user-adjusted) — same priority: + // a left-click repositions the selected face marker on the mesh surface. + if (FaceRigController::instance()->markerMode()) + { + FaceRigController::instance()->handleMarkerClick(m_pActiveWidget, e->pos()); + return; + } auto* editCtrl = EditModeController::instance(); diff --git a/src/TransformOperator.h b/src/TransformOperator.h index 94a4b8a31..026bec423 100755 --- a/src/TransformOperator.h +++ b/src/TransformOperator.h @@ -33,6 +33,9 @@ class TransformOperator : public QObject, public QtMouseListener public: static TransformOperator* getSingleton(); + /// Non-creating accessor: null until getSingleton() first runs. Use from + /// paths that must not construct the gizmo machinery (headless CLI). + static TransformOperator* getSingletonPtr() { return m_pSingleton; } static void kill(); enum TransformState diff --git a/src/commands/MorphCommands.cpp b/src/commands/MorphCommands.cpp index 4ab6853ca..e31ac73d1 100644 --- a/src/commands/MorphCommands.cpp +++ b/src/commands/MorphCommands.cpp @@ -105,7 +105,8 @@ void removePosesByName(Ogre::Mesh* mesh, const QString& name, Ogre::Entity* enti void buildPosesFromSlices(Ogre::Mesh* mesh, const QString& name, const std::vector& slices, - Ogre::Entity* entity) + Ogre::Entity* entity, + bool deferInit = false) { if (!mesh || slices.empty()) return; const std::string sn = name.toStdString(); @@ -141,7 +142,7 @@ void buildPosesFromSlices(Ogre::Mesh* mesh, kf->addPoseReference(poseIndices[i], 1.0f); } - if (entity) { + if (entity && !deferInit) { // Adding poses / a VAT_POSE animation to an already-loaded mesh means // the entity's pose (software + hardware) vertex-animation buffers were // never allocated — the importer sets poses up BEFORE the entity is @@ -150,6 +151,8 @@ void buildPosesFromSlices(Ogre::Mesh* mesh, // animation against null buffers and crashes (skinned meshes especially, // where skeletal + pose animation combine). Mirrors the AutoRig path, // which likewise re-initialises after mutating a live entity's mesh. + // deferInit lets a batch attach (face auto-rig) skip this per-shape and + // re-initialise ONCE after the last shape — O(shapes×mesh) → O(mesh). entity->_initialise(true); entity->refreshAvailableAnimationState(); } @@ -261,7 +264,7 @@ void AddMorphTargetCommand::redo() if (!mEntity) return; Ogre::MeshPtr mesh = mEntity->getMesh(); if (!mesh) return; - buildPosesFromSlices(mesh.get(), mName, mSlices, mEntity); + buildPosesFromSlices(mesh.get(), mName, mSlices, mEntity, mDeferInit); SentryReporter::addBreadcrumb("scene.anim.morph", QStringLiteral("add target '%1'").arg(mName)); } diff --git a/src/commands/MorphCommands.h b/src/commands/MorphCommands.h index b7ca7d213..13f08b7e0 100644 --- a/src/commands/MorphCommands.h +++ b/src/commands/MorphCommands.h @@ -51,10 +51,19 @@ class AddMorphTargetCommand : public QUndoCommand void undo() override; void redo() override; + // Batch attach optimisation (face auto-rig #889 attaches 51 shapes at once): + // re-initialising the live entity after EVERY shape is O(shapes × mesh) and + // freezes the UI on big/multi-submesh meshes. When deferInit is set, redo() + // builds the poses but skips entity->_initialise; the caller must call it + // ONCE after the last command (Ogre rebuilds the pose buffers in one pass). + // undo() always re-inits (single removal — cheap). + void setDeferInit(bool defer) { mDeferInit = defer; } + private: Ogre::Entity* mEntity = nullptr; QString mName; std::vector mSlices; + bool mDeferInit = false; }; // DeleteMorphTargetCommand: remove all same-named poses + the matching diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index d11a89be3..b1319e2c6 100755 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -118,6 +118,7 @@ #include "UVEditorController.h" #include "QuadRetopoController.h" #include "SkinWeightsController.h" +#include "FaceRigController.h" #include "LightsController.h" #include "LightRigLibrary.h" #include "SceneLightingController.h" @@ -573,6 +574,7 @@ MainWindow::~MainWindow() UVEditorController::kill(); QuadRetopoController::kill(); SkinWeightsController::kill(); + FaceRigController::kill(); LightsController::kill(); LightPropertiesController::kill(); SceneLightingController::kill(); @@ -766,6 +768,11 @@ void MainWindow::initToolBar() [](QQmlEngine* engine, QJSEngine*) -> QObject* { return SkinWeightsController::qmlInstance(engine, nullptr); }); + qmlRegisterSingletonType( + "PropertiesPanel", 1, 0, "FaceRigController", + [](QQmlEngine* engine, QJSEngine*) -> QObject* { + return FaceRigController::qmlInstance(engine, nullptr); + }); qmlRegisterSingletonType( "PropertiesPanel", 1, 0, "LightsController", [](QQmlEngine* engine, QJSEngine*) -> QObject* { diff --git a/src/test_main.cpp b/src/test_main.cpp index 8ec3dd95b..41c91be9c 100644 --- a/src/test_main.cpp +++ b/src/test_main.cpp @@ -139,12 +139,21 @@ int main(int argc, char **argv) // Prove headless GL works on this runner, then tear down: many suites // (Assimp processors, etc.) construct their own Ogre::Root and cannot // coexist with a live Manager singleton from a prior init. - if (!tryInitOgre()) { + // QTMESH_TESTS_SKIP_OGRE_PREFLIGHT=1 skips the GL proof so PURE-DATA suites + // can run (with --gtest_filter) on machines with no GL/WindowServer at all + // (remote shells, containers without Xvfb). Ogre-dependent fixtures still + // fail under it — this only moves the failure from "no test ran" to + // per-fixture. CI never sets it. + if (qEnvironmentVariableIsSet("QTMESH_TESTS_SKIP_OGRE_PREFLIGHT")) { + fprintf(stderr, "UnitTests: skipping Ogre GL preflight " + "(QTMESH_TESTS_SKIP_OGRE_PREFLIGHT set)\n"); + } else if (!tryInitOgre()) { fprintf(stderr, "UnitTests FATAL: tryInitOgre() failed — need working DISPLAY / Xvfb for GL.\n"); return 1; + } else { + Manager::kill(); } - Manager::kill(); if (QCoreApplication::instance()) QCoreApplication::processEvents(); QThread::msleep(50); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 85090bc64..4ef2ac01e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -75,6 +75,17 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/BoneWeightOverlay.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/NormalVisualizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/AnimationMerger.cpp + # Face auto-rig (#889): MCPServer/CLIPipeline/mainwindow reference + # these — without them the per-suite test executables fail to link. + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRigController.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/ArkitTemplate.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/NonRigidICP.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/SparseSolve.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/DeformationTransfer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/FaceRigger.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/FaceRigAttach.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/FaceLandmarkDetector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRig/FaceRigLandmarks.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MCPServer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MCPSettingsDialog.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MeshInfoOverlay.cpp