fernandotonon · fernandotonon · Jul 22, 2026 · Jul 12, 2026 · Jul 13, 2026 · Jul 13, 2026
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -733,6 +733,8 @@ jobs:
         host: 'linux'
         target: 'desktop'
         arch: 'linux_gcc_64'
+        # qtmultimedia: performance capture (ENABLE_MOCAP, epic #869)
+        modules: 'qtmultimedia'
 
     - name: change folder permissions
       run: |
@@ -781,6 +783,7 @@ jobs:
             -DCMAKE_CXX_FLAGS="-g" -DCMAKE_C_FLAGS="-g" \
             -DENABLE_STABLE_DIFFUSION=ON \
             -DENABLE_ONNX=ON \
+            -DENABLE_MOCAP=ON \
             -DENABLE_AUTO_UPDATER=OFF \
             -DASSIMP_DIR=/usr/local/lib/cmake/assimp-${{ env.ASSIMP_DIR_VERSION }} \
             -DASSIMP_INCLUDE_DIR=/usr/local/include/assimp \
@@ -958,6 +961,8 @@ jobs:
         host: 'linux'
         target: 'desktop'
         arch: 'linux_gcc_64'
+        # qtmultimedia: performance capture (ENABLE_MOCAP, epic #869)
+        modules: 'qtmultimedia'
 
     - name: change folder permissions
       run: |
@@ -1074,7 +1079,8 @@ jobs:
             -DBUILD_QT_MESH_EDITOR=OFF \
             -DENABLE_SENTRY=OFF \
             -DENABLE_PS1_RIP=ON \
-            -DENABLE_ONNX=ON
+            -DENABLE_ONNX=ON \
+            -DENABLE_MOCAP=ON
 
     - name: Run build-wrapper
       env:
@@ -1799,6 +1805,8 @@ jobs:
         target: 'desktop'
         arch: 'clang_64'
         tools: 'tools_cmake'
+        # qtmultimedia: performance capture (ENABLE_MOCAP, epic #869)
+        modules: 'qtmultimedia'
 
     - name: Debug Qt Installation
       run: |
@@ -1885,6 +1893,7 @@ jobs:
             -DCMAKE_CXX_FLAGS="-g" -DCMAKE_C_FLAGS="-g" \
             -DENABLE_STABLE_DIFFUSION=ON \
             -DENABLE_ONNX=ON \
+            -DENABLE_MOCAP=ON \
             -DCMAKE_OSX_ARCHITECTURES="$(uname -m)" \
             -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 \
             -DASSIMP_DIR=/usr/local/lib/cmake/assimp-${{ env.ASSIMP_DIR_VERSION }} \
@@ -2135,10 +2144,24 @@ jobs:
                 ls -la ${{github.workspace}}/bin/QtMeshEditor.app/Contents/MacOS/
             fi
 
-            # Finally sign the entire app bundle
-            echo "Signing app bundle..."
-            sudo codesign --force --sign - ${{github.workspace}}/bin/QtMeshEditor.app || echo "Failed to sign app bundle (non-fatal)"
-
+            # Finally sign the entire app bundle WITH the camera/microphone
+            # entitlements (epic #869). NOTE: this is still an ad-hoc signature
+            # (`--sign -`); macOS only PROMPTS for camera access when the app
+            # is signed with a real Apple Developer ID + notarized. Until this
+            # pipeline gains a Developer ID cert (secret) + notarization step,
+            # the live-camera capture will be blocked on end-user machines the
+            # same way it is on an ad-hoc dev build (the CLI `qtmesh mocap`
+            # video path and file-based capture are unaffected). The
+            # entitlements + Info.plist NSCameraUsageDescription are in place so
+            # that flipping to a Developer ID signature is the ONLY remaining
+            # step to enable the prompt. See docs/MOCAP.md.
+            echo "Signing app bundle (with camera entitlements)..."
+            ENT="${{github.workspace}}/cfg/QtMeshEditor.entitlements"
+            sudo codesign --force --sign - --entitlements "$ENT" \
+                ${{github.workspace}}/bin/QtMeshEditor.app \
+                || sudo codesign --force --sign - ${{github.workspace}}/bin/QtMeshEditor.app \
+                || echo "Failed to sign app bundle (non-fatal)"
+
             echo "Code signing process completed"
 
             # Verify the signing

diff --git a/.gitignore b/.gitignore
@@ -96,6 +96,7 @@ docs/*
 !docs/IMAGE_TO_3D_QUALITY.md
 !docs/TRIPOSG_EXPORT_NOTES.md
 !docs/MESH_SEGMENTATION_STRATEGY.md
+!docs/MOCAP.md
 !docs/FACE_RIG.md
 !docs/FACE_RIG_SPIKE.md
 !docs/TELEMETRY.md
@@ -157,3 +158,6 @@ __pycache__/
 !docs/img/twist_bar_90_lbs.png
 !docs/img/twist_bar_90_lbs_vp.png
 !docs/img/twist_bar_90_dqs_vp.png
+!docs/MOCAP_SPIKE.md
+.mocap_work/
+.venv-mocap/
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -157,6 +157,11 @@ qtmesh segment tree.glb --category vegetation  # force a category (skip the poin
 qtmesh segment car.glb --category vehicle      # vehicle_body/wheel/window/wing/rotor; `building` = wall/roof/window/door/chimney/foundation
 qtmesh segment model.fbx --no-model --up-axis y  # force the deterministic geometric fallback (skip the ONNX models; auto → body)
 qtmesh segment rigged.fbx --dump-training-data sample.json  # mine EXACT rig-prior labels from a SKINNED mesh → training sample (#410; feed to export-meshseg-onnx.py --real-data)
+qtmesh mocap talk.mp4 --face --mesh avatar.glb -o out.glb    # performance capture (#869, needs -DENABLE_MOCAP): facial expressions -> ARKit-blendshape weight keyframes + head rotation (Head bone or node); models download on first use
+qtmesh mocap dance.mp4 --body --mesh rigged.fbx -o out.glb   # full-body pose -> skeletal clip on the humanoid rig (root locked; --algo sam3dbody|pose-ik, sam3dbody falls back to pose-ik while its checkpoints are gated; --no-model forces the fallback)
+qtmesh mocap take.mp4 --face --body --mesh char.glb -o out.glb  # both in one decode pass ("<clip>_Body" for the body clip)
+qtmesh mocap clip.mp4 --face --mesh head.glb --map overrides.json --no-head --json  # custom channel mapping sidecar; JSON report (matched/unmatched channels always listed)
+qtmesh mocap x.mp4 --face --mesh m.glb --frames-dir frames/  # image-sequence input (headless CI/debug; no video decode)
 qtmesh cloud login                            # device flow (prints URL + code); stores session locally
 qtmesh cloud login --api-key <token>          # direct API-key login (CI)
 qtmesh cloud logout                           # revoke + clear saved session
@@ -177,7 +182,7 @@ qtmesh ps1 dump-vram game.cue --bios scph1001.bin --frames 300 -o vram.png  # sn
 #   xvfb-run -a qtmesh ps1 capture game.cue --bios scph1001.bin -o out.gltf
 ```
 
-CLI mode is activated by: (1) invoking via the `qtmesh` symlink, (2) passing `--cli`, or (3) using a recognized subcommand (`info`, `fix`, `convert`, `anim`, `validate`, `lod`, `pose`, `turntable`, `isometric`, `scan`, `material`, `hdri`, `light`, `pack-textures`, `normal-from-height`, `atlas`, `atlas-apply`, `memory`, `analyze`, `vertex-cache`, `decimate`, `optimize`, `uv`, `retopo`, `skin`, `rig`, `facerig`, `segment`, `generate3d`, `ps1`, `cloud`) as the first argument. Use `--verbose` to see Ogre/engine debug output. Use `--no-telemetry` to permanently opt out of anonymous usage data collection.
+CLI mode is activated by: (1) invoking via the `qtmesh` symlink, (2) passing `--cli`, or (3) using a recognized subcommand (`info`, `fix`, `convert`, `anim`, `validate`, `lod`, `pose`, `turntable`, `isometric`, `scan`, `material`, `hdri`, `light`, `pack-textures`, `normal-from-height`, `atlas`, `atlas-apply`, `memory`, `analyze`, `vertex-cache`, `decimate`, `optimize`, `uv`, `retopo`, `skin`, `rig`, `facerig`, `segment`, `generate3d`, `mocap`, `ps1`, `cloud`) as the first argument. Use `--verbose` to see Ogre/engine debug output. Use `--no-telemetry` to permanently opt out of anonymous usage data collection.
 
 If Xcode SDK is updated, clear CMake cache (`rm build_local/CMakeCache.txt`) and reconfigure.
 
@@ -340,6 +345,17 @@ The animation pipeline started skeleton-only; the #517 epic broadens it. Slices
 - **ScanEngine** (`src/ScanEngine.h/cpp`): Directory scanner for 3D asset linting. Loads every asset through `MeshImporterExporter` (the editor's own loader) and walks the resulting Ogre scene with `CLIPipeline::extractMeshInfo` — the same extractor `MeshInfoOverlay` uses, so the scan, the CLI `info` subcommand and the in-app overlay all report identical counts for the same asset. Redundant-keyframe analysis (and the `--fix` write-back since slice C4) goes through `AnimationMerger::analyzeRedundantKeyframes` / `simplifyAnimation`, the same code path as `qtmesh anim --simplify` and the Inspector "Simplify" button. The fix path re-exports via `MeshImporterExporter::exporter` for every supported format (FBX/glTF/glb/DAE/OBJ/PLY/STL/.mesh) — no `Assimp::Exporter`. ACMR is folded into the same Ogre walk so each file is loaded once per scan. Assimp's only remaining role is a no-process `ReadFile` to enumerate `aiMaterial::GetTexture` references that Ogre's TUS-name walk wouldn't see when a referenced texture file is missing on disk (needed for `require_textures_exist`). Quality rules driven by the Ogre walk: `max_texture_resolution` (largest texture dimension cap), `require_uv_channels` (per-submesh UV-set minimum), `detect_zero_weight_bones` (Mixamo bloat — bones with no vertex weights), `detect_overlapping_uvs_pct` (UV0 AABB sweep — lightmap quality), `detect_non_manifold_edges_pct` (edges shared by != 2 faces — boolean / printing safety). Enumerates files via glob patterns, evaluates configurable rules, produces text/JSON/SARIF reports. Per-file cleanup happens in `clearOgreSceneForScanImport` which destroys scene nodes and flushes MeshManager / SkeletonManager so a 1000-asset scan doesn't accumulate state.
 - **ScanConfig** (`src/ScanConfig.h/cpp`): Config loader for `qtmesh.yml`/`.json`. Includes a minimal YAML parser for the specific config schema (scalars, inline/block lists, one level of section nesting). Supports scan paths, rule configuration, fix behavior, and report output settings.
 
+### Performance capture (epic #869, `ENABLE_MOCAP`)
+
+Video/webcam -> facial morph + head + skeletal body animation, built on the existing ONNX/model-download/retarget/morph-keyframe infrastructure. Everything lives in `src/Mocap/`; the CMake flag `ENABLE_MOCAP` (default OFF; ON for macOS/Linux release + the Linux test lane) requires `ENABLE_ONNX` and pulls in **Qt6::Multimedia** (a new dependency — Windows/MinGW pending verification, so OFF there). Non-mocap builds print "rebuild with -DENABLE_MOCAP" on every surface.
+
+- **VideoFrameSource** (`src/Mocap/VideoFrameSource.{h,cpp}`): the frame abstraction — `FileFrameSource` (QMediaPlayer+QVideoSink, targetFps decimation, playback-driven), `CameraFrameSource` (QCamera, device enumeration, latest-wins `FrameMailbox` so live inference never falls behind), `ImageSequenceFrameSource` (headless test double / CLI `--frames-dir`). `FrameDecimator`/`FrameMailbox` are pure + tested.
+- **FaceCapPredictor** (ONNX consumer #9): 3 sessions (BlazeFace detector 128² [-1,1] -> Face Mesh V2 rotated 256² crop [0,1] -> 478 landmarks -> 146-subset px coords -> 52 blendshape scores) with detector-skip tracking (next ROI from previous landmarks). **PoseCapPredictor** (consumer #10): BlazePose detector+landmarks -> 33 world landmarks; tracking from the model's aux alignment landmarks. Both download to `AppData/ai_models/mocap/{face,pose}/` (`QTMESH_MOCAP_MODEL_BASE_URL` / `ai/mocapModelBaseUrl` / `QTMESH_MOCAP_NO_DOWNLOAD`). The exact pre/post-processing contract (anchors, letterbox, weighted NMS, **cv2-integer-index plain bilinear — never antialiased**, projection) was parity-proven in the Slice A spike: `docs/MOCAP_SPIKE.md`, `scripts/export-facecap-onnx.py` (converts AND asserts parity ≤0.6px/≤0.015 blendshape/≤1cm world).
+- **Pure-data core** (all headless-tested): `FaceCapGeom` (letterbox/anchors/decode/NMS/ROI/crop/projection), `FaceCapPose` (Horn quaternion weighted rigid fit + embedded canonical face model — no Eigen), `FaceCapMapper` (ARKit-52 -> mesh target names, side-suffix normalization + alias table + JSON override; unmatched ALWAYS reported), `OneEuroFilter` (scalar + hemisphere-aligned quat), `PoseIKSolver` (33 world landmarks -> 22 canonical WORLD quats; torso basis + parallel-transported limb twist reference — continuous, no candy-wrap), `FaceCapCanonicalData.h` (generated constants).
+- **MocapRecorder** (the Ogre-touching piece): `recordFace` writes weight keys via `MorphAnimationManager::writeWeightKeyOn` (the entity-explicit static the #519 path now delegates to) with epsilon suppression + gap hold-keys; head deltas (neutral = first confident frame) land on the Head bone (`<clip>_Head` on the MESH skeleton) or node TRS. `recordBody` feeds `[frame][22]` world quats into `AnimationMerger::applyMotionClip(worldFrame=true)` — the #411 retarget, no fork. `RecordMocapClipCommand`/`RecordBodyClipCommand` make a take ONE undo step (keyframe-level snapshots).
+- **MocapController** (QML_SINGLETON, `PropertiesPanel 1.0`): live mode — camera -> mailbox -> inference worker thread -> queued samples -> main-thread live drive (morph `setWeight` + manual Head bone) with EXACT snapshot/restore (weights, bone state, enabled AnimationStates) around the preview session. Panel: `qml/PropertiesPanel.qml` "Performance Capture" section (Animation-mode Mode Tools). SAM 3D Body is the declared body quality path but its checkpoints are HF-gated — dispatch falls back to pose-ik with `algorithmUsed`/`fallbackReason` until the export is hosted (decision record in `THIRD_PARTY_AI_MODELS.md`).
+- **Surfaces**: CLI `qtmesh mocap` (`src/Mocap/MocapCLI.cpp`, the SceneLightsCLI pattern); MCP `capture_face_from_video`/`capture_body_from_video` (heavy) + `list_capture_devices`/`start_live_capture`/`stop_live_capture` (GUI-attached). Sentry `ai.assist.mocap_face|mocap_body|mocap_live`; gamification cluster `mocap` (cloud-side `DISCOVERY_FEATURES` coordination required). Hosting: `scripts/upload-mocap-models.sh` -> HF models repo `mocap/{face,pose}/` (+ Apache NOTICE). User guide: `docs/MOCAP.md`.
+
 ### PS1 formats (static) and runtime extraction (experimental)
 
 - **Static parsers** (`src/PS1/`): `PS1TMD`, `PS1TIM`, `PS1RSD`, `PS1PLY`, `PS1MAT` for known PlayStation mesh/texture formats.

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -324,6 +324,29 @@ else()
     message(STATUS "Alembic vertex-animation import disabled (VAT_POSE playback still available)")
 endif()
 ##############################################################
+#  Performance capture — video/webcam mocap (epic #869)
+##############################################################
+# Slice B (#871): brings in Qt Multimedia (camera + video decode) and the
+# src/Mocap/ frame-source layer. Default OFF: Qt Multimedia is a new runtime
+# dependency (FFmpeg backend) that packaging has to carry per platform. The
+# ONNX predictors (Slices C/E) additionally require ENABLE_ONNX; requiring it
+# here keeps a single "mocap build" configuration instead of a half-working
+# one. Non-mocap builds print "rebuild with -DENABLE_MOCAP" on every surface.
+option(ENABLE_MOCAP "Enable performance capture (video/webcam mocap)" OFF)
+
+if(ENABLE_MOCAP)
+    if(NOT ENABLE_ONNX)
+        message(FATAL_ERROR "ENABLE_MOCAP requires ENABLE_ONNX (the face/pose "
+                "predictors run on ONNX Runtime). Configure with "
+                "-DENABLE_ONNX=ON -DENABLE_MOCAP=ON.")
+    endif()
+    find_package(Qt6 REQUIRED COMPONENTS Multimedia)
+    add_definitions(-DENABLE_MOCAP)
+    message(STATUS "Performance capture enabled (Qt Multimedia ${Qt6Multimedia_VERSION})")
+else()
+    message(STATUS "Performance capture disabled (rebuild with -DENABLE_MOCAP=ON -DENABLE_ONNX=ON)")
+endif()
+##############################################################
 #  PS1 runtime geometry extraction (experimental)
 ##############################################################
 option(ENABLE_PS1_RIP "Enable experimental PS1 runtime geometry extraction" OFF)

diff --git a/DEBIAN-control.in b/DEBIAN-control.in
@@ -6,7 +6,7 @@ Installed-Size: 757000
 Architecture: amd64 
 Bugs: https://github.com/fernandotonon/QtMeshEditor/issues
 Homepage: https://github.com/fernandotonon/QtMeshEditor 
-Depends: libxcb-cursor0, libqt6core6, libqt6gui6, libqt6widgets6, libqt6network6, libqt6opengl6, libqt6qml6, libqt6quick6
+Depends: libxcb-cursor0, libqt6core6, libqt6gui6, libqt6widgets6, libqt6network6, libqt6opengl6, libqt6qml6, libqt6quick6, libqt6multimedia6
 Maintainer: Fernando Tonon de Rossi <tonon.fernando@hotmail.com> 
 Description: Free 3D asset tool for indie game developers
  Merge Mixamo animations, convert between 40+ 3D formats, edit materials

diff --git a/README.md b/README.md
@@ -286,6 +286,7 @@ Split View|Skeleton Animation Controls
 - **Material editor** — visual editing with AI-assisted generation; PBR + HDR/IBL materials look correct on first import (bundled studio HDRI, ACES tonemap, env intensity/tint per material)
 - **Skeleton inspection** — bone weights, debug overlays, animation preview
 - **Scene management** — duplicate (Ctrl+D), group (Ctrl+G), snap, pivot modes
+- **Performance capture** — video/webcam → facial morph animation (ARKit blendshapes), head pose, and full-body skeletal capture onto humanoid rigs; live preview + record in the editor, `qtmesh mocap` on the CLI (`-DENABLE_MOCAP` builds)
 - **AI chat** — natural language scene editing via local LLMs
 - **MCP server** — 57+ tools for AI agents (Claude, Cursor, etc.), including HDR/IBL (`set_hdr_environment`, `set_tonemap`, …) and QtMesh Cloud (`cloud_*`)
 - **REST API** — HTTP interface for external automation