diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5d721cd73..6b0021f64 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -733,6 +733,8 @@ jobs: host: 'linux' target: 'desktop' arch: 'linux_gcc_64' + # qtmultimedia: performance capture (ENABLE_MOCAP, epic #869) + modules: 'qtmultimedia' - name: change folder permissions run: | @@ -781,6 +783,7 @@ jobs: -DCMAKE_CXX_FLAGS="-g" -DCMAKE_C_FLAGS="-g" \ -DENABLE_STABLE_DIFFUSION=ON \ -DENABLE_ONNX=ON \ + -DENABLE_MOCAP=ON \ -DENABLE_AUTO_UPDATER=OFF \ -DASSIMP_DIR=/usr/local/lib/cmake/assimp-${{ env.ASSIMP_DIR_VERSION }} \ -DASSIMP_INCLUDE_DIR=/usr/local/include/assimp \ @@ -958,6 +961,8 @@ jobs: host: 'linux' target: 'desktop' arch: 'linux_gcc_64' + # qtmultimedia: performance capture (ENABLE_MOCAP, epic #869) + modules: 'qtmultimedia' - name: change folder permissions run: | @@ -1074,7 +1079,8 @@ jobs: -DBUILD_QT_MESH_EDITOR=OFF \ -DENABLE_SENTRY=OFF \ -DENABLE_PS1_RIP=ON \ - -DENABLE_ONNX=ON + -DENABLE_ONNX=ON \ + -DENABLE_MOCAP=ON - name: Run build-wrapper env: @@ -1799,6 +1805,8 @@ jobs: target: 'desktop' arch: 'clang_64' tools: 'tools_cmake' + # qtmultimedia: performance capture (ENABLE_MOCAP, epic #869) + modules: 'qtmultimedia' - name: Debug Qt Installation run: | @@ -1885,6 +1893,7 @@ jobs: -DCMAKE_CXX_FLAGS="-g" -DCMAKE_C_FLAGS="-g" \ -DENABLE_STABLE_DIFFUSION=ON \ -DENABLE_ONNX=ON \ + -DENABLE_MOCAP=ON \ -DCMAKE_OSX_ARCHITECTURES="$(uname -m)" \ -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 \ -DASSIMP_DIR=/usr/local/lib/cmake/assimp-${{ env.ASSIMP_DIR_VERSION }} \ @@ -2135,10 +2144,24 @@ jobs: ls -la ${{github.workspace}}/bin/QtMeshEditor.app/Contents/MacOS/ fi - # Finally sign the entire app bundle - echo "Signing app bundle..." - sudo codesign --force --sign - ${{github.workspace}}/bin/QtMeshEditor.app || echo "Failed to sign app bundle (non-fatal)" - + # Finally sign the entire app bundle WITH the camera/microphone + # entitlements (epic #869). NOTE: this is still an ad-hoc signature + # (`--sign -`); macOS only PROMPTS for camera access when the app + # is signed with a real Apple Developer ID + notarized. Until this + # pipeline gains a Developer ID cert (secret) + notarization step, + # the live-camera capture will be blocked on end-user machines the + # same way it is on an ad-hoc dev build (the CLI `qtmesh mocap` + # video path and file-based capture are unaffected). The + # entitlements + Info.plist NSCameraUsageDescription are in place so + # that flipping to a Developer ID signature is the ONLY remaining + # step to enable the prompt. See docs/MOCAP.md. + echo "Signing app bundle (with camera entitlements)..." + ENT="${{github.workspace}}/cfg/QtMeshEditor.entitlements" + sudo codesign --force --sign - --entitlements "$ENT" \ + ${{github.workspace}}/bin/QtMeshEditor.app \ + || sudo codesign --force --sign - ${{github.workspace}}/bin/QtMeshEditor.app \ + || echo "Failed to sign app bundle (non-fatal)" + echo "Code signing process completed" # Verify the signing diff --git a/.gitignore b/.gitignore index d704d1614..9a56429a7 100755 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ docs/* !docs/IMAGE_TO_3D_QUALITY.md !docs/TRIPOSG_EXPORT_NOTES.md !docs/MESH_SEGMENTATION_STRATEGY.md +!docs/MOCAP.md !docs/FACE_RIG.md !docs/FACE_RIG_SPIKE.md !docs/TELEMETRY.md @@ -157,3 +158,6 @@ __pycache__/ !docs/img/twist_bar_90_lbs.png !docs/img/twist_bar_90_lbs_vp.png !docs/img/twist_bar_90_dqs_vp.png +!docs/MOCAP_SPIKE.md +.mocap_work/ +.venv-mocap/ diff --git a/CLAUDE.md b/CLAUDE.md index a65d61dd5..ff0f77f59 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -157,6 +157,11 @@ qtmesh segment tree.glb --category vegetation # force a category (skip the poin qtmesh segment car.glb --category vehicle # vehicle_body/wheel/window/wing/rotor; `building` = wall/roof/window/door/chimney/foundation qtmesh segment model.fbx --no-model --up-axis y # force the deterministic geometric fallback (skip the ONNX models; auto → body) qtmesh segment rigged.fbx --dump-training-data sample.json # mine EXACT rig-prior labels from a SKINNED mesh → training sample (#410; feed to export-meshseg-onnx.py --real-data) +qtmesh mocap talk.mp4 --face --mesh avatar.glb -o out.glb # performance capture (#869, needs -DENABLE_MOCAP): facial expressions -> ARKit-blendshape weight keyframes + head rotation (Head bone or node); models download on first use +qtmesh mocap dance.mp4 --body --mesh rigged.fbx -o out.glb # full-body pose -> skeletal clip on the humanoid rig (root locked; --algo sam3dbody|pose-ik, sam3dbody falls back to pose-ik while its checkpoints are gated; --no-model forces the fallback) +qtmesh mocap take.mp4 --face --body --mesh char.glb -o out.glb # both in one decode pass ("_Body" for the body clip) +qtmesh mocap clip.mp4 --face --mesh head.glb --map overrides.json --no-head --json # custom channel mapping sidecar; JSON report (matched/unmatched channels always listed) +qtmesh mocap x.mp4 --face --mesh m.glb --frames-dir frames/ # image-sequence input (headless CI/debug; no video decode) qtmesh cloud login # device flow (prints URL + code); stores session locally qtmesh cloud login --api-key # direct API-key login (CI) qtmesh cloud logout # revoke + clear saved session @@ -177,7 +182,7 @@ qtmesh ps1 dump-vram game.cue --bios scph1001.bin --frames 300 -o vram.png # sn # xvfb-run -a qtmesh ps1 capture game.cue --bios scph1001.bin -o out.gltf ``` -CLI mode is activated by: (1) invoking via the `qtmesh` symlink, (2) passing `--cli`, or (3) using a recognized subcommand (`info`, `fix`, `convert`, `anim`, `validate`, `lod`, `pose`, `turntable`, `isometric`, `scan`, `material`, `hdri`, `light`, `pack-textures`, `normal-from-height`, `atlas`, `atlas-apply`, `memory`, `analyze`, `vertex-cache`, `decimate`, `optimize`, `uv`, `retopo`, `skin`, `rig`, `facerig`, `segment`, `generate3d`, `ps1`, `cloud`) as the first argument. Use `--verbose` to see Ogre/engine debug output. Use `--no-telemetry` to permanently opt out of anonymous usage data collection. +CLI mode is activated by: (1) invoking via the `qtmesh` symlink, (2) passing `--cli`, or (3) using a recognized subcommand (`info`, `fix`, `convert`, `anim`, `validate`, `lod`, `pose`, `turntable`, `isometric`, `scan`, `material`, `hdri`, `light`, `pack-textures`, `normal-from-height`, `atlas`, `atlas-apply`, `memory`, `analyze`, `vertex-cache`, `decimate`, `optimize`, `uv`, `retopo`, `skin`, `rig`, `facerig`, `segment`, `generate3d`, `mocap`, `ps1`, `cloud`) as the first argument. Use `--verbose` to see Ogre/engine debug output. Use `--no-telemetry` to permanently opt out of anonymous usage data collection. If Xcode SDK is updated, clear CMake cache (`rm build_local/CMakeCache.txt`) and reconfigure. @@ -340,6 +345,17 @@ The animation pipeline started skeleton-only; the #517 epic broadens it. Slices - **ScanEngine** (`src/ScanEngine.h/cpp`): Directory scanner for 3D asset linting. Loads every asset through `MeshImporterExporter` (the editor's own loader) and walks the resulting Ogre scene with `CLIPipeline::extractMeshInfo` — the same extractor `MeshInfoOverlay` uses, so the scan, the CLI `info` subcommand and the in-app overlay all report identical counts for the same asset. Redundant-keyframe analysis (and the `--fix` write-back since slice C4) goes through `AnimationMerger::analyzeRedundantKeyframes` / `simplifyAnimation`, the same code path as `qtmesh anim --simplify` and the Inspector "Simplify" button. The fix path re-exports via `MeshImporterExporter::exporter` for every supported format (FBX/glTF/glb/DAE/OBJ/PLY/STL/.mesh) — no `Assimp::Exporter`. ACMR is folded into the same Ogre walk so each file is loaded once per scan. Assimp's only remaining role is a no-process `ReadFile` to enumerate `aiMaterial::GetTexture` references that Ogre's TUS-name walk wouldn't see when a referenced texture file is missing on disk (needed for `require_textures_exist`). Quality rules driven by the Ogre walk: `max_texture_resolution` (largest texture dimension cap), `require_uv_channels` (per-submesh UV-set minimum), `detect_zero_weight_bones` (Mixamo bloat — bones with no vertex weights), `detect_overlapping_uvs_pct` (UV0 AABB sweep — lightmap quality), `detect_non_manifold_edges_pct` (edges shared by != 2 faces — boolean / printing safety). Enumerates files via glob patterns, evaluates configurable rules, produces text/JSON/SARIF reports. Per-file cleanup happens in `clearOgreSceneForScanImport` which destroys scene nodes and flushes MeshManager / SkeletonManager so a 1000-asset scan doesn't accumulate state. - **ScanConfig** (`src/ScanConfig.h/cpp`): Config loader for `qtmesh.yml`/`.json`. Includes a minimal YAML parser for the specific config schema (scalars, inline/block lists, one level of section nesting). Supports scan paths, rule configuration, fix behavior, and report output settings. +### Performance capture (epic #869, `ENABLE_MOCAP`) + +Video/webcam -> facial morph + head + skeletal body animation, built on the existing ONNX/model-download/retarget/morph-keyframe infrastructure. Everything lives in `src/Mocap/`; the CMake flag `ENABLE_MOCAP` (default OFF; ON for macOS/Linux release + the Linux test lane) requires `ENABLE_ONNX` and pulls in **Qt6::Multimedia** (a new dependency — Windows/MinGW pending verification, so OFF there). Non-mocap builds print "rebuild with -DENABLE_MOCAP" on every surface. + +- **VideoFrameSource** (`src/Mocap/VideoFrameSource.{h,cpp}`): the frame abstraction — `FileFrameSource` (QMediaPlayer+QVideoSink, targetFps decimation, playback-driven), `CameraFrameSource` (QCamera, device enumeration, latest-wins `FrameMailbox` so live inference never falls behind), `ImageSequenceFrameSource` (headless test double / CLI `--frames-dir`). `FrameDecimator`/`FrameMailbox` are pure + tested. +- **FaceCapPredictor** (ONNX consumer #9): 3 sessions (BlazeFace detector 128² [-1,1] -> Face Mesh V2 rotated 256² crop [0,1] -> 478 landmarks -> 146-subset px coords -> 52 blendshape scores) with detector-skip tracking (next ROI from previous landmarks). **PoseCapPredictor** (consumer #10): BlazePose detector+landmarks -> 33 world landmarks; tracking from the model's aux alignment landmarks. Both download to `AppData/ai_models/mocap/{face,pose}/` (`QTMESH_MOCAP_MODEL_BASE_URL` / `ai/mocapModelBaseUrl` / `QTMESH_MOCAP_NO_DOWNLOAD`). The exact pre/post-processing contract (anchors, letterbox, weighted NMS, **cv2-integer-index plain bilinear — never antialiased**, projection) was parity-proven in the Slice A spike: `docs/MOCAP_SPIKE.md`, `scripts/export-facecap-onnx.py` (converts AND asserts parity ≤0.6px/≤0.015 blendshape/≤1cm world). +- **Pure-data core** (all headless-tested): `FaceCapGeom` (letterbox/anchors/decode/NMS/ROI/crop/projection), `FaceCapPose` (Horn quaternion weighted rigid fit + embedded canonical face model — no Eigen), `FaceCapMapper` (ARKit-52 -> mesh target names, side-suffix normalization + alias table + JSON override; unmatched ALWAYS reported), `OneEuroFilter` (scalar + hemisphere-aligned quat), `PoseIKSolver` (33 world landmarks -> 22 canonical WORLD quats; torso basis + parallel-transported limb twist reference — continuous, no candy-wrap), `FaceCapCanonicalData.h` (generated constants). +- **MocapRecorder** (the Ogre-touching piece): `recordFace` writes weight keys via `MorphAnimationManager::writeWeightKeyOn` (the entity-explicit static the #519 path now delegates to) with epsilon suppression + gap hold-keys; head deltas (neutral = first confident frame) land on the Head bone (`_Head` on the MESH skeleton) or node TRS. `recordBody` feeds `[frame][22]` world quats into `AnimationMerger::applyMotionClip(worldFrame=true)` — the #411 retarget, no fork. `RecordMocapClipCommand`/`RecordBodyClipCommand` make a take ONE undo step (keyframe-level snapshots). +- **MocapController** (QML_SINGLETON, `PropertiesPanel 1.0`): live mode — camera -> mailbox -> inference worker thread -> queued samples -> main-thread live drive (morph `setWeight` + manual Head bone) with EXACT snapshot/restore (weights, bone state, enabled AnimationStates) around the preview session. Panel: `qml/PropertiesPanel.qml` "Performance Capture" section (Animation-mode Mode Tools). SAM 3D Body is the declared body quality path but its checkpoints are HF-gated — dispatch falls back to pose-ik with `algorithmUsed`/`fallbackReason` until the export is hosted (decision record in `THIRD_PARTY_AI_MODELS.md`). +- **Surfaces**: CLI `qtmesh mocap` (`src/Mocap/MocapCLI.cpp`, the SceneLightsCLI pattern); MCP `capture_face_from_video`/`capture_body_from_video` (heavy) + `list_capture_devices`/`start_live_capture`/`stop_live_capture` (GUI-attached). Sentry `ai.assist.mocap_face|mocap_body|mocap_live`; gamification cluster `mocap` (cloud-side `DISCOVERY_FEATURES` coordination required). Hosting: `scripts/upload-mocap-models.sh` -> HF models repo `mocap/{face,pose}/` (+ Apache NOTICE). User guide: `docs/MOCAP.md`. + ### PS1 formats (static) and runtime extraction (experimental) - **Static parsers** (`src/PS1/`): `PS1TMD`, `PS1TIM`, `PS1RSD`, `PS1PLY`, `PS1MAT` for known PlayStation mesh/texture formats. diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d40753f4..ca828ea0b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -324,6 +324,29 @@ else() message(STATUS "Alembic vertex-animation import disabled (VAT_POSE playback still available)") endif() ############################################################## +# Performance capture — video/webcam mocap (epic #869) +############################################################## +# Slice B (#871): brings in Qt Multimedia (camera + video decode) and the +# src/Mocap/ frame-source layer. Default OFF: Qt Multimedia is a new runtime +# dependency (FFmpeg backend) that packaging has to carry per platform. The +# ONNX predictors (Slices C/E) additionally require ENABLE_ONNX; requiring it +# here keeps a single "mocap build" configuration instead of a half-working +# one. Non-mocap builds print "rebuild with -DENABLE_MOCAP" on every surface. +option(ENABLE_MOCAP "Enable performance capture (video/webcam mocap)" OFF) + +if(ENABLE_MOCAP) + if(NOT ENABLE_ONNX) + message(FATAL_ERROR "ENABLE_MOCAP requires ENABLE_ONNX (the face/pose " + "predictors run on ONNX Runtime). Configure with " + "-DENABLE_ONNX=ON -DENABLE_MOCAP=ON.") + endif() + find_package(Qt6 REQUIRED COMPONENTS Multimedia) + add_definitions(-DENABLE_MOCAP) + message(STATUS "Performance capture enabled (Qt Multimedia ${Qt6Multimedia_VERSION})") +else() + message(STATUS "Performance capture disabled (rebuild with -DENABLE_MOCAP=ON -DENABLE_ONNX=ON)") +endif() +############################################################## # PS1 runtime geometry extraction (experimental) ############################################################## option(ENABLE_PS1_RIP "Enable experimental PS1 runtime geometry extraction" OFF) diff --git a/DEBIAN-control.in b/DEBIAN-control.in index 68fd1dbd9..63ff981dd 100644 --- a/DEBIAN-control.in +++ b/DEBIAN-control.in @@ -6,7 +6,7 @@ Installed-Size: 757000 Architecture: amd64 Bugs: https://github.com/fernandotonon/QtMeshEditor/issues Homepage: https://github.com/fernandotonon/QtMeshEditor -Depends: libxcb-cursor0, libqt6core6, libqt6gui6, libqt6widgets6, libqt6network6, libqt6opengl6, libqt6qml6, libqt6quick6 +Depends: libxcb-cursor0, libqt6core6, libqt6gui6, libqt6widgets6, libqt6network6, libqt6opengl6, libqt6qml6, libqt6quick6, libqt6multimedia6 Maintainer: Fernando Tonon de Rossi Description: Free 3D asset tool for indie game developers Merge Mixamo animations, convert between 40+ 3D formats, edit materials diff --git a/README.md b/README.md index 8ac2ee432..e433ff14b 100755 --- a/README.md +++ b/README.md @@ -286,6 +286,7 @@ Split View|Skeleton Animation Controls - **Material editor** — visual editing with AI-assisted generation; PBR + HDR/IBL materials look correct on first import (bundled studio HDRI, ACES tonemap, env intensity/tint per material) - **Skeleton inspection** — bone weights, debug overlays, animation preview - **Scene management** — duplicate (Ctrl+D), group (Ctrl+G), snap, pivot modes +- **Performance capture** — video/webcam → facial morph animation (ARKit blendshapes), head pose, and full-body skeletal capture onto humanoid rigs; live preview + record in the editor, `qtmesh mocap` on the CLI (`-DENABLE_MOCAP` builds) - **AI chat** — natural language scene editing via local LLMs - **MCP server** — 57+ tools for AI agents (Claude, Cursor, etc.), including HDR/IBL (`set_hdr_environment`, `set_tonemap`, …) and QtMesh Cloud (`cloud_*`) - **REST API** — HTTP interface for external automation diff --git a/THIRD_PARTY_AI_MODELS.md b/THIRD_PARTY_AI_MODELS.md index 27af2d523..cc53c3569 100644 --- a/THIRD_PARTY_AI_MODELS.md +++ b/THIRD_PARTY_AI_MODELS.md @@ -14,7 +14,13 @@ the binary). Attribution + licenses for the models and their training data: > [`QtMeshEditor-pbrify-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-pbrify-onnx), > [`QtMeshEditor-realesrgan-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-realesrgan-onnx), > [`QtMeshEditor-u2net-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-u2net-onnx), -> [`QtMeshEditor-smolvlm-gguf`](https://huggingface.co/fernandotonon/QtMeshEditor-smolvlm-gguf) +> [`QtMeshEditor-smolvlm-gguf`](https://huggingface.co/fernandotonon/QtMeshEditor-smolvlm-gguf), +> and the five performance-capture (#869) MediaPipe graphs +> [`QtMeshEditor-blazeface-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-blazeface-onnx), +> [`QtMeshEditor-facemesh-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-facemesh-onnx), +> [`QtMeshEditor-faceblendshapes-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-faceblendshapes-onnx), +> [`QtMeshEditor-blazepose-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-blazepose-onnx), +> [`QtMeshEditor-poselandmarks-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-poselandmarks-onnx) > (plus the in-house > [`QtMeshEditor-rmib-inbetween`](https://huggingface.co/fernandotonon/QtMeshEditor-rmib-inbetween), > [`QtMeshEditor-mesh-segmentation`](https://huggingface.co/fernandotonon/QtMeshEditor-mesh-segmentation), @@ -330,6 +336,65 @@ the binary). Attribution + licenses for the models and their training data: 2d106det) were rejected — their weights carry research-only / non-commercial terms; MediaPipe FaceMesh (Apache-2.0) is the clean choice. +## Performance capture (epic #869) + +### MediaPipe Face Landmarker + Pose Landmarker — face/pose capture (#870/#872/#874) + +- **Models:** Google MediaPipe `face_landmarker.task` (BlazeFace short-range + detector + Face Mesh V2 478-landmark model + 52-blendshape MLP-Mixer) and + `pose_landmarker_full.task` (BlazePose detector + 39-landmark model with + world coordinates), converted TFLite → ONNX. +- **License:** **Apache-2.0, code AND models** (Google's MediaPipe release — + the stack the entire VTuber ecosystem builds on). Ship the Apache-2.0 + notice next to the hosted weights. +- The ONNX export is produced by `scripts/export-facecap-onnx.py` (one-time, + offline dev tool — not shipped), which also **asserts numerical parity** + against the Python `mediapipe` reference (landmarks ≤ 0.59 px, blendshapes + ≤ 0.0148 abs, pose world landmarks ≤ 1.02 cm on the test set). Conversion + recipe, pre/post-processing contracts and measured latencies: + `docs/MOCAP_SPIKE.md`. The app runs the five graphs via ONNX Runtime, + downloading them on first use to `AppData/ai_models/mocap/{face,pose}/`. +- **Rejected face alternatives:** DECA / EMOCA / SPECTRE (all regress the + FLAME 3DMM — research-only license), ARKit (iOS-only), OpenSeeFace (MIT + code but weaker blendshape story). + +### SAM 3D Body + MHR — body capture quality path (#870/#874) — decision record + +- **MHR (Momentum Human Rig)** — the 127-joint parametric rig SAM 3D Body + poses — is **Apache-2.0** (https://github.com/facebookresearch/MHR, assets + v1.0.1). Skeleton definition (names, hierarchy, pre-rotations, derived rest + world pose) extracted to `mhr_skeleton.json` by + `scripts/export-bodycap-onnx.py --mhr-assets`. Apache-2.0 permits use, + modification, and redistribution (including our derived `mhr_skeleton.json`) + for any purpose incl. commercial, provided the LICENSE + NOTICE are retained + and modifications are marked — we ship the Apache-2.0 notice next to the + derived skeleton on the HF repo. No copyleft, no field-of-use restriction. +- **SAM 3D Body checkpoints** (`facebook/sam-3d-body-dinov3`, 2.1 GB) are + under the **SAM License** (2025-11-19 text reviewed in full, 2026-07-12). + **Verdict: PASS with conditions** — usable as an OPTIONAL, + downloaded-on-first-use backend, never bundled: + - §1.a grants use/reproduction/distribution/modification (ONNX conversion + is a permitted modification, not the prohibited "reverse engineering" of + §1.b.iv). §1.b.i permits redistributing derivatives **only under the SAM + License with a copy attached** — rehosting converted ONNX on our HF repo + is compliant with the license file shipped next to the weights. + - **No non-commercial clause; model outputs are unrestricted** (§3 only + disclaims warranty over outputs). No EU exclusion (unlike Hunyuan3D). + - Conditions/risks recorded: the SAM backend is NOT permissive-equivalent — + users of that one optional feature are bound by the SAM License + (AUP-style trade-controls/military restrictions pass through); Meta may + unilaterally amend the terms (§8); upstream access is gated (HF + click-through sharing contact info with Meta). + - **Status:** checkpoint download blocked pending the gated-access + acceptance on the HF model page (our token is not yet on the authorized + list). Export recipe proven by the community port + (AmmarkoV/SAM3DBody-cpp: DINOv3-H+ backbone ~4.8 GB fp32 + 93 MB decoder + → 519 MHR params). Until unblocked the body path ships **MediaPipe Pose + + analytic IK only** (Apache-2.0, zero conditions). +- **Rejected body alternatives:** WHAM / GVHMR / TRAM / 4D-Humans (all regress + **SMPL** — weights non-commercial, Meshcapade sells the commercial + license), OpenPose (CMU non-commercial), FreeMoCap (AGPL). + All of the above clear QtMeshEditor's permissive-redistribution bar (MIT app, distributed via Homebrew / WinGet / Snap / Docker). GPL/CC-BY-NC/unlicensed models are deliberately excluded (e.g. RigNet was rejected for #408 — GPL code + diff --git a/action.yml b/action.yml index 94880a853..3e86059c5 100644 --- a/action.yml +++ b/action.yml @@ -8,7 +8,7 @@ branding: inputs: command: - description: 'Subcommand: scan, info, validate, convert, fix, anim, lod, pose, turntable, isometric, generate3d' + description: 'Subcommand: scan, info, validate, convert, fix, anim, lod, pose, turntable, isometric, generate3d, mocap' required: true input-file: description: 'Directory or file to scan (relative to workspace). Defaults to . (workspace root).' diff --git a/cfg/QtMeshEditor.entitlements b/cfg/QtMeshEditor.entitlements new file mode 100644 index 000000000..4498a5ca1 --- /dev/null +++ b/cfg/QtMeshEditor.entitlements @@ -0,0 +1,23 @@ + + + + + + com.apple.security.device.camera + + com.apple.security.device.microphone + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-library-validation + + + diff --git a/docs/MOCAP.md b/docs/MOCAP.md new file mode 100644 index 000000000..8f442e120 --- /dev/null +++ b/docs/MOCAP.md @@ -0,0 +1,120 @@ +# Performance Capture (mocap) + +Record a video of yourself — or point a webcam at yourself live — and +reproduce the performance on a mesh: facial expressions land as morph-target +weight keyframes, head rotation on the Head bone (or the node for static +meshes), and full-body pose as a skeletal clip on a humanoid rig. Everything +is recorded as ordinary animation clips: they play on the timeline, show in +the dope sheet, undo with a single Ctrl+Z, and export through the normal +exporters. + +Requires a build with `-DENABLE_MOCAP=ON -DENABLE_ONNX=ON` (release macOS + +Linux builds ship with it; Windows/MinGW is pending Qt Multimedia +verification). Models (all Apache-2.0 Google MediaPipe conversions, ~31 MB +total) download on first use to `/ai_models/mocap/{face,pose}/`. + +## Quick start + +```bash +# facial performance from a video onto an ARKit-blendshape head: +qtmesh mocap talk.mp4 --face --mesh avatar.glb -o out.glb + +# full-body performance onto a rigged humanoid: +qtmesh mocap dance.mp4 --body --mesh rigged.fbx -o out.glb + +# both in one pass over the same video: +qtmesh mocap take.mp4 --face --body --mesh character.glb -o out.glb +``` + +Live mode: **Animation Mode → Mode Tools → Performance Capture** — pick a +camera, `Preview` drives the selection in real time, `● Record` writes the +take as a clip (status line shows the result; Ctrl+Z discards it). + +## What the mesh needs + +- **Face capture** drives **ARKit-style blendshape morph targets** + (`jawOpen`, `mouthSmileLeft`, `eyeBlinkRight`, `browInnerUp`, … — the + 52-name vocabulary MediaPipe/ARKit standardized). Sources of compatible + heads: [Ready Player Me](https://readyplayer.me) avatars export all 52; + Character Creator / iClone heads use recognized aliases (`Jaw_Open`, + `Mouth_Smile_L`, …). You can also author targets manually in the + Edit-Mode **Vertex Morph Animation** section. + Name matching is forgiving (case/separators/`_L`/`.R` side suffixes all + normalize), and **unmatched channels are always reported**, never silently + dropped. Custom names bind via a JSON override sidecar: + + ```json + { "map": { "jawOpen": "MyJawTarget" }, "ignore": ["tongueOut"] } + ``` + + passed as `--map overrides.json` (CLI) / `map_path` (MCP). + +- **Head pose** needs a bone that resolves as the canonical Head + (`Head`, `mixamorig:Head`, …). Static meshes get node-TRS keyframes + instead. The take's first confident frame calibrates neutral ("look at + the camera at the start"); in live mode the `Neutral` button re-bases it. + +- **Body capture** needs a **humanoid skeleton** resolving at least half of + the 22 canonical roles (hips/spine/neck/head, both arms, both legs — + Mixamo and most generic naming conventions resolve). Unrigged meshes: run + `qtmesh rig --skeleton humanoid --skin` first. The root stays locked to + the standing pose (v1 accepts some foot slide). + +## Backends + +- **Face / head / body-fallback**: converted Google MediaPipe models + (Apache-2.0), small and fast — the face pipeline runs ~90 fps on an + M-series CPU. +- **Body quality path (`--algo sam3dbody`)**: Meta's SAM 3D Body is wired as + the preferred backend but its checkpoints are gated behind a license + click-through, so today every request falls back to the analytic + **pose-ik** backend with `algorithmUsed`/`fallbackReason` in the report. + See `THIRD_PARTY_AI_MODELS.md` for the licensing decision record. + `--algo pose-ik` / `--no-model` force the fallback explicitly. + +## Tuning + +- `--fps N` (default 30): capture rate; video frames are decimated to it. +- `--smooth-cutoff HZ` (default 1.0) / `--no-smooth`: One-Euro filter + minimum cutoff — lower is smoother at rest, `beta` tracks fast motion. + The live panel exposes the same smoothing. +- `--clip-name NAME`: default `FaceCap` / `BodyCap` (`_Head` for the + head track, `_Body` when `--face --body` run together). +- `--frames-dir DIR`: use an image sequence instead of a video (headless + debugging/CI; no video decode involved). +- Model base URL override: `QTMESH_MOCAP_MODEL_BASE_URL` / + `QSettings ai/mocapModelBaseUrl`; offline guard `QTMESH_MOCAP_NO_DOWNLOAD`. + +## macOS camera permission + +On first **Preview**, macOS asks to allow camera access; click Allow and the +app then appears under System Settings → Privacy & Security → Camera. For the +prompt to appear the app must be **signed with an Apple Developer ID and +notarized** — macOS silently denies camera access to ad-hoc-signed apps +(no dialog, and the app never shows in the Camera list). The entitlements +(`cfg/QtMeshEditor.entitlements`) and `NSCameraUsageDescription` are in place, +so an official notarized release build prompts normally; a locally-built / +ad-hoc dev build will NOT get the prompt. If you hit that on a dev build, use +the file-based capture (`qtmesh mocap