From 8d707b632c86a0d9c4e7cc2d2c8e2005197a4e57 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 3 Aug 2026 23:15:05 -0400 Subject: [PATCH 01/11] fix(mocap): retarget live body drive via landmark directions Drive Mixamo skeletons from MediaPipe landmark segment directions (same aim math as applyMotionClip) instead of quaternion deltas and mirror-L/R swaps. Adds PoseIK debug overlay, per-frame skinning refresh, and tests. Co-authored-by: Cursor --- docs/MOCAP.md | 33 +- qml/PropertiesPanel.qml | 38 ++- src/AnimationMerger.cpp | 408 ++++++++++++++++++----- src/AnimationMerger.h | 53 ++- src/AnimationMerger_test.cpp | 217 +++++++++++++ src/CMakeLists.txt | 2 + src/Mocap/FaceCapPose_test.cpp | 19 ++ src/Mocap/MocapBodyDriveDebug.cpp | 295 +++++++++++++++++ src/Mocap/MocapBodyDriveDebug.h | 36 ++ src/Mocap/MocapCameraHints.cpp | 15 + src/Mocap/MocapCameraHints.h | 5 + src/Mocap/MocapController.cpp | 488 +++++++++++++++++++++------- src/Mocap/MocapController.h | 6 + src/Mocap/MocapLiveTypes.h | 21 ++ src/Mocap/MocapPoseDebugOverlay.cpp | 304 +++++++++++++++++ src/Mocap/MocapPoseDebugOverlay.h | 36 ++ src/Mocap/MocapPoseFix.h | 98 ++++++ src/Mocap/MocapRecorder.cpp | 13 +- src/Mocap/PoseIKSolver.cpp | 135 +++++++- src/Mocap/PoseIKSolver.h | 26 +- src/Mocap/PoseIKSolver_test.cpp | 90 +++++ src/Mocap/VideoFrameSource.cpp | 90 ++++- src/Mocap/VideoFrameSource.h | 7 +- src/Mocap/VideoFrameSource_test.cpp | 12 + src/test_main.cpp | 7 + 25 files changed, 2195 insertions(+), 259 deletions(-) create mode 100644 src/Mocap/MocapBodyDriveDebug.cpp create mode 100644 src/Mocap/MocapBodyDriveDebug.h create mode 100644 src/Mocap/MocapLiveTypes.h create mode 100644 src/Mocap/MocapPoseDebugOverlay.cpp create mode 100644 src/Mocap/MocapPoseDebugOverlay.h create mode 100644 src/Mocap/MocapPoseFix.h diff --git a/docs/MOCAP.md b/docs/MOCAP.md index 0b17bef5c..ba4111451 100644 --- a/docs/MOCAP.md +++ b/docs/MOCAP.md @@ -51,14 +51,17 @@ take as a clip (status line shows the result; Ctrl+Z discards it). - **Head pose** needs a bone that resolves as the canonical Head (`Head`, `mixamorig:Head`, …). Static meshes get node-TRS keyframes - instead. The take's first confident frame calibrates neutral ("look at - the camera at the start"); in live mode the `Neutral` button re-bases it. + instead. The first confident preview frame is the neutral reference + (look at the camera, relaxed face); the **Neutral** button re-bases head + and body. - **Body capture** needs a **humanoid skeleton** resolving at least half of the 22 canonical roles (hips/spine/neck/head, both arms, both legs — Mixamo and most generic naming conventions resolve). Unrigged meshes: run `qtmesh rig --skeleton humanoid --skin` first. The root stays locked to - the standing pose (v1 accepts some foot slide). + the standing pose (v1 accepts some foot slide). Body limbs calibrate on + the first visible frame too — start preview with arms in a natural rest + pose (similar to the character's idle) so raised/movement reads correctly. ## Backends @@ -116,17 +119,31 @@ The Snap is strictly confined, so webcam access needs two things: panel suggests the `snap connect` command when no cameras appear or permission is denied. +If Preview stays on **“Starting camera…”** for more than a few seconds, the +camera opened but Qt never decoded frames. On Linux builds this was usually +(a) missing `libqjpeg` imageformat plugin for MJPEG webcams, or (b) a missing +FFmpeg stub — both are bundled from 3.25.4 onward. Close other apps using +the webcam, retry Preview, or use **Load Video…** as a workaround. + On desktop Linux outside Snap, allow camera access via your desktop portal / -privacy settings when prompted. +privacy settings when prompted. Qt's FFmpeg backend probes VA-API during camera +enumeration; on some NVIDIA + X11 setups that probe can crash unless hardware +decode is disabled — the app sets `QT_FFMPEG_DECODING_HW_DEVICE_TYPES=,` at +startup (software decode only; fine for live webcam preview). ## Known limitations (v1) - Single person per frame; the highest-scoring detection wins. - Head pose is camera-relative — walking around the camera reads as head - rotation. Keep the camera static. -- Body root is locked (no root motion); some foot slide is expected. -- Live mode drives face, head, and (humanoid rig) body; the SAM 3D Body - quality backend is offline-only (CLI/MCP), body-live uses pose-ik. + rotation. Keep the camera static. Up/down (pitch) is corrected for + Mixamo-style rigs. +- Body retargeting uses MediaPipe landmark directions (same geometry as the + PoseIK debug overlay) to aim skeleton bones — no mirror-L/R toggle. +- Body root is locked (no root motion); some foot slide is expected. Live + pose-ik uses anatomical bone names (no CMU L/R swap) and CMU-aligned solver + output; recorded body clips use the same path. +- Live mode drives face, head, and (humanoid rig) body; when Face + Body are + both enabled, head rotation always comes from the face graph (not PoseIK). - Live camera needs a notarized build on macOS (see above); the CLI/MCP video paths work regardless. - Video decode is playback-driven (a 60 s video takes 60 s to capture). diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index b70159359..6fa3fa6f8 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -635,6 +635,14 @@ Rectangle { Component.onCompleted: content = performanceCaptureComponent + // Enumerate cameras only after the deferred Loader has + // finished instantiating the device combo (QMediaDevices during + // synchronous Loader startup can crash when combined with a + // devicesChanged rebinding cascade). + onContentReady: Qt.callLater(function() { + MocapController.refreshDevices() + }) + // never leave the camera running when the section disappears // (mode change / deselect) — the AutoRig marker-session // precedent. @@ -2351,9 +2359,11 @@ Rectangle { color: PropertiesPanelController.textColor font.pixelSize: 10 text: mocapReady - ? "Webcam performance capture: Preview drives the selection's " - + "morph targets, Head bone, and (on a humanoid rig) full " - + "body live; Record writes undoable clips onto the timeline." + ? "Webcam performance capture: Preview drives morph targets, " + + "Head bone, and (on a humanoid rig) body live; Record " + + "writes undoable clips. On start the first frame " + + "calibrates neutral — face the camera with arms relaxed " + + "at your sides. Use Neutral to re-base if you moved." : MocapController.unavailableReason } @@ -2400,7 +2410,6 @@ Rectangle { textRole: "description" valueRole: "id" enabled: MocapController.state === 0 - Component.onCompleted: MocapController.refreshDevices() } Rectangle { @@ -2435,6 +2444,27 @@ Rectangle { } } + InspectorCheckBox { + width: parent.width - 16 + visible: mocapReady && MocapController.bodyAvailable + text: "Show PoseIK debug skeleton" + checked: MocapController.showPoseDebug + onToggled: MocapController.showPoseDebug = checked + } + + Text { + width: parent.width - 16 + visible: mocapReady && MocapController.showPoseDebug + wrapMode: Text.Wrap + opacity: 0.75 + color: PropertiesPanelController.textColor + font.pixelSize: 9 + text: "Debug (beside character): cyan = MediaPipe landmarks, " + + "yellow = PoseIK 22-bone FK. If cyan matches you but " + + "Mixamo does not, retarget is wrong; if cyan is wrong, " + + "capture or lighting is wrong." + } + // Video-file source — the path for macOS where the camera is // blocked/unavailable. Picks a file and drives the selection from // it (same live preview + recording as the webcam). diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 1238fff8d..6c8dd686a 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -18,6 +18,10 @@ #include #include +#ifdef ENABLE_MOCAP +#include "Mocap/PoseIKSolver.h" +#endif + // Registry: skeleton name → up-axis (1=Y-up, 2=Z-up). // Populated by AnimationMerger::registerSkeletonUpAxis() at import time. static QMap s_skeletonUpAxis; @@ -1435,13 +1439,77 @@ TargetBindFrame readTargetBindFrame(Ogre::Skeleton* skel, if (!placed[static_cast(i)]) tb.order.push_back(i); return tb; } + +// HANDEDNESS COMPENSATION (shared by applyMotionClip + BodyRetargeter). Bone +// names are anatomically correct, but a mesh may face opposite CMU — then the +// rig's "Left" bones sit on −X while canonical left joints expect +X. Detect +// from world-X of a left vs right bone pair and swap canonical L/R indices. +void compensateCanonicalHandedness(Ogre::Skeleton* skel, + std::vector& boneToCanon) +{ + if (!skel) return; + skel->reset(true); + skel->_updateTransforms(); + const int nBones = static_cast(boneToCanon.size()); + auto worldXForCanon = [&](int canon) -> double { + for (int i = 0; i < nBones; ++i) + if (boneToCanon[static_cast(i)] == canon) + return skel->getBone(static_cast(i)) + ->_getDerivedPosition().x; + return 0.0; + }; + double lx = worldXForCanon(11), rx = worldXForCanon(7); // arms + if (std::abs(lx - rx) < 1e-4) { + lx = worldXForCanon(19); + rx = worldXForCanon(15); // legs + } + if (lx < rx - 1e-5) { + static const int kLR[][2] = {{6, 10}, {7, 11}, {8, 12}, {9, 13}, + {14, 18}, {15, 19}, {16, 20}, {17, 21}}; + auto swapCanon = [&](int& c) { + for (auto& p : kLR) { + if (c == p[0]) { + c = p[1]; + return; + } + if (c == p[1]) { + c = p[0]; + return; + } + } + }; + for (int i = 0; i < nBones; ++i) + if (boneToCanon[static_cast(i)] >= 0) + swapCanon(boneToCanon[static_cast(i)]); + } +} + } // namespace +namespace { +// Same parent indices applyMotionClip uses for world-frame PoseIK quats. +constexpr int kParentCanon[22] = { + -1, 0, 1, 2, 3, 4, // hip, abdomen, chest, neck, neck1, head + 2, 6, 7, 8, // rcollar, rshoulder, relbow, rhand + 2, 10, 11, 12, // lcollar, lshoulder, lelbow, lhand + 0, 14, 15, 16, // rbuttock, rhip, rknee, rfoot + 0, 18, 19, 20 }; // lbuttock, lhip, lknee, lfoot + +// PoseIK leaves collar/buttock at identity (unresolved) — walk up to chest/hip +// for parent-relative articulation, matching MocapPoseDebugOverlay / PoseIK. +int effectiveParentRole(int role, uint32_t resolvedMask) +{ + int p = kParentCanon[role]; + while (p >= 0 && !(resolvedMask & (1u << static_cast(p)))) + p = kParentCanon[p]; + return p; +} +} // namespace + // ── BodyRetargeter — the applyMotionClip direction-match, per single frame ── struct BodyRetargeter::Impl { TargetBindFrame tb; std::vector boneToCanon; // per bone - Ogre::Quaternion CtInv = Ogre::Quaternion::IDENTITY; int nBones = 0; int Jc = 0; // Harvested STANDING pose per bone (the calmest frame of the rig's existing @@ -1451,18 +1519,53 @@ struct BodyRetargeter::Impl { std::vector standLocal; // per bone std::vector haveStand; // per bone std::vector Mc, McInv; // per bone (roll correction) + std::vector canonDup; // bones sharing a canonical role bool restsAreIdentity = true; bool haveAnyStand = false; - // neutral (first-frame) canonical quats — the reference the per-frame - // local-articulation delta is taken against. + // neutral (first-frame) canonical WORLD quats — reference for PoseIK alignment. std::array, 22> neutral{}; + uint32_t neutralResolvedMask = 0; bool haveNeutral = false; + bool neutralHadTorso = false; + bool yaw180 = false; +#ifdef ENABLE_MOCAP + // Landmark direction retarget (live mocap): neutral ref dirs in skeleton + // world + per-bone Qbase (bind aligned to neutral), same as applyMotionClip. + std::vector neutralDref; // per canonical role + std::vector dirQbase; // per bone + bool haveNeutralDir = false; +#endif }; -BodyRetargeter::BodyRetargeter(Ogre::Skeleton* skel) +#ifdef ENABLE_MOCAP +namespace { +void collectCanonicalLiveDirections( + const float* world33, const float* visibility33, int Jc, + std::vector& outCanonDir) +{ + std::array, PoseIK::kLandmarkCount> canon{}; + PoseIK::Solver::canonicalizeMediaPipeWorld(world33, canon); + outCanonDir.assign(static_cast(Jc), Ogre::Vector3::ZERO); + for (int c = 0; c < Jc; ++c) { + std::array dir{}; + if (PoseIK::Solver::canonicalLiveDirection( + c, canon, visibility33, 0.3f, dir)) { + Ogre::Vector3 v(dir[0], dir[1], dir[2]); + if (v.squaredLength() > 1e-12f) { + v.normalise(); + outCanonDir[static_cast(c)] = v; + } + } + } +} +} // namespace +#endif + +BodyRetargeter::BodyRetargeter(Ogre::Skeleton* skel, bool yaw180) { if (!skel) return; d = std::make_shared(); + d->yaw180 = yaw180; d->nBones = static_cast(skel->getNumBones()); d->Jc = MotionInbetween::canonicalJointCount(); d->boneToCanon.assign(static_cast(d->nBones), -1); @@ -1470,8 +1573,14 @@ BodyRetargeter::BodyRetargeter(Ogre::Skeleton* skel) d->boneToCanon[static_cast(i)] = MotionInbetween::canonicalIndexForBone(QString::fromStdString( skel->getBone(static_cast(i))->getName())); + // Pose-ik mocap: anatomical name→role only (NO CMU handedness swap — that + // swap is for BVH/library clips and mirrors live limb motion on Mixamo). d->tb = readTargetBindFrame(skel, d->boneToCanon); - d->CtInv = d->tb.Ct.Inverse(); + + d->canonDup.assign(static_cast(d->Jc), 0); + for (int i = 0; i < d->nBones; ++i) + if (d->boneToCanon[static_cast(i)] >= 0) + ++d->canonDup[static_cast(d->boneToCanon[static_cast(i)])]; // ── Harvest the STANDING pose (calmest frame of the rig's first authored, // non-generated animation) — the same pose applyMotionClip's legacy @@ -1550,92 +1659,238 @@ BodyRetargeter::BodyRetargeter(Ogre::Skeleton* skel) m_valid = mapped * 2 >= d->Jc; } +bool BodyRetargeter::hasNeutralReference() const +{ + return m_valid && d && d->haveNeutral; +} + +void BodyRetargeter::setNeutralReference( + const std::array, 22>& canonicalQuats, + uint32_t resolvedMask, + const float* mediaPipeWorld33, + const float* mediaPipeVisibility33) +{ + if (!m_valid || !d) + return; + const TargetBindFrame& tb = d->tb; + const int Jc = d->Jc; + auto clipQ = [&](const std::array, 22>& src, int joint) + -> Ogre::Quaternion { + const auto& q = src[static_cast(joint)]; + return Ogre::Quaternion(q[3], q[0], q[1], q[2]); + }; + d->neutral = canonicalQuats; + d->neutralResolvedMask = resolvedMask; + d->haveNeutral = true; + constexpr uint32_t kTorsoMask = + (1u << 0) | (1u << 1) | (1u << 2); // hip, abdomen, chest + d->neutralHadTorso = (resolvedMask & kTorsoMask) == kTorsoMask; + if (d->haveAnyStand && d->restsAreIdentity && d->neutralHadTorso) { + std::vector standW(static_cast(d->nBones), + Ogre::Quaternion::IDENTITY); + for (int i : tb.order) { + const int pi = tb.parentIdx[static_cast(i)]; + const Ogre::Quaternion Wp = (pi >= 0) + ? standW[static_cast(pi)] : Ogre::Quaternion::IDENTITY; + const Ogre::Quaternion localRot = d->haveStand[static_cast(i)] + ? d->standLocal[static_cast(i)] + : tb.bindLocal[static_cast(i)]; + standW[static_cast(i)] = Wp * localRot; + } + for (int i = 0; i < d->nBones; ++i) { + const int c = d->boneToCanon[static_cast(i)]; + if (c <= 0 || c >= Jc) + continue; + d->Mc[static_cast(i)] = + standW[static_cast(i)].Inverse() * clipQ(d->neutral, c); + d->McInv[static_cast(i)] = + d->Mc[static_cast(i)].Inverse(); + } + } +#ifdef ENABLE_MOCAP + d->haveNeutralDir = false; + if (mediaPipeWorld33) { + const Ogre::Quaternion CtInv = tb.Ct.Inverse(); + std::vector neutralCanon( + static_cast(Jc), Ogre::Vector3::ZERO); + collectCanonicalLiveDirections( + mediaPipeWorld33, mediaPipeVisibility33, Jc, neutralCanon); + d->neutralDref.assign(static_cast(Jc), Ogre::Vector3::ZERO); + d->dirQbase.assign(static_cast(d->nBones), + Ogre::Quaternion::IDENTITY); + for (int c = 0; c < Jc; ++c) { + const Ogre::Vector3& nc = neutralCanon[static_cast(c)]; + if (nc.squaredLength() < 1e-12f) + continue; + Ogre::Vector3 ref = CtInv * nc; + if (ref.squaredLength() < 1e-12f) + continue; + ref.normalise(); + d->neutralDref[static_cast(c)] = ref; + d->haveNeutralDir = true; + } + if (d->haveNeutralDir) { + for (int i = 0; i < d->nBones; ++i) { + const int c = d->boneToCanon[static_cast(i)]; + if (c < 0 || c >= Jc || c == 0) + continue; + if (d->neutralDref[static_cast(c)].squaredLength() + < 1e-12f) + continue; + if (tb.tgtBindDir[static_cast(c)].squaredLength() + < 1e-12f) + continue; + d->dirQbase[static_cast(i)] = + tb.tgtBindDir[static_cast(c)].getRotationTo( + d->neutralDref[static_cast(c)]) + * tb.bindWorld[static_cast(i)]; + } + } + } +#endif +} + std::vector> BodyRetargeter::evaluateFrame( const std::array, 22>& canonicalQuats, - uint32_t resolvedMask) const + uint32_t resolvedMask, + uint32_t skipRolesMask, + const float* mediaPipeWorld33, + const float* mediaPipeVisibility33) const { std::vector> out; if (!m_valid || !d) return out; const TargetBindFrame& tb = d->tb; const int Jc = d->Jc; + const int nBones = d->nBones; + auto clipQ = [&](const std::array, 22>& src, int joint) -> Ogre::Quaternion { const auto& q = src[static_cast(joint)]; - return Ogre::Quaternion(q[3], q[0], q[1], q[2]); // (w,x,y,z) + return Ogre::Quaternion(q[3], q[0], q[1], q[2]); }; - // First frame becomes the NEUTRAL reference — the clip's "calm" frame, which - // is what applyMotionClip's legacy transport takes its per-joint LOCAL - // articulation delta and its Mc roll-correction frame against. Composing the - // delta onto the harvested STANDING pose (arms at chest) is what makes the - // live preview match the Record path. - if (!d->haveNeutral) { - d->neutral = canonicalQuats; - d->haveNeutral = true; - // Mc[i] = standWorldOf(bone)⁻¹ · clipQ(neutral, c) — Mixamo-only roll - // correction; on non-identity-rest rigs it is left identity. - if (d->haveAnyStand && d->restsAreIdentity) { - std::vector standW(static_cast(d->nBones), - Ogre::Quaternion::IDENTITY); - for (int i : tb.order) { - const int pi = tb.parentIdx[static_cast(i)]; - const Ogre::Quaternion Wp = (pi >= 0) - ? standW[static_cast(pi)] : Ogre::Quaternion::IDENTITY; - const Ogre::Quaternion localRot = d->haveStand[static_cast(i)] + // PoseIK emits WORLD quats per role. Collar/buttock stay identity (unresolved) + // — skip them and use chest/hip as the parent (same as the debug overlay). + auto parentRelativeLocal = [&](const std::array, 22>& src, + int role, uint32_t mask) -> Ogre::Quaternion { + const int pc = effectiveParentRole(role, mask); + if (pc >= 0 && pc < Jc) + return clipQ(src, pc).Inverse() * clipQ(src, role); + return clipQ(src, role); + }; + const bool haveNeutral = d->haveNeutral; + +#ifdef ENABLE_MOCAP + // Live mocap: aim bind bones at landmark segment directions (same math as + // applyMotionClip direction retarget) — matches the PoseIK debug overlay. + if (mediaPipeWorld33 && haveNeutral && d->haveNeutralDir) { + std::vector liveCanon( + static_cast(Jc), Ogre::Vector3::ZERO); + collectCanonicalLiveDirections( + mediaPipeWorld33, mediaPipeVisibility33, Jc, liveCanon); + const Ogre::Quaternion CtInv = tb.Ct.Inverse(); + std::vector W(static_cast(nBones)); + for (int i : tb.order) { + const int c = d->boneToCanon[static_cast(i)]; + if (c < 0 || c >= Jc) + continue; + if (skipRolesMask & (1u << static_cast(c))) + continue; + const Ogre::Quaternion base = + (d->haveStand[static_cast(i)] ? d->standLocal[static_cast(i)] - : tb.bindLocal[static_cast(i)]; - standW[static_cast(i)] = Wp * localRot; - } - for (int i = 0; i < d->nBones; ++i) { - const int c = d->boneToCanon[static_cast(i)]; - if (c <= 0 || c >= Jc) continue; // root (c==0) keeps identity Mc - d->Mc[static_cast(i)] = - standW[static_cast(i)].Inverse() * clipQ(d->neutral, c); - d->McInv[static_cast(i)] = d->Mc[static_cast(i)].Inverse(); + : tb.bindLocal[static_cast(i)]); + const int pi = tb.parentIdx[static_cast(i)]; + const Ogre::Quaternion Wp = + (pi >= 0) ? W[static_cast(pi)] + : Ogre::Quaternion::IDENTITY; + Ogre::Quaternion local; + if (c == 0) { + local = base; + W[static_cast(i)] = Wp * local; + } else if (liveCanon[static_cast(c)].squaredLength() + > 1e-12f + && d->neutralDref[static_cast(c)].squaredLength() + > 1e-12f + && tb.tgtBindDir[static_cast(c)].squaredLength() + > 1e-12f) { + Ogre::Vector3 ds = CtInv * liveCanon[static_cast(c)]; + ds.normalise(); + const Ogre::Quaternion R = + d->neutralDref[static_cast(c)].getRotationTo(ds); + const Ogre::Quaternion Wt = + R * d->dirQbase[static_cast(i)]; + local = Wp.Inverse() * Wt; + W[static_cast(i)] = Wt; + } else { + local = base; + W[static_cast(i)] = Wp * local; } + out.emplace_back(static_cast(i), local); } + return out; } +#endif + (void)mediaPipeWorld33; + (void)mediaPipeVisibility33; + for (int i : tb.order) { const int c = d->boneToCanon[static_cast(i)]; if (c < 0 || c >= Jc) continue; - const int pc = MotionInbetween::canonicalParentOf(c); - // world -> parent-relative LOCAL articulation, current and neutral - auto localArtic = [&](const std::array, 22>& src) { - return (pc >= 0 && pc < Jc) - ? clipQ(src, pc).Inverse() * clipQ(src, c) - : clipQ(src, c); - }; - // base = harvested standing pose local (arms at chest), else bind local. - const Ogre::Quaternion base = d->haveStand[static_cast(i)] - ? d->standLocal[static_cast(i)] - : tb.bindLocal[static_cast(i)]; - // UNRESOLVED role (e.g. legs of a seated subject) → hold the standing - // pose; never drive a bone from a role PoseIK couldn't observe. + if (skipRolesMask & (1u << static_cast(c))) + continue; + const Ogre::Quaternion base = + (d->haveStand[static_cast(i)] + ? d->standLocal[static_cast(i)] + : tb.bindLocal[static_cast(i)]); const bool resolved = (resolvedMask & (1u << static_cast(c))) != 0u; if (!resolved) { out.emplace_back(static_cast(i), base); continue; } - if (c == 0) { - // Root/hip: MediaPipe/CMU bake the whole-body FACING into the hip. - // Locking the root to the standing pose keeps the figure upright - // (applying the full hip delta folds the torso forward). A subtle - // pelvic sway could be re-added later; upright + stable first. + if (c == 0 || !haveNeutral) { out.emplace_back(static_cast(i), base); continue; } - const Ogre::Quaternion cur = localArtic(canonicalQuats); - const Ogre::Quaternion ref = localArtic(d->neutral); - const Ogre::Quaternion delta = ref.Inverse() * cur; - // roll-corrected articulation, composed onto the standing pose. - const Ogre::Quaternion artic = - d->McInv[static_cast(i)] * delta * d->Mc[static_cast(i)]; - const Ogre::Quaternion local = base * artic; - out.emplace_back(static_cast(i), local); + // PoseIK alignment: parent-relative delta vs neutral, transported onto + // the rig standing pose with the same Mc frame map as applyMotionClip. + const Ogre::Quaternion localCur = + parentRelativeLocal(canonicalQuats, c, resolvedMask); + const Ogre::Quaternion localRef = + parentRelativeLocal(d->neutral, c, d->neutralResolvedMask); + Ogre::Quaternion delta = localRef.Inverse() * localCur; + Ogre::Quaternion artic = delta; + if (d->haveAnyStand && d->restsAreIdentity && d->neutralHadTorso) { + artic = d->McInv[static_cast(i)] * delta + * d->Mc[static_cast(i)]; + } else if (d->yaw180 && !d->haveAnyStand && c > 0) { + static const Ogre::Quaternion kYawPi(0.0f, 0.0f, 1.0f, 0.0f); + artic = kYawPi.Inverse() * delta * kYawPi; + } + const int dup = std::max(1, d->canonDup[static_cast(c)]); + if (dup > 1) + artic = Ogre::Quaternion::Slerp(1.0f / static_cast(dup), + Ogre::Quaternion::IDENTITY, artic, + true); + out.emplace_back(static_cast(i), base * artic); } return out; } +void BodyRetargeter::resetLiveNeutral() +{ + if (!d) + return; + d->haveNeutral = false; + d->neutralHadTorso = false; + d->neutralResolvedMask = 0; +#ifdef ENABLE_MOCAP + d->haveNeutralDir = false; + d->neutralDref.clear(); + d->dirQbase.clear(); +#endif +} + float AnimationMerger::currentArmSpace(Ogre::Skeleton* skel, const std::string& animName) { @@ -2042,7 +2297,8 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( const std::vector>& clipRestDir, bool modelClip, const std::vector& clipRootY, - bool verticalDescent) + bool verticalDescent, + bool cmuLibraryHandedness) { ApplyMotionResult res; if (!skel) { res.error = QStringLiteral("no skeleton"); return res; } @@ -2079,34 +2335,8 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( if (!canonSeen[c]) { canonSeen[c] = 1; ++distinct; } } } - // HANDEDNESS COMPENSATION. The bone NAMES are anatomically correct for the - // mesh, but a mesh may face opposite to the CMU data — then the rig's "Left" - // bones sit on the −X side while CMU's left canonical joints are at +X. If we - // mapped name→canon directly the motion would play MIRRORED. So detect the - // rig's handedness from the actual world-X of a left vs right bone and, when - // it's opposite CMU's (+X = left), SWAP the canonical L/R indices in the - // mapping — labels stay correct, motion stays correct. Uses the upper-arm - // pair (canon 11 = left, 7 = right) with a fallback to the leg pair (19/15). - { - auto worldXForCanon = [&](int canon) -> double { - for (int i = 0; i < nBones; ++i) - if (boneToCanon[i] == canon) - return skel->getBone(static_cast(i))->_getDerivedPosition().x; - return 0.0; - }; - double lx = worldXForCanon(11), rx = worldXForCanon(7); // arms - if (std::abs(lx - rx) < 1e-4) { lx = worldXForCanon(19); rx = worldXForCanon(15); } // legs - // CMU: left at +X. If the rig's "left" bone is more −X than its "right", - // the rig is mirrored vs CMU → swap L/R canonical targets. - if (lx < rx - 1e-5) { - static const int kLR[][2] = {{6,10},{7,11},{8,12},{9,13},{14,18},{15,19},{16,20},{17,21}}; - auto swapCanon = [&](int& c) { - for (auto& p : kLR) { if (c == p[0]) { c = p[1]; return; } if (c == p[1]) { c = p[0]; return; } } - }; - for (int i = 0; i < nBones; ++i) - if (boneToCanon[i] >= 0) swapCanon(boneToCanon[i]); - } - } + if (cmuLibraryHandedness) + compensateCanonicalHandedness(skel, boneToCanon); res.canonicalJoints = distinct; // Bones-per-role: rigs segment chains differently (Mixamo has Spine AND diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index 655009b01..e188e49b5 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -7,24 +7,24 @@ #include #include -// Stateful per-frame body retargeter — the SAME legacy-transport math -// applyMotionClip bakes into a clip, exposed for LIVE drive (mocap preview) -// so the live path and the recorded clip can never diverge. Construct once -// from the target skeleton (captures the bind frame, torso frame Ct, per-role -// bind directions, and the rig's harvested STANDING pose from the calmest -// frame of its first authored animation), then call evaluateFrame() per pose. +// Stateful per-frame body retargeter for live mocap and clip baking. Construct +// once from the target skeleton (bind frame, torso Ct, per-role bind directions, +// harvested STANDING pose). evaluateFrame() per pose. // -// Input: 22 canonical-role WORLD quaternions (x,y,z,w) from PoseIK — the same -// array recordBody feeds applyMotionClip. Each joint's parent-relative LOCAL -// articulation delta (vs the FIRST frame, cached as the neutral reference on -// the first call) is composed onto the standing pose, with the Mixamo roll -// correction Mc: local = standLocal · (Mc⁻¹ · delta · Mc). The root/hip is -// locked to standing (facing is baked into the hip); unresolved roles hold -// standing. Output: per-bone ABSOLUTE LOCAL orientation for -// Bone::setOrientation() (Ogre node keyframes are absolute, not deltas). +// Live mocap (mediaPipeWorld33 passed): landmark-direction matching — the same +// per-frame aim math applyMotionClip uses for direction retarget. MediaPipe +// landmarks are canonicalized and each bone is aimed at its live segment +// direction relative to the neutral calibration frame. This matches the PoseIK +// debug overlay geometry without a mirror-L/R swap. +// +// Clip bake / tests (no landmarks): parent-relative PoseIK quaternion delta vs +// neutral, composed onto standing with Mc roll correction. class BodyRetargeter { public: - explicit BodyRetargeter(Ogre::Skeleton* skel); + // yaw180: pass detectBackwardFacing(entity) for rigs with no harvested + // standing animation — conjugates limb deltas 180° about +Y (same bridge + // applyMotionClip uses for -Z-facing meshes). + explicit BodyRetargeter(Ogre::Skeleton* skel, bool yaw180 = false); bool valid() const { return m_valid; } // resolvedMask bit i set => canonical role i is tracked this frame; roles // not set hold the standing pose. Returns {boneHandle -> local quat}. @@ -33,7 +33,22 @@ class BodyRetargeter { // use only (the mocap live/bake paths); call frames in order. std::vector> evaluateFrame(const std::array, 22>& canonicalQuats, - uint32_t resolvedMask) const; + uint32_t resolvedMask, + uint32_t skipRolesMask = 0, + const float* mediaPipeWorld33 = nullptr, + const float* mediaPipeVisibility33 = nullptr) const; + // Live mocap: capture the reference pose and precompute direction anchors. + // When mediaPipeWorld33 is supplied, neutral landmark directions + Qbase + // are stored for the direction-matching path. Until set, evaluateFrame() + // holds the standing pose. + void setNeutralReference( + const std::array, 22>& canonicalQuats, + uint32_t resolvedMask = 0xFFFFFFFFu, + const float* mediaPipeWorld33 = nullptr, + const float* mediaPipeVisibility33 = nullptr); + bool hasNeutralReference() const; + // Forget the neutral reference so the next setNeutralReference() re-calibrates. + void resetLiveNeutral(); private: struct Impl; std::shared_ptr d; // shared_ptr so the class stays copyable/movable @@ -230,7 +245,11 @@ class AnimationMerger { // crouches instead of running in place. Locomotion clips pass empty / // false to keep the root flat. const std::vector& clipRootY = {}, - bool verticalDescent = false); + bool verticalDescent = false, + // CMU BVH / motion-library clips store LEFT at +X; Ogre rigs with LEFT + // at −X need L/R bone-index swap. Pose-ik mocap already labels L/R + // anatomically — leave this false for qtmesh mocap / live drive. + bool cmuLibraryHandedness = true); /// One skeletal animation extracted onto the 22-joint canonical skeleton /// (#839, the REVERSE of applyMotionClip's world-frame path): per frame, diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index eac6bc43a..35b0df953 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -1140,6 +1140,223 @@ TEST_F(AnimationMergerTest, ArmSpaceFollowsAnimationRename) EXPECT_GT(degBetween(wide, neutral), 15.0f); // it actually moved back } +TEST_F(AnimationMergerTest, BodyRetargeterUsesChestWhenCollarUnresolved) +{ + // PoseIK keeps collar at identity (unresolved). Parent-relative deltas must + // walk up to chest — otherwise a rotating torso bleeds into the shoulder + // delta and the skinned mesh diverges from the PoseIK debug skeleton. + auto skel = Ogre::SkeletonManager::getSingleton().create( + "body_rt_skel", Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + unsigned short h = 0; + auto bone = [&](const char* n, const Ogre::Vector3& p, Ogre::Bone* parent) { + auto* b = skel->createBone(n, h++); + b->setPosition(p); + if (parent) parent->addChild(b); + return b; + }; + auto* hips = bone("Hips", {0, 1.0f, 0}, nullptr); + auto* spine = bone("Spine", {0, 0.2f, 0}, hips); + auto* chest = bone("Spine2", {0, 0.25f, 0}, spine); + bone("Neck", {0, 0.15f, 0}, chest); + bone("Head", {0, 0.15f, 0}, chest); + auto* lArm = bone("LeftArm", {0.25f, 0.05f, 0}, chest); + bone("LeftForeArm", {0.25f, 0, 0}, lArm); + bone("LeftHand", {0.15f, 0, 0}, lArm); + auto* rArm = bone("RightArm", {-0.25f, 0.05f, 0}, chest); + bone("RightForeArm", {-0.25f, 0, 0}, rArm); + bone("RightHand", {-0.15f, 0, 0}, rArm); + bone("LeftUpLeg", {0.12f, -0.05f, 0}, hips); + bone("LeftLeg", {0, -0.35f, 0}, hips); + bone("LeftFoot", {0, -0.35f, 0.05f}, hips); + bone("RightUpLeg", {-0.12f, -0.05f, 0}, hips); + bone("RightLeg", {0, -0.35f, 0}, hips); + bone("RightFoot", {0, -0.35f, 0.05f}, hips); + skel->setBindingPose(); + auto mesh = createInMemoryMesh("body_rt_mesh", skel); + Ogre::Entity* ent = Manager::getSingleton()->getSceneMgr()->createEntity( + "body_rt_ent", mesh); + ASSERT_NE(ent, nullptr); + Ogre::SkeletonInstance* skelInst = ent->getSkeleton(); + BodyRetargeter rt(skelInst); + ASSERT_TRUE(rt.valid()); + + auto toArr = [](const Ogre::Quaternion& q) -> std::array { + return {q.x, q.y, q.z, q.w}; + }; + auto identityFrame = []() { + std::array, 22> q{}; + for (auto& row : q) + row = {0.f, 0.f, 0.f, 1.f}; + return q; + }; + auto armDir = [&]() -> Ogre::Vector3 { + skelInst->_updateTransforms(); + return (skelInst->getBone("LeftForeArm")->_getDerivedPosition() + - skelInst->getBone("LeftArm")->_getDerivedPosition()) + .normalisedCopy(); + }; + auto armDirInChest = [&]() -> Ogre::Vector3 { + skelInst->_updateTransforms(); + Ogre::Bone* chestBone = skelInst->getBone("Spine2"); + const Ogre::Vector3 world = armDir(); + return (chestBone->_getDerivedOrientation().Inverse() * world) + .normalisedCopy(); + }; + auto applyLocals = + [&](const std::vector>& locals) { + skelInst->reset(true); + for (const auto& [handle, local] : locals) { + Ogre::Bone* b = skelInst->getBone(handle); + b->setManuallyControlled(true); + b->setOrientation(local); + } + for (Ogre::Bone* root : skelInst->getRootBones()) + root->_update(true, true); + }; + + skelInst->reset(true); + skelInst->_updateTransforms(); + const Ogre::Vector3 armBindLocal = armDirInChest(); + + // Resolved: hip, abdomen, chest, lshoulder, lelbow — NOT lcollar (10). + const uint32_t mask = (1u << 0) | (1u << 1) | (1u << 2) | (1u << 11) + | (1u << 12); + + std::array, 22> neutral = identityFrame(); + neutral[2] = toArr(Ogre::Quaternion::IDENTITY); // chest + neutral[11] = toArr(Ogre::Quaternion::IDENTITY); // lshoulder + + const Ogre::Quaternion torsoTurn(Ogre::Degree(35), Ogre::Vector3::UNIT_Y); + const Ogre::Quaternion armRaise(Ogre::Degree(55), Ogre::Vector3::UNIT_Z); + std::array, 22> frame = identityFrame(); + frame[2] = toArr(torsoTurn); + frame[11] = toArr(torsoTurn * armRaise); + + rt.setNeutralReference(neutral, mask); + applyLocals(rt.evaluateFrame(frame, mask)); + const float armMotion = degBetween(armBindLocal, armDirInChest()); + + // Torso-only rotation with no arm raise should barely move the upper arm + // relative to the chest (shoulder world quat follows chest). + std::array, 22> torsoOnly = identityFrame(); + torsoOnly[2] = toArr(torsoTurn); + torsoOnly[11] = toArr(torsoTurn); + applyLocals(rt.evaluateFrame(torsoOnly, mask)); + const float torsoOnlyMotion = degBetween(armBindLocal, armDirInChest()); + + EXPECT_GT(armMotion, 30.0f); + EXPECT_LT(torsoOnlyMotion, 12.0f); +} + +#ifdef ENABLE_MOCAP +#include "Mocap/PoseIKSolver.h" + +TEST_F(AnimationMergerTest, BodyRetargeterLandmarkDirectionMovesArm) +{ + // Live mocap: landmark segment directions aim bind bones (same geometry as + // the PoseIK debug overlay), not parent-relative quaternion deltas. + auto skel = Ogre::SkeletonManager::getSingleton().create( + "body_rt_lm_skel", Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + unsigned short h = 0; + auto bone = [&](const char* n, const Ogre::Vector3& p, Ogre::Bone* parent) { + auto* b = skel->createBone(n, h++); + b->setPosition(p); + if (parent) parent->addChild(b); + return b; + }; + auto* hips = bone("Hips", {0, 1.0f, 0}, nullptr); + auto* spine = bone("Spine", {0, 0.2f, 0}, hips); + auto* chest = bone("Spine2", {0, 0.25f, 0}, spine); + bone("Neck", {0, 0.15f, 0}, chest); + bone("Head", {0, 0.15f, 0}, chest); + auto* lArm = bone("LeftArm", {0.25f, 0.05f, 0}, chest); + bone("LeftForeArm", {0.25f, 0, 0}, lArm); + bone("LeftHand", {0.15f, 0, 0}, lArm); + auto* rArm = bone("RightArm", {-0.25f, 0.05f, 0}, chest); + bone("RightForeArm", {-0.25f, 0, 0}, rArm); + bone("RightHand", {-0.15f, 0, 0}, rArm); + bone("LeftUpLeg", {0.12f, -0.05f, 0}, hips); + bone("LeftLeg", {0, -0.35f, 0}, hips); + bone("LeftFoot", {0, -0.35f, 0.05f}, hips); + bone("RightUpLeg", {-0.12f, -0.05f, 0}, hips); + bone("RightLeg", {0, -0.35f, 0}, hips); + bone("RightFoot", {0, -0.35f, 0.05f}, hips); + skel->setBindingPose(); + auto mesh = createInMemoryMesh("body_rt_lm_mesh", skel); + Ogre::Entity* ent = Manager::getSingleton()->getSceneMgr()->createEntity( + "body_rt_lm_ent", mesh); + ASSERT_NE(ent, nullptr); + Ogre::SkeletonInstance* skelInst = ent->getSkeleton(); + BodyRetargeter rt(skelInst); + ASSERT_TRUE(rt.valid()); + + using Landmarks = std::array; + auto setLm = [](Landmarks& l, int lm, float x, float y, float z) { + l[lm * 3 + 0] = x; + l[lm * 3 + 1] = y; + l[lm * 3 + 2] = z; + }; + auto tPoseLm = [&]() { + Landmarks l{}; + setLm(l, 11, 0.18f, -0.45f, 0.f); + setLm(l, 12, -0.18f, -0.45f, 0.f); + setLm(l, 13, 0.45f, -0.45f, 0.f); + setLm(l, 14, -0.45f, -0.45f, 0.f); + setLm(l, 15, 0.70f, -0.45f, 0.f); + setLm(l, 16, -0.70f, -0.45f, 0.f); + setLm(l, 23, 0.10f, 0.f, 0.f); + setLm(l, 24, -0.10f, 0.f, 0.f); + setLm(l, 25, 0.10f, 0.40f, 0.f); + setLm(l, 26, -0.10f, 0.40f, 0.f); + setLm(l, 27, 0.10f, 0.80f, 0.f); + setLm(l, 28, -0.10f, 0.80f, 0.f); + setLm(l, 0, 0.f, -0.65f, -0.10f); + setLm(l, 7, 0.08f, -0.62f, 0.02f); + setLm(l, 8, -0.08f, -0.62f, 0.02f); + return l; + }; + + auto armDir = [&]() -> Ogre::Vector3 { + skelInst->_updateTransforms(); + return (skelInst->getBone("LeftForeArm")->_getDerivedPosition() + - skelInst->getBone("LeftArm")->_getDerivedPosition()) + .normalisedCopy(); + }; + auto applyLocals = + [&](const std::vector>& locals) { + skelInst->reset(true); + for (const auto& [handle, local] : locals) { + Ogre::Bone* b = skelInst->getBone(handle); + b->setManuallyControlled(true); + b->setOrientation(local); + } + for (Ogre::Bone* root : skelInst->getRootBones()) + root->_update(true, true); + }; + + PoseIK::Solver solver; + Landmarks neutralLm = tPoseLm(); + const auto neutralFr = solver.solveFrame(neutralLm.data()); + + rt.setNeutralReference(neutralFr.quats, neutralFr.resolvedMask, + neutralLm.data(), nullptr); + + applyLocals(rt.evaluateFrame(neutralFr.quats, neutralFr.resolvedMask, 0, + neutralLm.data(), nullptr)); + const Ogre::Vector3 tPoseArm = armDir(); + + Landmarks raisedLm = tPoseLm(); + setLm(raisedLm, 15, 0.18f, -0.75f, 0.f); + setLm(raisedLm, 13, 0.18f, -0.55f, 0.f); + const auto raisedFr = solver.solveFrame(raisedLm.data()); + applyLocals(rt.evaluateFrame(raisedFr.quats, raisedFr.resolvedMask, 0, + raisedLm.data(), nullptr)); + const float raisedMotion = degBetween(tPoseArm, armDir()); + + EXPECT_GT(raisedMotion, 25.0f); +} +#endif // ENABLE_MOCAP + // ── #857: twist transport in the bind-referenced direction retarget ───────── namespace { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6253a58fd..67bc2da24 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -192,6 +192,8 @@ Mocap/PoseIKSolver.cpp Mocap/MocapCLI.cpp Mocap/MocapController.cpp Mocap/MocapCameraHints.cpp +Mocap/MocapPoseDebugOverlay.cpp +Mocap/MocapBodyDriveDebug.cpp commands/RecordMocapClipCommand.cpp ApplyAtlas.cpp EmbeddedTextureCache.cpp diff --git a/src/Mocap/FaceCapPose_test.cpp b/src/Mocap/FaceCapPose_test.cpp index 6d6b7e267..1da02aa4d 100644 --- a/src/Mocap/FaceCapPose_test.cpp +++ b/src/Mocap/FaceCapPose_test.cpp @@ -162,4 +162,23 @@ TEST(FaceCapPose, HeadPoseRecoversYaw) EXPECT_LT(quatAngle(res.rotation, q), 1e-3); } +TEST(FaceCapPose, HeadPoseRecoversPitch) +{ + // rotate the canonical model 20 degrees about +X (nod down), project to + // the image frame (y,z flip), expect the same rotation back + const Quat q = axisAngle(1.f, 0.f, 0.f, static_cast(M_PI) / 9.f); + std::vector landmarks(FaceCap::kCanonicalVertexCount * 3); + for (int i = 0; i < FaceCap::kCanonicalVertexCount; ++i) { + float r[3]; + rotate(q, &FaceCap::kCanonicalFaceModel[i * 3], r); + landmarks[i * 3 + 0] = r[0]; + landmarks[i * 3 + 1] = -r[1]; + landmarks[i * 3 + 2] = -r[2]; + } + const auto res = FaceCapPose::solveHeadPose( + landmarks.data(), FaceCap::kCanonicalVertexCount); + ASSERT_TRUE(res.ok); + EXPECT_LT(quatAngle(res.rotation, q), 1e-3); +} + #endif // ENABLE_MOCAP diff --git a/src/Mocap/MocapBodyDriveDebug.cpp b/src/Mocap/MocapBodyDriveDebug.cpp new file mode 100644 index 000000000..91b995173 --- /dev/null +++ b/src/Mocap/MocapBodyDriveDebug.cpp @@ -0,0 +1,295 @@ +#ifdef ENABLE_MOCAP + +#include "MocapBodyDriveDebug.h" + +#include "../AnimationMerger.h" +#include "../MotionInbetween.h" +#include "PoseIKSolver.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +using Vec3 = std::array; + +Vec3 sub(const Vec3& a, const Vec3& b) +{ + return {a[0] - b[0], a[1] - b[1], a[2] - b[2]}; +} +float len(const Vec3& a) +{ + return std::sqrt(a[0] * a[0] + a[1] * a[1] + a[2] * a[2]); +} +Vec3 norm(Vec3 a) +{ + const float n = len(a); + if (n > 1e-6f) + for (float& c : a) c /= n; + return a; +} +float degBetween(const Vec3& a, const Vec3& b) +{ + const float d = std::max(-1.f, std::min(1.f, + a[0] * b[0] + a[1] * b[1] + a[2] * b[2])); + return std::acos(d) * 180.f / 3.14159265f; +} + +Ogre::Quaternion quatFromArray(const std::array& q) +{ + return Ogre::Quaternion(q[3], q[0], q[1], q[2]); +} + +int effectiveParentRole(int role, uint32_t resolvedMask) +{ + int p = MotionInbetween::canonicalParentOf(role); + while (p >= 0 && !(resolvedMask & (1u << static_cast(p)))) + p = MotionInbetween::canonicalParentOf(p); + return p; +} + +Ogre::Quaternion localArtic( + const std::array, PoseIK::kCanonicalRoles>& quats, + int role, uint32_t resolvedMask) +{ + const int ep = effectiveParentRole(role, resolvedMask); + const Ogre::Quaternion oq = quatFromArray(quats[static_cast(role)]); + if (ep < 0) + return oq; + const Ogre::Quaternion op = quatFromArray(quats[static_cast(ep)]); + return op.Inverse() * oq; +} + +void fkPoseIkJoints( + const std::array, PoseIK::kCanonicalRoles>& quats, + uint32_t resolvedMask, + const std::array, PoseIK::kLandmarkCount>& canonLmPts, + std::array& out) +{ + out.fill({0.f, 0.f, 0.f}); + const Vec3 hip = { + (canonLmPts[23][0] + canonLmPts[24][0]) * 0.5f, + (canonLmPts[23][1] + canonLmPts[24][1]) * 0.5f, + (canonLmPts[23][2] + canonLmPts[24][2]) * 0.5f}; + out[static_cast(PoseIK::Hip)] = hip; + + struct BoneSeg { + int role; + int fromLm; + int toLm; + }; + static const BoneSeg segs[] = { + {PoseIK::Abdomen, 23, 11}, {PoseIK::Chest, 11, 12}, + {PoseIK::Neck, 12, 0}, {PoseIK::Head, 0, 8}, + {PoseIK::RShoulder, 12, 14}, {PoseIK::RElbow, 14, 16}, + {PoseIK::RHand, 16, 16}, + {PoseIK::LShoulder, 11, 13}, {PoseIK::LElbow, 13, 15}, + {PoseIK::LHand, 15, 15}, + {PoseIK::RHip, 24, 26}, {PoseIK::RKnee, 26, 28}, + {PoseIK::RFoot, 28, 32}, + {PoseIK::LHip, 23, 25}, {PoseIK::LKnee, 25, 27}, + {PoseIK::LFoot, 27, 31}, + }; + + std::array restOffset{}; + for (const BoneSeg& s : segs) { + Vec3 dir = sub(canonLmPts[static_cast(s.toLm)], + canonLmPts[static_cast(s.fromLm)]); + const float d = len(dir); + if (d < 1e-5f) + dir = {0.f, 0.12f, 0.f}; + else + dir = norm(dir); + restOffset[static_cast(s.role)] = { + dir[0] * std::max(d, 0.05f), + dir[1] * std::max(d, 0.05f), + dir[2] * std::max(d, 0.05f)}; + } + + std::array worldRot{}; + worldRot.fill(Ogre::Quaternion::IDENTITY); + + for (int role = 0; role < PoseIK::kCanonicalRoles; ++role) { + if (!(resolvedMask & (1u << static_cast(role)))) + continue; + const int parent = MotionInbetween::canonicalParentOf(role); + const Ogre::Quaternion local = localArtic(quats, role, resolvedMask); + if (parent >= 0 && (resolvedMask & (1u << static_cast(parent)))) { + worldRot[static_cast(role)] = + worldRot[static_cast(parent)] * local; + const Ogre::Vector3 off( + restOffset[static_cast(role)][0], + restOffset[static_cast(role)][1], + restOffset[static_cast(role)][2]); + const Ogre::Vector3 w = + worldRot[static_cast(parent)] * off; + out[static_cast(role)] = { + out[static_cast(parent)][0] + w.x, + out[static_cast(parent)][1] + w.y, + out[static_cast(parent)][2] + w.z}; + } else if (role == PoseIK::Hip) { + worldRot[0] = quatFromArray(quats[0]); + } else { + out[static_cast(role)] = { + hip[0] + restOffset[static_cast(role)][0], + hip[1] + restOffset[static_cast(role)][1], + hip[2] + restOffset[static_cast(role)][2]}; + } + } +} + +const char* roleName(int role) +{ + static const char* names[] = { + "Hip", "Abdomen", "Chest", "Neck", "Neck1", "Head", + "RCollar", "RShoulder", "RElbow", "RHand", + "LCollar", "LShoulder", "LElbow", "LHand", + "RButtock", "RHip", "RKnee", "RFoot", + "LButtock", "LHip", "LKnee", "LFoot"}; + if (role >= 0 && role < 22) + return names[role]; + return "?"; +} + +Ogre::Vector3 entityLocalPos(Ogre::Entity* entity, const Ogre::Vector3& world) +{ + if (!entity || !entity->getParentSceneNode()) + return world; + return entity->getParentSceneNode()->convertWorldToLocalPosition(world); +} + +} // namespace + +namespace MocapBodyDriveDebug { + +void logFrame( + Ogre::Entity* entity, + Ogre::SkeletonInstance* skel, + const BodyLiveFrame& body, + const BodyRetargeter* retargeter, + double timeSec, + bool bodyNeutralReady, + bool haveNeutralRef, + int warmupFrames, + int warmupTarget, + size_t localsApplied) +{ + if (!qEnvironmentVariableIsSet("QTMESH_MOCAP_DEBUG") || !entity || !skel + || !body.valid) + return; + + static int sLastBucket = -1; + const int bucket = static_cast(timeSec * 2.0); // ~0.5 Hz + if (bucket == sLastBucket) + return; + sLastBucket = bucket; + + std::array, PoseIK::kLandmarkCount> canon{}; + PoseIK::Solver::canonicalizeMediaPipeWorld(body.world.data(), canon); + + float mn = 1e9f, mx = -1e9f; + for (const auto& v : canon) { + mn = std::min(mn, v[1]); + mx = std::max(mx, v[1]); + } + const float skelH = std::max(1e-4f, mx - mn); + const Ogre::AxisAlignedBox box = entity->getBoundingBox(); + const float entityH = box.getMaximum().y - box.getMinimum().y; + const float scale = (entityH > 1e-3f ? entityH : 1.8f) / skelH; + + std::array fk{}; + fkPoseIkJoints(body.quats, body.resolvedMask, canon, fk); + + skel->_updateTransforms(); + + fprintf(stderr, + "[mocap] t=%.2f retargeter=%s neutralReady=%d haveNeutral=%d " + "torsoStable=%d/%d locals=%zu resolvedMask=0x%06x\n", + timeSec, + (retargeter && retargeter->valid()) ? "ok" : "NO", + bodyNeutralReady ? 1 : 0, + haveNeutralRef ? 1 : 0, + warmupFrames, warmupTarget, + localsApplied, + body.resolvedMask & 0xFFFFFFu); + + if (!bodyNeutralReady || !haveNeutralRef) { + fprintf(stderr, + "[mocap] low torso visibility (%d/%d stable frames) — " + "limbs may follow; spine needs hip+chest in frame.\n", + warmupFrames, warmupTarget); + } + + static const int kLimbRoles[] = { + PoseIK::LShoulder, PoseIK::LElbow, PoseIK::RShoulder, PoseIK::RElbow, + PoseIK::LHip, PoseIK::LKnee, PoseIK::RHip, PoseIK::RKnee, + PoseIK::Chest, PoseIK::Head}; + + for (int role : kLimbRoles) { + if (!(body.resolvedMask & (1u << static_cast(role)))) + continue; + + Ogre::Bone* bone = nullptr; + Ogre::Bone* child = nullptr; + for (unsigned short i = 0; i < skel->getNumBones(); ++i) { + Ogre::Bone* b = skel->getBone(i); + if (MotionInbetween::canonicalIndexForBone( + QString::fromStdString(b->getName())) != role) + continue; + bone = b; + for (unsigned short c = 0; c < b->numChildren(); ++c) { + if (auto* cb = dynamic_cast(b->getChild(c))) { + child = cb; + break; + } + } + break; + } + if (!bone) + continue; + + const Ogre::Vector3 fkP( + fk[static_cast(role)][0] * scale, + fk[static_cast(role)][1] * scale, + fk[static_cast(role)][2] * scale); + const Ogre::Vector3 skelP = + entityLocalPos(entity, bone->_getDerivedPosition()); + + float segAngle = -1.f; + if (child) { + const Ogre::Vector3 bW = bone->_getDerivedPosition(); + const Ogre::Vector3 cW = child->_getDerivedPosition(); + const Vec3 skelDir = norm({ + cW.x - bW.x, cW.y - bW.y, cW.z - bW.z}); + const int childRole = MotionInbetween::canonicalChildOf(role); + Vec3 fkDir{0.f, 1.f, 0.f}; + if (childRole >= 0) { + fkDir = norm(sub(fk[static_cast(childRole)], + fk[static_cast(role)])); + } + segAngle = degBetween(skelDir, fkDir); + } + + const Ogre::Vector3 delta = skelP - fkP; + fprintf(stderr, + "[mocap] %2d %-10s bone=%-16s poseIk=(%+.2f,%+.2f,%+.2f) " + "skel=(%+.2f,%+.2f,%+.2f) posErr=%.3f segErr=%.1f° manual=%d\n", + role, roleName(role), bone->getName().c_str(), + fkP.x, fkP.y, fkP.z, + skelP.x, skelP.y, skelP.z, + delta.length(), + segAngle, + bone->isManuallyControlled() ? 1 : 0); + } + (void)retargeter; +} + +} // namespace MocapBodyDriveDebug + +#endif // ENABLE_MOCAP diff --git a/src/Mocap/MocapBodyDriveDebug.h b/src/Mocap/MocapBodyDriveDebug.h new file mode 100644 index 000000000..baba4081c --- /dev/null +++ b/src/Mocap/MocapBodyDriveDebug.h @@ -0,0 +1,36 @@ +#ifndef MOCAPBODYDRIVEDEBUG_H +#define MOCAPBODYDRIVEDEBUG_H + +#ifdef ENABLE_MOCAP + +#include "MocapLiveTypes.h" + +#include + +namespace Ogre { +class Entity; +class SkeletonInstance; +} + +class BodyRetargeter; + +// stderr diagnostics when QTMESH_MOCAP_DEBUG=1 — compare PoseIK FK joints vs +// the driven skeleton so live mocap mismatches are easy to spot in a terminal. +namespace MocapBodyDriveDebug { + +void logFrame( + Ogre::Entity* entity, + Ogre::SkeletonInstance* skel, + const BodyLiveFrame& body, + const BodyRetargeter* retargeter, + double timeSec, + bool bodyNeutralReady, + bool haveNeutralRef, + int warmupFrames, + int warmupTarget, + size_t localsApplied); + +} // namespace MocapBodyDriveDebug + +#endif // ENABLE_MOCAP +#endif // MOCAPBODYDRIVEDEBUG_H diff --git a/src/Mocap/MocapCameraHints.cpp b/src/Mocap/MocapCameraHints.cpp index 8c4f8b3b2..cfe3127cf 100644 --- a/src/Mocap/MocapCameraHints.cpp +++ b/src/Mocap/MocapCameraHints.cpp @@ -6,6 +6,21 @@ namespace MocapCameraHints { +void ensureMultimediaBackendSafe() +{ +#ifdef Q_OS_LINUX + static bool done = false; + if (done) + return; + done = true; + // Empty list disables all FFmpeg hw decode/encode backends (Qt docs). + if (!qEnvironmentVariableIsSet("QT_FFMPEG_DECODING_HW_DEVICE_TYPES")) + qputenv("QT_FFMPEG_DECODING_HW_DEVICE_TYPES", ","); + if (!qEnvironmentVariableIsSet("QT_FFMPEG_ENCODING_HW_DEVICE_TYPES")) + qputenv("QT_FFMPEG_ENCODING_HW_DEVICE_TYPES", ","); +#endif +} + bool runningAsSnap() { #ifdef Q_OS_LINUX diff --git a/src/Mocap/MocapCameraHints.h b/src/Mocap/MocapCameraHints.h index 5dfb3c914..b893b1dbc 100644 --- a/src/Mocap/MocapCameraHints.h +++ b/src/Mocap/MocapCameraHints.h @@ -5,6 +5,11 @@ namespace MocapCameraHints { +// Must run before the first Qt Multimedia call on Linux. The FFmpeg backend +// initializes VA-API during QPlatformMediaIntegration::instance(); on some +// NVIDIA + X11 setups that SIGSEGVs inside QMediaDevices::videoInputs(). +void ensureMultimediaBackendSafe(); + bool runningAsSnap(); QString snapConnectHint(); diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index be5a1daa9..375a6a331 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -7,6 +7,10 @@ #include "PoseIKSolver.h" #include "VideoFrameSource.h" #include "MocapCameraHints.h" +#include "MocapLiveTypes.h" +#include "MocapPoseDebugOverlay.h" +#include "MocapBodyDriveDebug.h" +#include "MocapPoseFix.h" #include "../AnimationMerger.h" #include "../MotionInbetween.h" #include "../Manager.h" @@ -21,11 +25,16 @@ #include #include #include +#include #include +#include + +#include #include #include #include +#include #endif #include @@ -100,6 +109,8 @@ QString MocapController::clipName() const { return QStringLiteral("FaceCap"); } void MocapController::setClipName(const QString&) {} double MocapController::smoothingCutoff() const { return 1.0; } void MocapController::setSmoothingCutoff(double) {} +bool MocapController::showPoseDebug() const { return false; } +void MocapController::setShowPoseDebug(bool) {} void MocapController::refreshDevices() {} bool MocapController::startPreview(const QString&) { return false; } bool MocapController::startPreviewFromVideo(const QString&) { return false; } @@ -117,11 +128,6 @@ void MocapController::calibrateNeutral() {} // One canonical-role world-quat frame from the pose solver, marshalled to the // main thread alongside the face sample (avoids a second queued signal type). -struct BodyLiveFrame { - bool valid = false; - std::array, PoseIK::kCanonicalRoles> quats; - uint32_t resolvedMask = 0; -}; Q_DECLARE_METATYPE(BodyLiveFrame) // queued sampleReady(FaceSample, BodyLiveFrame, QImage) across the worker @@ -135,6 +141,25 @@ struct MocapMetaTypeRegistrar { } }; const MocapMetaTypeRegistrar mocapMetaTypeRegistrar; + +struct BodyDriveBone { + int role = -1; + std::string boneName; + Ogre::Quaternion bindLocal = Ogre::Quaternion::IDENTITY; + bool wasManuallyControlled = false; +}; + +struct BodyManualBoneSnapshot { + std::string boneName; + Ogre::Quaternion bindLocal = Ogre::Quaternion::IDENTITY; + bool wasManuallyControlled = false; +}; + +struct BodyAnimMaskEntry { + std::string animName; + unsigned short boneHandle = 0; + float weight = 1.f; +}; } // namespace class MocapInferenceWorker : public QObject @@ -152,6 +177,7 @@ class MocapInferenceWorker : public QObject std::shared_ptr posePredictor; PoseIK::Solver poseSolver; std::array roleFilters; + bool requestPoseReset = false; public slots: void processPending() @@ -160,6 +186,11 @@ public slots: return; MocapFrame frame; while (mailbox->take(&frame)) { + if (requestPoseReset) { + poseSolver.reset(); + requestPoseReset = false; + } + FaceSample s = predictor.predict(frame.image, frame.timeSec); if (smooth && s.confidence > 0.f) { for (int c = 0; c < 52; ++c) @@ -170,7 +201,7 @@ public slots: BodyLiveFrame body; if (bodyEnabled && posePredictor) { - const PoseSample ps = + PoseSample ps = posePredictor->predict(frame.image, frame.timeSec); if (ps.confidence > 0.f) { PoseIK::FrameResult fr = poseSolver.solveFrame( @@ -182,6 +213,8 @@ public slots: body.valid = true; body.quats = fr.quats; body.resolvedMask = fr.resolvedMask; + body.world = ps.world; + body.visibility = ps.visibility; } } @@ -203,6 +236,7 @@ struct MocapController::Impl { QString status; QString clipName = QStringLiteral("FaceCap"); double smoothingCutoff = 1.0; + QVariantList cachedDevices; // [{id, description}] — populated by refreshDevices() // The live source feeding the worker: a CameraFrameSource (webcam) OR a // FileFrameSource (video-file preview, the macOS-camera-blocked path). @@ -223,6 +257,8 @@ struct MocapController::Impl { bool headWasManuallyControlled = false; QHash savedWeights; // mesh target -> weight QStringList savedEnabledAnimations; + bool savedSkipAnimStateUpdate = false; + bool savedAlwaysUpdateMainSkeleton = false; // channel enables (persist across sessions; body gated on a humanoid rig) bool faceEnabled = true; @@ -230,30 +266,18 @@ struct MocapController::Impl { bool bodyEnabled = false; bool bodyRigOk = false; // selection resolved >= half the canonical roles - // body live-drive state: one canonical role -> the rig bone it drives, plus - // that bone's bind orientations (for the same world-delta math recordBody - // uses) and its pre-preview manual-control flag (restore contract). - struct BodyBone { - int role = -1; - std::string boneName; - Ogre::Quaternion bindLocal = Ogre::Quaternion::IDENTITY; - Ogre::Quaternion bindWorld = Ogre::Quaternion::IDENTITY; - // The PARENT bone's bind-pose world orientation. A world-space rotation - // delta must be transported through the PARENT's frame (not the bone's - // own) to become a valid parent-relative local, matching the offline - // applyMotionClip retarget. Using the bone's own bindWorld is only - // correct when the local bind is identity (~head); it mis-rotates - // bones with a non-trivial local bind (arms/shoulders). - Ogre::Quaternion parentBindWorld = Ogre::Quaternion::IDENTITY; - bool wasManuallyControlled = false; - }; - std::vector bodyBones; - bool bodyCalibrated = false; - std::array bodyNeutral; - // Shared per-frame retargeter (same math as the recorded clip); built once - // at preview start for the driven entity. - std::shared_ptr bodyRetargeter; + // body live-drive: landmark-direction retarget (BodyRetargeter) + restore list. + std::unique_ptr bodyRetargeter; + std::vector bodyBones; + std::vector bodyManualRestore; + std::vector bodyAnimMaskRestore; bool bodyDetected = false; + int bodyTorsoStableFrames = 0; + bool bodyNeutralReady = false; + uint32_t bodyNeutralCapturedMask = 0; + static constexpr int kBodyTorsoStableFrames = 3; + static constexpr uint32_t kTorsoResolvedMask = + (1u << 0) | (1u << 1) | (1u << 2); // hip, abdomen, chest // recording bool recordPending = false; @@ -269,6 +293,36 @@ struct MocapController::Impl { QElapsedTimer clock; QString previewDataUrl; int sampleCount = 0; + QTimer* cameraStartupTimer = nullptr; + MocapPoseDebugOverlay poseDebugOverlay; + bool showPoseDebug = false; + + // Ogre only recomputes GPU bone matrices / software-skinned vertex buffers when + // AnimationStateSet or manual bones are dirty. Mocap samples arrive on the Qt + // event loop (~30 Hz) while the render loop runs faster — without a per-frame + // refresh the mesh stays frozen in bind pose even though Bone::setOrientation + // succeeded (Mixamo: debug overlay moves, skin does not). + struct SkinningFrameListener : public Ogre::FrameListener { + Impl* impl = nullptr; + + bool frameRenderingQueued(const Ogre::FrameEvent&) override + { + if (!impl || impl->state == MocapController::Idle) + return true; + Ogre::Entity* entity = impl->entity(); + if (!entity || !entity->hasSkeleton()) + return true; + Ogre::SkeletonInstance* skel = entity->getSkeleton(); + skel->_notifyManualBonesDirty(); + if (auto* states = entity->getAllAnimationStates()) + states->_notifyDirty(); + entity->_updateAnimation(); + return true; + } + }; + + std::unique_ptr skinningListener; + bool addedSoftwareAnimRequest = false; Ogre::Entity* entity() const { @@ -311,14 +365,7 @@ int MocapController::state() const { return d->state; } QVariantList MocapController::availableDevices() const { - QVariantList out; - for (const auto& dev : CameraFrameSource::availableDevices()) { - QVariantMap m; - m.insert(QStringLiteral("id"), dev.id); - m.insert(QStringLiteral("description"), dev.description); - out.append(m); - } - return out; + return d->cachedDevices; } bool MocapController::faceDetected() const { return d->faceDetected; } @@ -379,7 +426,50 @@ void MocapController::setSmoothingCutoff(double hz) emit smoothingChanged(); } -void MocapController::refreshDevices() { emit devicesChanged(); } +bool MocapController::showPoseDebug() const +{ + return d->showPoseDebug; +} + +void MocapController::setShowPoseDebug(bool on) +{ + if (on == d->showPoseDebug) + return; + d->showPoseDebug = on; + if (on && d->state != Idle) { + if (Ogre::Entity* entity = d->entity()) + d->poseDebugOverlay.attach(Manager::getSingleton()->getSceneMgr(), + entity->getParentSceneNode()); + } else { + d->poseDebugOverlay.detach(); + } + emit previewSettingsChanged(); +} + +void MocapController::resetLiveCaptureCalibration() +{ + d->calibrated = false; + d->bodyTorsoStableFrames = 0; + d->bodyNeutralReady = false; + d->bodyNeutralCapturedMask = 0; + if (d->bodyRetargeter) + d->bodyRetargeter->resetLiveNeutral(); +} + +void MocapController::refreshDevices() +{ + QVariantList out; + for (const auto& dev : CameraFrameSource::availableDevices()) { + QVariantMap m; + m.insert(QStringLiteral("id"), dev.id); + m.insert(QStringLiteral("description"), dev.description); + out.append(m); + } + if (out == d->cachedDevices) + return; + d->cachedDevices = std::move(out); + emit devicesChanged(); +} void MocapController::setStatusMessage(const QString& message) { @@ -477,7 +567,7 @@ void MocapController::refreshMappingForSelection() if (role < 0 || role >= PoseIK::kCanonicalRoles || roleSeen[role]) continue; roleSeen[role] = true; - Impl::BodyBone bb; + BodyDriveBone bb; bb.role = role; bb.boneName = bone->getName(); d->bodyBones.push_back(bb); @@ -496,7 +586,6 @@ void MocapController::refreshMappingForSelection() bool MocapController::beginPreview(const QString& deviceId) { - // Webcam path: build the camera source and run the shared preview. return beginPreviewWithLiveSource( std::make_unique(deviceId), tr("Starting camera…")); @@ -511,9 +600,6 @@ bool MocapController::startPreviewFromVideo(const QString& filePath) emit errorOccurred(d->status); return false; } - // Video-file path: no camera permission needed (the macOS-blocked-camera - // fallback). FileFrameSource plays at real time and drops frames - // latest-wins into the mailbox, same as the webcam preview. return beginPreviewWithLiveSource( std::make_unique(filePath), tr("Playing video…")); @@ -608,45 +694,96 @@ bool MocapController::beginPreviewWithLiveSource( } } } - // Head-bone drive is used only when body is NOT driving (body owns the - // whole skeleton incl. the head when enabled, so they never fight). + // Prevent the render loop's Entity::updateAnimation → setAnimationState → + // Skeleton::reset from re-applying disabled clips (and racing our manual + // bone writes). We drive bones ourselves and call _updateAnimation() after + // each sample to refresh GPU skinning matrices. + d->savedSkipAnimStateUpdate = entity->getSkipAnimationStateUpdate(); + entity->setSkipAnimationStateUpdate(true); + d->savedAlwaysUpdateMainSkeleton = entity->getAlwaysUpdateMainSkeleton(); + entity->setAlwaysUpdateMainSkeleton(true); + // Head-bone drive uses FaceCap (dense landmarks) even when Body is on — + // PoseIK's head role is coarse and fights the face solve. const bool headBoneDrive = - d->headEnabled && !d->headBone.isEmpty() && !bodyDrivable; - if (headBoneDrive && entity->hasSkeleton()) { + d->headEnabled && !d->headBone.isEmpty(); + if (entity->hasSkeleton()) { Ogre::SkeletonInstance* skel = entity->getSkeleton(); - Ogre::Bone* bone = skel->getBone(d->headBone.toStdString()); - d->headWasManuallyControlled = bone->isManuallyControlled(); - d->headBindLocal = bone->getOrientation(); - d->headBindWorld = bone->_getDerivedOrientation(); - bone->setManuallyControlled(true); - } else { - d->headBone.clear(); // signals onSample to skip the head-bone path + skel->reset(true); + skel->_updateTransforms(); + if (headBoneDrive) { + Ogre::Bone* bone = skel->getBone(d->headBone.toStdString()); + d->headWasManuallyControlled = bone->isManuallyControlled(); + d->headBindLocal = bone->getOrientation(); + d->headBindWorld = bone->_getDerivedOrientation(); + bone->setManuallyControlled(true); + } } - // body drive setup: build the SHARED retargeter (same math as the recorded - // clip) from the rig's bind pose, and put the driven bones under manual - // control so we can write their orientations each frame. + // body drive setup: BodyRetargeter + manual bone control for live drive. d->bodyRetargeter.reset(); + d->bodyManualRestore.clear(); + d->bodyBones.clear(); + d->bodyAnimMaskRestore.clear(); if (bodyDrivable && entity->hasSkeleton()) { Ogre::SkeletonInstance* skel = entity->getSkeleton(); - // The retargeter reads the bind frame off the skeleton, so capture it - // BEFORE any bone goes manual (reset() gives the bind pose). - d->bodyRetargeter = - std::make_shared(entity->getMesh()->getSkeleton().get()); - for (auto& bb : d->bodyBones) { - Ogre::Bone* bone = skel->getBone(bb.boneName); - bb.wasManuallyControlled = bone->isManuallyControlled(); - bb.bindLocal = bone->getOrientation(); // for restore-on-stop - bone->setManuallyControlled(true); + const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + d->bodyRetargeter = std::make_unique(skel, yaw180); + if (!d->bodyRetargeter->valid()) { + d->bodyRetargeter.reset(); + d->bodyBones.clear(); + } else { + for (unsigned short i = 0; i < skel->getNumBones(); ++i) { + Ogre::Bone* bone = skel->getBone(i); + BodyManualBoneSnapshot snap; + snap.boneName = bone->getName(); + snap.bindLocal = bone->getOrientation(); + snap.wasManuallyControlled = bone->isManuallyControlled(); + d->bodyManualRestore.push_back(std::move(snap)); + bone->setManuallyControlled(true); + const int role = MotionInbetween::canonicalIndexForBone( + QString::fromStdString(bone->getName())); + if (role >= 0) { + BodyDriveBone bb; + bb.role = role; + bb.boneName = bone->getName(); + bb.bindLocal = bone->getOrientation(); + bb.wasManuallyControlled = snap.wasManuallyControlled; + d->bodyBones.push_back(std::move(bb)); + } + // Animation tracks still write manually-controlled bones unless + // the blend mask is zero (TransformOperator bone-drag pattern). + if (auto* states = entity->getAllAnimationStates()) { + const auto nBones = static_cast(skel->getNumBones()); + for (const auto& [animName, st] : states->getAnimationStates()) { + if (!st) + continue; + if (!st->hasBlendMask()) + st->createBlendMask(nBones, 1.0f); + const float before = st->getBlendMaskEntry(i); + d->bodyAnimMaskRestore.push_back( + {animName, i, before}); + st->setBlendMaskEntry(i, 0.0f); + } + } + } } - if (!d->bodyRetargeter->valid()) - d->bodyRetargeter.reset(); // non-humanoid — skip body drive } else { - d->bodyBones.clear(); // not driving this session + d->bodyBones.clear(); + } + + if (!d->skinningListener) + d->skinningListener = std::make_unique(); + d->skinningListener->impl = d.get(); + Ogre::Root::getSingleton().addFrameListener(d->skinningListener.get()); + if (!d->addedSoftwareAnimRequest) { + entity->addSoftwareAnimationRequest(true); + d->addedSoftwareAnimRequest = true; } d->calibrated = false; - d->bodyCalibrated = false; + d->bodyTorsoStableFrames = 0; + d->bodyNeutralReady = false; + d->bodyNeutralCapturedMask = 0; d->bodyDetected = false; d->faceDetected = false; d->liveFps = 0; @@ -703,6 +840,7 @@ bool MocapController::beginPreviewWithLiveSource( skel->getBone(bb.boneName)->setManuallyControlled( bb.wasManuallyControlled); d->bodyBones.clear(); + d->bodyRetargeter.reset(); } } d->worker->moveToThread(&d->workerThread); @@ -728,9 +866,30 @@ bool MocapController::beginPreviewWithLiveSource( setStatusMessage(startingMessage); d->camera->start(); + if (d->cameraStartupTimer) { + d->cameraStartupTimer->stop(); + d->cameraStartupTimer->deleteLater(); + } + d->cameraStartupTimer = new QTimer(this); + d->cameraStartupTimer->setSingleShot(true); + connect(d->cameraStartupTimer, &QTimer::timeout, this, [this]() { + if (d->state != CameraStarting) + return; + setStatusMessage( + tr("Camera opened but no frames arrived. Close other apps using " + "the webcam%1, then click Preview again.") + .arg(MocapCameraHints::snapConnectHint())); + emit errorOccurred(d->status); + stopPreview(); + }); + d->cameraStartupTimer->start(10000); + SentryReporter::addBreadcrumb("ai.assist.mocap_live", "preview start"); GamificationManager::noteFeature(QStringLiteral("mocap"), GamificationManager::Surface::Gui); + if (d->showPoseDebug) + d->poseDebugOverlay.attach(Manager::getSingleton()->getSceneMgr(), + entity->getParentSceneNode()); return true; } @@ -798,6 +957,11 @@ void MocapController::onSample(const FaceSample& sample, if (d->state == Idle) return; if (d->state == CameraStarting) { + if (d->cameraStartupTimer) { + d->cameraStartupTimer->stop(); + d->cameraStartupTimer->deleteLater(); + d->cameraStartupTimer = nullptr; + } d->state = Previewing; emit stateChanged(); setStatusMessage(tr("Live — driving the selection.")); @@ -825,58 +989,117 @@ void MocapController::onSample(const FaceSample& sample, } Ogre::Entity* entity = d->entity(); - if (!entity || sample.confidence <= 0.f) + if (!entity) return; - // live drive — morphs - auto* morphMgr = MorphAnimationManager::instance(); - for (const auto& ch : d->mapping.channels) - morphMgr->setWeight(entity, ch.meshTargetName, - sample.weights[ch.canonicalIndex]); - - // live drive — head (skipped when body owns the skeleton: the body - // retargeter already drives the Head bone from the pose landmarks, and - // letting both write the same bone makes them fight frame-to-frame). - const bool bodyOwnsSkeleton = - body.valid && d->bodyRetargeter && d->bodyRetargeter->valid(); - if (!d->headBone.isEmpty() && entity->hasSkeleton() && !bodyOwnsSkeleton) { - if (!d->calibrated) { - d->neutral = Ogre::Quaternion( + bool skeletonDriven = false; + + // live drive — morphs + head (face graph) + if (sample.confidence > 0.f) { + auto* morphMgr = MorphAnimationManager::instance(); + for (const auto& ch : d->mapping.channels) + morphMgr->setWeight(entity, ch.meshTargetName, + sample.weights[ch.canonicalIndex]); + + if (d->headEnabled && !d->headBone.isEmpty() && entity->hasSkeleton()) { + if (!d->calibrated) { + d->neutral = Ogre::Quaternion( + sample.headRotation[3], sample.headRotation[0], + sample.headRotation[1], sample.headRotation[2]); + d->calibrated = true; + } + const Ogre::Quaternion current( sample.headRotation[3], sample.headRotation[0], sample.headRotation[1], sample.headRotation[2]); - d->calibrated = true; + Ogre::Quaternion delta = current * d->neutral.Inverse(); + delta = MocapPoseFix::invertCameraPitchDelta(delta); + const Ogre::Quaternion local = + d->headBindWorld.Inverse() * delta * d->headBindWorld; + Ogre::SkeletonInstance* skel = entity->getSkeleton(); + Ogre::Bone* bone = skel->getBone(d->headBone.toStdString()); + bone->setOrientation(d->headBindLocal * local); + bone->needUpdate(true); + skel->_notifyManualBonesDirty(); + if (auto* states = entity->getAllAnimationStates()) + states->_notifyDirty(); + skeletonDriven = true; } - const Ogre::Quaternion current( - sample.headRotation[3], sample.headRotation[0], - sample.headRotation[1], sample.headRotation[2]); - const Ogre::Quaternion delta = current * d->neutral.Inverse(); - const Ogre::Quaternion local = - d->headBindWorld.Inverse() * delta * d->headBindWorld; - Ogre::SkeletonInstance* skel = entity->getSkeleton(); - Ogre::Bone* bone = skel->getBone(d->headBone.toStdString()); - bone->setOrientation(d->headBindLocal * local); - skel->_notifyManualBonesDirty(); } - // live drive — body. Uses the SHARED BodyRetargeter, i.e. the EXACT same - // legacy-transport math applyMotionClip bakes into the recorded clip, so - // live Preview and Record can't diverge: each joint's parent-relative - // articulation delta (vs the first frame) composed onto the rig's harvested - // standing pose. setOrientation takes the absolute local the retargeter - // returns (Ogre node keys are absolute, not deltas). + // live drive — body (independent of face confidence). PoseIK quats are + // aligned onto Mixamo via BodyRetargeter — same math as applyMotionClip. if (body.valid && d->bodyRetargeter && d->bodyRetargeter->valid() && entity->hasSkeleton()) { - const auto locals = - d->bodyRetargeter->evaluateFrame(body.quats, body.resolvedMask); + std::array, 22> canonQuats{}; + for (int r = 0; r < PoseIK::kCanonicalRoles; ++r) + canonQuats[static_cast(r)] = body.quats[r]; + + const bool torsoOk = + (body.resolvedMask & Impl::kTorsoResolvedMask) == Impl::kTorsoResolvedMask; + if (torsoOk) + ++d->bodyTorsoStableFrames; + else + d->bodyTorsoStableFrames = 0; + + if (torsoOk + && d->bodyTorsoStableFrames >= Impl::kBodyTorsoStableFrames) { + if (!d->bodyRetargeter->hasNeutralReference()) { + d->bodyRetargeter->setNeutralReference( + canonQuats, body.resolvedMask, body.world.data(), + body.visibility.data()); + d->bodyNeutralCapturedMask = body.resolvedMask; + } else if ((d->bodyNeutralCapturedMask & Impl::kTorsoResolvedMask) + != Impl::kTorsoResolvedMask) { + // First neutral was captured before hip/chest were visible — + // parent-relative math was wrong (world quats as locals). + d->bodyRetargeter->resetLiveNeutral(); + d->bodyRetargeter->setNeutralReference( + canonQuats, body.resolvedMask, body.world.data(), + body.visibility.data()); + d->bodyNeutralCapturedMask = body.resolvedMask; + } + } + d->bodyNeutralReady = d->bodyRetargeter->hasNeutralReference(); + + const uint32_t skipHead = + (d->headEnabled && !d->headBone.isEmpty()) ? (1u << 5) : 0u; + const auto locals = d->bodyRetargeter->evaluateFrame( + canonQuats, body.resolvedMask, skipHead, body.world.data(), + body.visibility.data()); Ogre::SkeletonInstance* skel = entity->getSkeleton(); - for (const auto& [handle, local] : locals) - skel->getBone(handle)->setOrientation(local); + for (const auto& [handle, local] : locals) { + Ogre::Bone* bone = skel->getBone(handle); + bone->setManuallyControlled(true); + bone->setOrientation(local); + bone->needUpdate(true); + } + for (Ogre::Bone* root : skel->getRootBones()) + root->_update(true, true); skel->_notifyManualBonesDirty(); + skel->_updateTransforms(); + if (auto* states = entity->getAllAnimationStates()) + states->_notifyDirty(); + entity->_updateAnimation(); + if (Ogre::SceneNode* node = entity->getParentSceneNode()) + node->needUpdate(); + skeletonDriven = true; + + MocapBodyDriveDebug::logFrame( + entity, skel, body, d->bodyRetargeter.get(), sample.timeSec, + d->bodyNeutralReady, d->bodyRetargeter->hasNeutralReference(), + d->bodyTorsoStableFrames, Impl::kBodyTorsoStableFrames, + locals.size()); + } + + if (d->showPoseDebug && body.valid && entity) { + const Ogre::AxisAlignedBox box = entity->getBoundingBox(); + const float height = box.getMaximum().y - box.getMinimum().y; + d->poseDebugOverlay.update(body, height); } if (d->state == Recording) { d->take.push_back(sample); - if (!d->bodyBones.empty()) + if (d->bodyRetargeter) d->bodyTake.push_back(body); d->lastSampleTime = sample.timeSec; } @@ -884,8 +1107,11 @@ void MocapController::onSample(const FaceSample& sample, void MocapController::calibrateNeutral() { - d->calibrated = false; // the next confident sample becomes neutral - setStatusMessage(tr("Hold a neutral face…")); + resetLiveCaptureCalibration(); + // Next frame with a stable full torso will capture neutral immediately. + d->bodyTorsoStableFrames = Impl::kBodyTorsoStableFrames; + setStatusMessage( + tr("Hold a relaxed neutral pose (face the camera, arms at your sides)…")); } bool MocapController::startRecording() @@ -920,10 +1146,11 @@ void MocapController::stopRecording() double clipLen = 0.0; // face + head clip (only when a face/head channel actually drove) - if (!d->mapping.channels.isEmpty() || !d->headBone.isEmpty()) { + if (!d->mapping.channels.isEmpty() + || (d->headEnabled && !d->headBone.isEmpty())) { MocapRecorder::FaceRecordOptions options; options.clipName = d->clipName; - options.head = !d->headBone.isEmpty(); + options.head = d->headEnabled && !d->headBone.isEmpty(); auto* cmd = new RecordMocapClipCommand(d->entityName, d->take, d->mapping, options); UndoManager::getSingleton()->push(cmd); @@ -946,7 +1173,7 @@ void MocapController::stopRecording() // body clip: convert the buffered live frames to the [frame][22] world-quat // stream recordBody expects (identity for roles unresolved that frame), and // push a SEPARATE undo command so face + body each undo cleanly. - if (!d->bodyBones.empty() && d->bodyTake.size() >= 2) { + if (d->bodyRetargeter && d->bodyTake.size() >= 2) { std::vector>> clipQuats; clipQuats.reserve(d->bodyTake.size()); for (const auto& bf : d->bodyTake) { @@ -992,6 +1219,12 @@ void MocapController::restoreEntityState() auto* morphMgr = MorphAnimationManager::instance(); for (auto it = d->savedWeights.begin(); it != d->savedWeights.end(); ++it) morphMgr->setWeight(entity, it.key(), it.value()); + entity->setSkipAnimationStateUpdate(d->savedSkipAnimStateUpdate); + entity->setAlwaysUpdateMainSkeleton(d->savedAlwaysUpdateMainSkeleton); + if (d->addedSoftwareAnimRequest) { + entity->removeSoftwareAnimationRequest(true); + d->addedSoftwareAnimRequest = false; + } if (entity->hasSkeleton()) { Ogre::SkeletonInstance* skel = entity->getSkeleton(); if (!d->headBone.isEmpty()) { @@ -999,12 +1232,22 @@ void MocapController::restoreEntityState() bone->setOrientation(d->headBindLocal); bone->setManuallyControlled(d->headWasManuallyControlled); } - // restore every body-driven bone to its pre-preview state exactly - for (const auto& bb : d->bodyBones) { - Ogre::Bone* bone = skel->getBone(bb.boneName); - bone->setOrientation(bb.bindLocal); - bone->setManuallyControlled(bb.wasManuallyControlled); + for (const auto& snap : d->bodyManualRestore) { + Ogre::Bone* bone = skel->getBone(snap.boneName); + bone->setOrientation(snap.bindLocal); + bone->setManuallyControlled(snap.wasManuallyControlled); } + if (auto* states = entity->getAllAnimationStates()) { + for (const auto& entry : d->bodyAnimMaskRestore) { + if (!states->hasAnimationState(entry.animName)) + continue; + Ogre::AnimationState* st = + states->getAnimationState(entry.animName); + if (st && st->hasBlendMask()) + st->setBlendMaskEntry(entry.boneHandle, entry.weight); + } + } + d->bodyAnimMaskRestore.clear(); skel->_notifyManualBonesDirty(); } if (auto* states = entity->getAllAnimationStates()) { @@ -1020,6 +1263,11 @@ void MocapController::stopPreview() { if (d->state == Idle) return; + if (d->cameraStartupTimer) { + d->cameraStartupTimer->stop(); + d->cameraStartupTimer->deleteLater(); + d->cameraStartupTimer = nullptr; + } if (d->state == Recording) stopRecording(); // commit the take rather than dropping it @@ -1045,6 +1293,12 @@ void MocapController::stopPreview() d->camera.reset(); d->injectedSource = nullptr; d->bodyRetargeter.reset(); + d->poseDebugOverlay.detach(); + + if (d->skinningListener) { + Ogre::Root::getSingleton().removeFrameListener(d->skinningListener.get()); + d->skinningListener->impl = nullptr; + } restoreEntityState(); diff --git a/src/Mocap/MocapController.h b/src/Mocap/MocapController.h index a93dbdaab..3531ed87f 100644 --- a/src/Mocap/MocapController.h +++ b/src/Mocap/MocapController.h @@ -73,6 +73,8 @@ class MocapController : public QObject NOTIFY clipNameChanged) Q_PROPERTY(double smoothingCutoff READ smoothingCutoff WRITE setSmoothingCutoff NOTIFY smoothingChanged) + Q_PROPERTY(bool showPoseDebug READ showPoseDebug WRITE setShowPoseDebug + NOTIFY previewSettingsChanged) public: enum State { Idle = 0, CameraStarting, Previewing, Recording }; @@ -107,6 +109,8 @@ class MocapController : public QObject void setClipName(const QString& name); double smoothingCutoff() const; void setSmoothingCutoff(double hz); + bool showPoseDebug() const; + void setShowPoseDebug(bool on); Q_INVOKABLE void refreshDevices(); // Start the camera + live drive of the SELECTED entity. Empty deviceId = @@ -151,6 +155,7 @@ class MocapController : public QObject void channelsChanged(); void clipNameChanged(); void smoothingChanged(); + void previewSettingsChanged(); void errorOccurred(const QString& message); private: @@ -167,6 +172,7 @@ class MocapController : public QObject // the source spins up. Returns false (and restores state) on any failure. bool beginPreviewWithLiveSource(std::unique_ptr source, const QString& startingMessage); + void resetLiveCaptureCalibration(); // Inspect the currently-selected entity and (re)build the face mapping, // head bone, and body-rig role set — so matchedChannelCount / headAvailable // / bodyAvailable reflect the selection BEFORE a preview runs (the channel diff --git a/src/Mocap/MocapLiveTypes.h b/src/Mocap/MocapLiveTypes.h new file mode 100644 index 000000000..464298fb5 --- /dev/null +++ b/src/Mocap/MocapLiveTypes.h @@ -0,0 +1,21 @@ +#ifndef MOCAPLIVETYPES_H +#define MOCAPLIVETYPES_H + +#ifdef ENABLE_MOCAP + +#include "PoseIKSolver.h" + +#include +#include + +// One body frame marshalled from the inference worker to the main thread. +struct BodyLiveFrame { + bool valid = false; + std::array, PoseIK::kCanonicalRoles> quats; + uint32_t resolvedMask = 0; + std::array world{}; + std::array visibility{}; +}; + +#endif // ENABLE_MOCAP +#endif // MOCAPLIVETYPES_H diff --git a/src/Mocap/MocapPoseDebugOverlay.cpp b/src/Mocap/MocapPoseDebugOverlay.cpp new file mode 100644 index 000000000..7da6a268c --- /dev/null +++ b/src/Mocap/MocapPoseDebugOverlay.cpp @@ -0,0 +1,304 @@ +#ifdef ENABLE_MOCAP + +#include "MocapPoseDebugOverlay.h" + +#include "MocapLiveTypes.h" +#include "../MotionInbetween.h" + +#include + +#include +#include +#include + +namespace { + +constexpr const char* kMatName = "MocapPoseDebug/Unlit"; + +using Vec3 = std::array; + +Vec3 canonLm(const std::array, PoseIK::kLandmarkCount>& p, + int i) +{ + return p[static_cast(i)]; +} + +Vec3 add(const Vec3& a, const Vec3& b) +{ + return {a[0] + b[0], a[1] + b[1], a[2] + b[2]}; +} +Vec3 sub(const Vec3& a, const Vec3& b) +{ + return {a[0] - b[0], a[1] - b[1], a[2] - b[2]}; +} +Vec3 mul(const Vec3& a, float s) +{ + return {a[0] * s, a[1] * s, a[2] * s}; +} +float dot(const Vec3& a, const Vec3& b) +{ + return a[0] * b[0] + a[1] * b[1] + a[2] * b[2]; +} +float len(const Vec3& a) +{ + return std::sqrt(dot(a, a)); +} +Vec3 mid(const Vec3& a, const Vec3& b) +{ + return mul(add(a, b), 0.5f); +} + +Vec3 mulVec3(const Ogre::Quaternion& q, const Vec3& v) +{ + const Ogre::Vector3 r = q * Ogre::Vector3(v[0], v[1], v[2]); + return {r.x, r.y, r.z}; +} + +Ogre::Vector3 toOgre(const Vec3& v, float scale) +{ + return {v[0] * scale, v[1] * scale, v[2] * scale}; +} + +Ogre::Quaternion quatFromArray(const std::array& q) +{ + return Ogre::Quaternion(q[3], q[0], q[1], q[2]); +} + +int effectiveParentRole(int role, uint32_t resolvedMask) +{ + int p = MotionInbetween::canonicalParentOf(role); + while (p >= 0 && !(resolvedMask & (1u << static_cast(p)))) + p = MotionInbetween::canonicalParentOf(p); + return p; +} + +Ogre::Quaternion localArtic( + const std::array, PoseIK::kCanonicalRoles>& src, + int role, uint32_t resolvedMask) +{ + const int ep = effectiveParentRole(role, resolvedMask); + if (ep >= 0) + return quatFromArray(src[static_cast(ep)]).Inverse() + * quatFromArray(src[static_cast(role)]); + return quatFromArray(src[static_cast(role)]); +} + +const int kLmEdges[][2] = { + {0, 1}, {1, 2}, {2, 3}, {3, 7}, {0, 4}, {4, 5}, {5, 6}, + {6, 8}, {9, 10}, {11, 12}, {11, 13}, {13, 15}, {15, 17}, {15, 19}, + {15, 21}, {17, 19}, {12, 14}, {14, 16}, {16, 18}, {16, 20}, {16, 22}, + {18, 20}, {11, 23}, {12, 24}, {23, 24}, {23, 25}, {24, 26}, {25, 27}, + {26, 28}, {27, 29}, {28, 30}, {29, 31}, {30, 32}, {27, 31}, {28, 32}, +}; + +float skeletonHeight(const std::array, PoseIK::kLandmarkCount>& p) +{ + float mn = 1e9f, mx = -1e9f; + for (const auto& v : p) { + mn = std::min(mn, v[1]); + mx = std::max(mx, v[1]); + } + const float h = mx - mn; + return h > 1e-4f ? h : 1.65f; +} + +void fkPoseIkJoints( + const std::array, PoseIK::kCanonicalRoles>& quats, + uint32_t resolvedMask, + const std::array, PoseIK::kLandmarkCount>& canonLmPts, + std::array& out) +{ + out.fill({0.f, 0.f, 0.f}); + const Vec3 hip = mid(canonLmPts[23], canonLmPts[24]); + out[static_cast(PoseIK::Hip)] = hip; + + struct BoneSeg { + int role; + int fromLm; + int toLm; + }; + static const BoneSeg segs[] = { + {PoseIK::Abdomen, 23, 11}, + {PoseIK::Chest, 11, 12}, + {PoseIK::Neck, 12, 0}, + {PoseIK::Head, 0, 8}, + {PoseIK::RShoulder, 12, 14}, + {PoseIK::RElbow, 14, 16}, + {PoseIK::RHand, 16, 16}, + {PoseIK::LShoulder, 11, 13}, + {PoseIK::LElbow, 13, 15}, + {PoseIK::LHand, 15, 15}, + {PoseIK::RHip, 24, 26}, + {PoseIK::RKnee, 26, 28}, + {PoseIK::RFoot, 28, 32}, + {PoseIK::LHip, 23, 25}, + {PoseIK::LKnee, 25, 27}, + {PoseIK::LFoot, 27, 31}, + }; + + std::array restOffset{}; + for (const BoneSeg& s : segs) { + Vec3 dir = sub(canonLmPts[static_cast(s.toLm)], + canonLmPts[static_cast(s.fromLm)]); + const float d = len(dir); + if (d < 1e-5f) + dir = {0.f, 0.12f, 0.f}; + else + dir = mul(dir, 1.f / d); + restOffset[static_cast(s.role)] = mul(dir, std::max(d, 0.05f)); + } + + std::array worldRot{}; + worldRot.fill(Ogre::Quaternion::IDENTITY); + + for (int role = 0; role < PoseIK::kCanonicalRoles; ++role) { + if (!(resolvedMask & (1u << static_cast(role)))) + continue; + const int parent = MotionInbetween::canonicalParentOf(role); + const Ogre::Quaternion local = localArtic(quats, role, resolvedMask); + if (parent >= 0 && (resolvedMask & (1u << static_cast(parent)))) { + worldRot[static_cast(role)] = + worldRot[static_cast(parent)] * local; + out[static_cast(role)] = + add(out[static_cast(parent)], + mulVec3(worldRot[static_cast(parent)], + restOffset[static_cast(role)])); + } else if (role == PoseIK::Hip) { + worldRot[0] = quatFromArray(quats[0]); + } else { + worldRot[static_cast(role)] = quatFromArray(quats[role]); + out[static_cast(role)] = + add(hip, restOffset[static_cast(role)]); + } + } +} + +void appendLines(Ogre::ManualObject* mo, + const std::vector>& segs, float scale, + const Ogre::ColourValue& colour) +{ + for (const auto& [a, b] : segs) { + mo->position(toOgre(a, scale)); + mo->colour(colour); + mo->position(toOgre(b, scale)); + mo->colour(colour); + } +} + +} // namespace + +void MocapPoseDebugOverlay::ensureMaterial() +{ + if (Ogre::MaterialManager::getSingleton().resourceExists(kMatName)) + return; + auto mat = Ogre::MaterialManager::getSingleton().create( + kMatName, Ogre::ResourceGroupManager::INTERNAL_RESOURCE_GROUP_NAME); + if (!mat->getNumTechniques()) + mat->createTechnique(); + Ogre::Pass* pass = mat->getTechnique(0)->getPass(0); + pass->setLightingEnabled(false); + pass->setVertexColourTracking(Ogre::TVC_DIFFUSE); + pass->setCullingMode(Ogre::CULL_NONE); + pass->setDepthCheckEnabled(true); + pass->setDepthWriteEnabled(false); + pass->setSceneBlending(Ogre::SBT_TRANSPARENT_ALPHA); +} + +void MocapPoseDebugOverlay::rebuildDrawables() +{ + if (!m_sceneMgr || !m_root) + return; + ensureMaterial(); + if (m_landmarks) + m_sceneMgr->destroyManualObject(m_landmarks); + if (m_poseIk) + m_sceneMgr->destroyManualObject(m_poseIk); + m_landmarks = m_sceneMgr->createManualObject("MocapPoseDebug/Landmarks"); + m_poseIk = m_sceneMgr->createManualObject("MocapPoseDebug/PoseIK"); + m_landmarks->setCastShadows(false); + m_poseIk->setCastShadows(false); + m_root->attachObject(m_landmarks); + m_root->attachObject(m_poseIk); +} + +void MocapPoseDebugOverlay::attach(Ogre::SceneManager* sceneMgr, + Ogre::SceneNode* entityNode) +{ + detach(); + if (!sceneMgr || !entityNode) + return; + m_sceneMgr = sceneMgr; + m_anchor = entityNode; + m_root = entityNode->createChildSceneNode("MocapPoseDebugRoot"); + m_root->setPosition(0.8f, 0.f, 0.f); + rebuildDrawables(); +} + +void MocapPoseDebugOverlay::detach() +{ + if (!m_sceneMgr) + return; + if (m_landmarks) { + m_sceneMgr->destroyManualObject(m_landmarks); + m_landmarks = nullptr; + } + if (m_poseIk) { + m_sceneMgr->destroyManualObject(m_poseIk); + m_poseIk = nullptr; + } + if (m_root && m_anchor) { + m_anchor->removeChild(m_root); + m_sceneMgr->destroySceneNode(m_root); + } + m_root = nullptr; + m_anchor = nullptr; + m_sceneMgr = nullptr; +} + +void MocapPoseDebugOverlay::update(const BodyLiveFrame& body, float entityHeightLocal) +{ + if (!m_landmarks || !m_poseIk || !body.valid) + return; + + std::array, PoseIK::kLandmarkCount> canon{}; + PoseIK::Solver::canonicalizeMediaPipeWorld(body.world.data(), canon); + + const float skelH = skeletonHeight(canon); + const float scale = + (entityHeightLocal > 1e-3f ? entityHeightLocal : 1.8f) / skelH; + + std::vector> lmLines; + lmLines.reserve(std::size(kLmEdges)); + for (const auto& e : kLmEdges) + lmLines.emplace_back(canonLm(canon, e[0]), canonLm(canon, e[1])); + + std::array joints{}; + fkPoseIkJoints(body.quats, body.resolvedMask, canon, joints); + + std::vector> ikLines; + for (int role = 0; role < PoseIK::kCanonicalRoles; ++role) { + const int parent = MotionInbetween::canonicalParentOf(role); + if (parent < 0) + continue; + if (!(body.resolvedMask & (1u << static_cast(role))) + || !(body.resolvedMask & (1u << static_cast(parent)))) + continue; + ikLines.emplace_back(joints[static_cast(parent)], + joints[static_cast(role)]); + } + + m_landmarks->clear(); + m_landmarks->begin("MocapPoseDebug/Unlit", Ogre::RenderOperation::OT_LINE_LIST); + appendLines(m_landmarks, lmLines, scale, + Ogre::ColourValue(0.1f, 0.95f, 1.f, 0.95f)); + m_landmarks->end(); + + m_poseIk->clear(); + m_poseIk->begin("MocapPoseDebug/Unlit", Ogre::RenderOperation::OT_LINE_LIST); + appendLines(m_poseIk, ikLines, scale, + Ogre::ColourValue(1.f, 0.85f, 0.1f, 0.95f)); + m_poseIk->end(); + m_lastScale = scale; +} + +#endif // ENABLE_MOCAP diff --git a/src/Mocap/MocapPoseDebugOverlay.h b/src/Mocap/MocapPoseDebugOverlay.h new file mode 100644 index 000000000..1fa34373e --- /dev/null +++ b/src/Mocap/MocapPoseDebugOverlay.h @@ -0,0 +1,36 @@ +#ifndef MOCAPPOSEDEBUGOVERLAY_H +#define MOCAPPOSEDEBUGOVERLAY_H + +#ifdef ENABLE_MOCAP + +#include "MocapLiveTypes.h" + +namespace Ogre { +class ManualObject; +class SceneManager; +class SceneNode; +} + +// Live debug draw: what PoseIK / MediaPipe think the body looks like (canonical +// +Y up, +Z forward frame), scaled beside the driven entity. Cyan = raw +// landmark stick figure; yellow = 22-joint FK from PoseIK quats. +class MocapPoseDebugOverlay { +public: + void attach(Ogre::SceneManager* sceneMgr, Ogre::SceneNode* entityNode); + void detach(); + void update(const BodyLiveFrame& body, float entityHeightLocal); + +private: + void ensureMaterial(); + void rebuildDrawables(); + + Ogre::SceneManager* m_sceneMgr = nullptr; + Ogre::SceneNode* m_anchor = nullptr; + Ogre::SceneNode* m_root = nullptr; + Ogre::ManualObject* m_landmarks = nullptr; + Ogre::ManualObject* m_poseIk = nullptr; + float m_lastScale = 1.f; +}; + +#endif // ENABLE_MOCAP +#endif // MOCAPPOSEDEBUGOVERLAY_H diff --git a/src/Mocap/MocapPoseFix.h b/src/Mocap/MocapPoseFix.h new file mode 100644 index 000000000..07f12cdf4 --- /dev/null +++ b/src/Mocap/MocapPoseFix.h @@ -0,0 +1,98 @@ +#ifndef MOCAPPOSEFIX_H +#define MOCAPPOSEFIX_H + +#ifdef ENABLE_MOCAP + +#include +#include + +#include +#include +#include + +#include "PoseIKSolver.h" + +namespace MocapPoseFix { + +// Front-facing webcams usually deliver a mirrored image; MediaPipe keeps +// anatomical left/right labels but they end up swapped relative to the user. +// FaceCap is unaffected (dense face mesh); body pose needs this correction. +inline void swapMediaPipeLeftRightLandmarks(float* world33x3, + float* visibility33 = nullptr) +{ + static constexpr int kPairs[][2] = { + {7, 8}, // ears + {11, 12}, {13, 14}, {15, 16}, // arms + {23, 24}, {25, 26}, {27, 28}, // legs + {31, 32}, // feet + }; + auto swapLm = [&](int a, int b) { + for (int k = 0; k < 3; ++k) + std::swap(world33x3[a * 3 + k], world33x3[b * 3 + k]); + if (visibility33) + std::swap(visibility33[a], visibility33[b]); + }; + for (const auto& p : kPairs) + swapLm(p[0], p[1]); +} + +inline void swapCanonicalLeftRight(PoseIK::FrameResult& fr) +{ + static constexpr int kPairs[][2] = { + {6, 10}, {7, 11}, {8, 12}, {9, 13}, + {14, 18}, {15, 19}, {16, 20}, {17, 21}, + }; + for (const auto& p : kPairs) { + std::swap(fr.quats[static_cast(p[0])], + fr.quats[static_cast(p[1])]); + const uint32_t bitA = 1u << static_cast(p[0]); + const uint32_t bitB = 1u << static_cast(p[1]); + const bool resA = (fr.resolvedMask & bitA) != 0u; + const bool resB = (fr.resolvedMask & bitB) != 0u; + fr.resolvedMask &= ~(bitA | bitB); + if (resA) + fr.resolvedMask |= bitB; + if (resB) + fr.resolvedMask |= bitA; + } +} + +inline Ogre::Quaternion kPoseToSkeletonYawPi() +{ + return Ogre::Quaternion(Ogre::Degree(180), Ogre::Vector3::UNIT_Y); +} + +// Mixamo Head (and similar rigs) often need camera-frame pitch inverted while +// yaw stays correct. For unit quaternions, negating the X component mirrors +// rotation about +X (pitch) while leaving yaw (Y) unchanged to first order. +inline Ogre::Quaternion invertCameraPitchDelta(const Ogre::Quaternion& delta) +{ + return Ogre::Quaternion(delta.w, -delta.x, delta.y, delta.z); +} + +// Selfie/webcam mirror: head yaw is opposite the rig unless corrected. +inline Ogre::Quaternion invertCameraYawDelta(const Ogre::Quaternion& delta) +{ + return Ogre::Quaternion(delta.w, delta.x, -delta.y, delta.z); +} + +// Legacy 180°-yaw bridge for CMU library clips whose bind frame differs. +// Live PoseIK mocap does NOT use these — PoseIK and Mixamo both face +Z with +// anatomical L/R bone names; a yaw flip made the body drive backward. +inline Ogre::Vector3 poseDirectionToSkeleton(const Ogre::Vector3& v) +{ + return Ogre::Vector3(-v.x, v.y, -v.z); +} + +inline Ogre::Quaternion poseRotationToSkeleton(const Ogre::Quaternion& q) +{ + const Ogre::Quaternion ry = kPoseToSkeletonYawPi(); + Ogre::Quaternion out = ry * q * ry.Inverse(); + out.normalise(); + return out; +} + +} // namespace MocapPoseFix + +#endif // ENABLE_MOCAP +#endif // MOCAPPOSEFIX_H diff --git a/src/Mocap/MocapRecorder.cpp b/src/Mocap/MocapRecorder.cpp index cc7387f4f..c8ca585b7 100644 --- a/src/Mocap/MocapRecorder.cpp +++ b/src/Mocap/MocapRecorder.cpp @@ -1,6 +1,7 @@ #ifdef ENABLE_MOCAP #include "MocapRecorder.h" +#include "MocapPoseFix.h" #include "FaceCapCanonicalData.h" #include "../AnimationMerger.h" @@ -176,7 +177,8 @@ FaceRecordReport recordFace(Ogre::Entity* entity, toOgre(samples[confident.front()].headRotation); auto deltaAt = [&](int i) { // rotation that takes the neutral pose to this frame's pose - return toOgre(samples[i].headRotation) * neutral.Inverse(); + return MocapPoseFix::invertCameraPitchDelta( + toOgre(samples[i].headRotation) * neutral.Inverse()); }; const std::vector keys = selectKeyIndices( confident, times, options.gapHoldSeconds, options.headEpsilonRad, @@ -339,7 +341,8 @@ BodyRecordReport recordBody( // rendered clip validates the live path headlessly (the live drive can't // be render-captured directly). Default path stays applyMotionClip. if (qEnvironmentVariableIntValue("QTMESH_MOCAP_USE_RETARGETER")) { - BodyRetargeter rt(skel.get()); + const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + BodyRetargeter rt(skel.get(), yaw180); if (!rt.valid()) { report.error = QStringLiteral("retargeter: not a humanoid rig"); return report; @@ -357,6 +360,8 @@ BodyRecordReport recordBody( for (size_t f = 0; f < clipQuats.size(); ++f) { std::array, 22> q{}; for (int r = 0; r < 22 && r < PoseIK::kCanonicalRoles; ++r) q[r] = clipQuats[f][r]; + if (f == 0) + rt.setNeutralReference(q); const auto locals = rt.evaluateFrame(q, 0xFFFFFFFFu); for (const auto& [handle, local] : locals) { auto it = tracks.find(handle); @@ -391,7 +396,9 @@ BodyRecordReport recordBody( const auto res = AnimationMerger::applyMotionClip( skel.get(), clip, clipQuats, fps, /*worldFrame=*/true, /*cmuRestWorld=*/{}, - /*refineWithModel=*/false, /*refineStride=*/8, yaw180); + /*refineWithModel=*/false, /*refineStride=*/8, yaw180, + /*clipRestDir=*/{}, /*modelClip=*/false, /*clipRootY=*/{}, + /*verticalDescent=*/false, /*cmuLibraryHandedness=*/false); if (!res.ok) { report.error = res.error.isEmpty() ? QStringLiteral("retarget failed") diff --git a/src/Mocap/PoseIKSolver.cpp b/src/Mocap/PoseIKSolver.cpp index 68bc7a9f4..2a2dd5cde 100644 --- a/src/Mocap/PoseIKSolver.cpp +++ b/src/Mocap/PoseIKSolver.cpp @@ -94,6 +94,127 @@ bool basisFromPrimary(Vec3 y, Vec3 secondary, Quat& out, Vec3* usedSecondary) } // namespace +void Solver::canonicalizeMediaPipeWorld( + const float* world33x3, + std::array, kLandmarkCount>& out) +{ + for (int i = 0; i < kLandmarkCount; ++i) + out[static_cast(i)] = { + world33x3[i * 3 + 0], -world33x3[i * 3 + 1], world33x3[i * 3 + 2]}; +} + +bool Solver::limbSegmentDirection( + int role, const std::array, kLandmarkCount>& p, + const float* visibility, float minVisibility, std::array& outDir) +{ + struct Segment { + Role r; + int from, to; + }; + static const Segment segments[] = { + {RShoulder, 12, 14}, {RElbow, 14, 16}, + {LShoulder, 11, 13}, {LElbow, 13, 15}, + {RHip, 24, 26}, {RKnee, 26, 28}, {RFoot, 28, 32}, + {LHip, 23, 25}, {LKnee, 25, 27}, {LFoot, 27, 31}, + }; + auto visible = [&](int lm) { + return !visibility || visibility[lm] >= minVisibility; + }; + for (const Segment& seg : segments) { + if (static_cast(seg.r) != role) + continue; + if (!visible(seg.from) || !visible(seg.to)) + return false; + const Vec3& a = p[static_cast(seg.from)]; + const Vec3& b = p[static_cast(seg.to)]; + Vec3 dir = sub(b, a); + if (!normalize(dir)) + return false; + outDir = dir; + return true; + } + return false; +} + +bool Solver::canonicalLiveDirection( + int role, const std::array, kLandmarkCount>& p, + const float* visibility, float minVisibility, + std::array& outDir) +{ + if (limbSegmentDirection(role, p, visibility, minVisibility, outDir)) + return true; + + auto visible = [&](int lm) { + return !visibility || visibility[lm] >= minVisibility; + }; + auto mid = [&](int a, int b) -> Vec3 { + return mul(add(p[static_cast(a)], p[static_cast(b)]), 0.5f); + }; + + switch (role) { + case Hip: + case Abdomen: + case Chest: + if (!visible(LShoulderLm) || !visible(RShoulderLm) || !visible(LHipLm) + || !visible(RHipLm)) + return false; + { + const Vec3 hipMid = mid(LHipLm, RHipLm); + const Vec3 shoulderMid = mid(LShoulderLm, RShoulderLm); + Vec3 dir = sub(shoulderMid, hipMid); + if (!normalize(dir)) + return false; + outDir = dir; + return true; + } + case Neck: + case Neck1: + if (!visible(LShoulderLm) || !visible(RShoulderLm) || !visible(Nose)) + return false; + { + const Vec3 shoulderMid = mid(LShoulderLm, RShoulderLm); + Vec3 dir = sub(p[Nose], shoulderMid); + if (!normalize(dir)) + return false; + outDir = dir; + return true; + } + case Head: + if (!visible(Nose) || !visible(LEar) || !visible(REar)) + return false; + { + const Vec3 earMid = mid(LEar, REar); + Vec3 dir = sub(p[Nose], earMid); + if (!normalize(dir)) + return false; + outDir = dir; + return true; + } + case RHand: + if (!visible(RElbowLm) || !visible(RWrist)) + return false; + { + Vec3 dir = sub(p[RWrist], p[RElbowLm]); + if (!normalize(dir)) + return false; + outDir = dir; + return true; + } + case LHand: + if (!visible(LElbowLm) || !visible(LWrist)) + return false; + { + Vec3 dir = sub(p[LWrist], p[LElbowLm]); + if (!normalize(dir)) + return false; + outDir = dir; + return true; + } + default: + return false; + } +} + void Solver::reset() { m_hasPrev = false; @@ -107,15 +228,15 @@ FrameResult Solver::solveFrame(const float* world, const float* visibility, if (m_hasPrev) result.quats = m_prevQuats; // unresolved roles hold their last pose - // Canonicalize MediaPipe's frame (x = subject-left, y = down, z = toward - // camera) into the canonical rig frame: y-UP, subject FACING +Z, right- - // handed. The subject faces the camera (MediaPipe -z), but canonical rigs - // face +Z, so we must flip the facing: map (x,y,z) -> (-x, -y, +z). This - // is a 180° yaw that cancels the constant (0,-1,0,0) hip offset the old - // (x,-y,-z) mapping produced (which left the whole body driven backwards). + // Canonicalize MediaPipe's frame (+x subject-left, +y down, +z toward camera) + // into the CMU/canonical rig frame (+Y up, +Z forward, LEFT at +X). Flip Y + // and keep Z (subject faces +Z); do NOT negate X — (-x,-y,+z) mirrored L/R + // on Mixamo-style rigs whose left bones already sit at +X. Vec3 p[kLandmarkCount]; + std::array, kLandmarkCount> canon{}; + canonicalizeMediaPipeWorld(world, canon); for (int i = 0; i < kLandmarkCount; ++i) - p[i] = {-world[i * 3 + 0], -world[i * 3 + 1], world[i * 3 + 2]}; + p[i] = canon[static_cast(i)]; auto visible = [&](int lm) { return !visibility || visibility[lm] >= minVisibility; diff --git a/src/Mocap/PoseIKSolver.h b/src/Mocap/PoseIKSolver.h index 833376014..1f7e94935 100644 --- a/src/Mocap/PoseIKSolver.h +++ b/src/Mocap/PoseIKSolver.h @@ -6,8 +6,9 @@ // // Input: MediaPipe pose WORLD landmarks per frame (33 x xyz, metres, // hip-centred, MediaPipe's frame: +x subject's-left, +y DOWN, +z toward the -// camera). Output: WORLD orientation quaternions (x,y,z,w) for the 22 -// canonical CMU roles MotionInbetween/AnimationMerger retarget with +// camera). Canonicalized to CMU (+Y up, +Z forward, LEFT at +X) via (x,-y,+z). +// Output: WORLD orientation quaternions (x,y,z,w) for the 22 canonical CMU +// roles MotionInbetween/AnimationMerger retarget with // (`applyMotionClip(..., worldFrame=true)` takes the delta vs frame 0 and // transports it onto the rig, so only CONSISTENCY over time matters, not the // absolute basis). @@ -66,6 +67,27 @@ class Solver { void reset(); + // Canonicalize MediaPipe world landmarks (+x subject-left, +y down, +z + // toward camera) into the CMU frame (+Y up, +Z forward, LEFT at +X). + static void canonicalizeMediaPipeWorld( + const float* world33x3, + std::array, kLandmarkCount>& out); + + // Unit segment direction for a limb role (shoulder→elbow, etc.), or false + // when landmarks are missing / below minVisibility. + static bool limbSegmentDirection( + int role, const std::array, kLandmarkCount>& p, + const float* visibility, float minVisibility, + std::array& outDir); + + // Live canonical bone-axis direction for any role (limbs + torso/head). + // Used by BodyRetargeter live drive to aim bind bones at landmark geometry + // — the same directions PoseIK derives internally, without the quat/Mc path. + static bool canonicalLiveDirection( + int role, const std::array, kLandmarkCount>& p, + const float* visibility, float minVisibility, + std::array& outDir); + private: bool m_hasPrev = false; // previous secondary (twist-reference) + primary axes per role, world frame diff --git a/src/Mocap/PoseIKSolver_test.cpp b/src/Mocap/PoseIKSolver_test.cpp index 8620fda01..2ee3e3e46 100644 --- a/src/Mocap/PoseIKSolver_test.cpp +++ b/src/Mocap/PoseIKSolver_test.cpp @@ -7,6 +7,7 @@ #include #include "Mocap/PoseIKSolver.h" +#include "Mocap/MocapPoseFix.h" namespace { @@ -96,6 +97,28 @@ TEST(PoseIKSolver, ElbowBendRecoversNinetyDegrees) f2.quats[PoseIK::LShoulder]), 1e-3); } +TEST(PoseIKSolver, LeftArmRaiseDoesNotMoveRightArm) +{ + PoseIK::Solver solver; + Landmarks pose = tPose(); + const auto f1 = solver.solveFrame(pose.data()); + // raise the subject's LEFT arm (MP +x side): lift left wrist in MP space + set(pose, 15, 0.18f, -0.75f, 0.f); + set(pose, 13, 0.18f, -0.55f, 0.f); + const auto f2 = solver.solveFrame(pose.data()); + + const double leftDelta = quatAngle(f1.quats[PoseIK::LShoulder], + f2.quats[PoseIK::LShoulder]) + + quatAngle(f1.quats[PoseIK::LElbow], + f2.quats[PoseIK::LElbow]); + const double rightDelta = quatAngle(f1.quats[PoseIK::RShoulder], + f2.quats[PoseIK::RShoulder]) + + quatAngle(f1.quats[PoseIK::RElbow], + f2.quats[PoseIK::RElbow]); + EXPECT_GT(leftDelta, 0.15); + EXPECT_LT(rightDelta, 0.05); +} + TEST(PoseIKSolver, TorsoTwistShowsOnChestNotHips) { PoseIK::Solver solver; @@ -171,4 +194,71 @@ TEST(PoseIKSolver, LowVisibilityInvalidatesRoles) EXPECT_TRUE(f.resolved(PoseIK::LShoulder)); // shoulder->elbow unaffected } +TEST(PoseIKSolver, HeadNodChangesHeadRotation) +{ + PoseIK::Solver solver; + Landmarks pose = tPose(); + const auto f1 = solver.solveFrame(pose.data()); + ASSERT_TRUE(f1.resolved(PoseIK::Head)); + + // nod down: move nose toward chest (+y in MediaPipe = down) + set(pose, 0, 0.f, -0.55f, -0.08f); + const auto f2 = solver.solveFrame(pose.data()); + ASSERT_TRUE(f2.resolved(PoseIK::Head)); + + const double nod = quatAngle(f1.quats[PoseIK::Head], f2.quats[PoseIK::Head]); + EXPECT_GT(nod, 0.05); + // yaw should stay roughly stable when only nodding + EXPECT_LT(quatAngle(f1.quats[PoseIK::Hip], f2.quats[PoseIK::Hip]), 0.05); +} + +TEST(PoseIKSolver, LimbSegmentDirectionMatchesRaise) +{ + Landmarks pose = tPose(); + std::array, PoseIK::kLandmarkCount> canon{}; + PoseIK::Solver::canonicalizeMediaPipeWorld(pose.data(), canon); + std::array refDir{}, upDir{}; + ASSERT_TRUE(PoseIK::Solver::limbSegmentDirection( + PoseIK::LShoulder, canon, nullptr, 0.3f, refDir)); + // raise left arm: elbow + wrist move up in canonical (+Y) + set(pose, 15, 0.18f, -0.75f, 0.f); + set(pose, 13, 0.18f, -0.55f, 0.f); + PoseIK::Solver::canonicalizeMediaPipeWorld(pose.data(), canon); + ASSERT_TRUE(PoseIK::Solver::limbSegmentDirection( + PoseIK::LShoulder, canon, nullptr, 0.3f, upDir)); + EXPECT_GT(upDir[1], refDir[1] + 0.2f); + std::array rDir{}; + ASSERT_TRUE(PoseIK::Solver::limbSegmentDirection( + PoseIK::RShoulder, canon, nullptr, 0.3f, rDir)); + EXPECT_LT(rDir[1], refDir[1] + 0.05f); // right arm stayed level +} + +TEST(PoseIKSolver, WebcamMirrorSwapFixesRightArmRaise) +{ + PoseIK::Solver solver; + Landmarks pose = tPose(); + const auto f0 = solver.solveFrame(pose.data()); + // Mirrored webcam: user raises their physical RIGHT arm but MediaPipe tracks + // the LEFT landmark chain (positions on the MP +x / subject-left side). + set(pose, 15, 0.18f, -0.75f, 0.f); + set(pose, 13, 0.18f, -0.55f, 0.f); + const auto fWrong = solver.solveFrame(pose.data()); + const double wrongLeft = quatAngle(f0.quats[PoseIK::LShoulder], + fWrong.quats[PoseIK::LShoulder]); + const double wrongRight = quatAngle(f0.quats[PoseIK::RShoulder], + fWrong.quats[PoseIK::RShoulder]); + EXPECT_GT(wrongLeft, 0.08); + EXPECT_GT(wrongLeft, wrongRight); + + Landmarks fixed = tPose(); + set(fixed, 15, 0.18f, -0.75f, 0.f); + set(fixed, 13, 0.18f, -0.55f, 0.f); + MocapPoseFix::swapMediaPipeLeftRightLandmarks(fixed.data()); + const auto fOk = solver.solveFrame(fixed.data()); + const double fixedRight = quatAngle(f0.quats[PoseIK::RShoulder], + fOk.quats[PoseIK::RShoulder]); + EXPECT_GT(fixedRight, 0.08); + EXPECT_GT(fixedRight, wrongRight); +} + #endif // ENABLE_MOCAP diff --git a/src/Mocap/VideoFrameSource.cpp b/src/Mocap/VideoFrameSource.cpp index 8aebb6deb..a4e3eaea0 100644 --- a/src/Mocap/VideoFrameSource.cpp +++ b/src/Mocap/VideoFrameSource.cpp @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include namespace { @@ -30,6 +32,64 @@ QImage mocapFrameToRgb888(const QImage& image) return image.convertToFormat(QImage::Format_RGB888); } +QImage mocapFrameFromVideoFrame(const QVideoFrame& frame) +{ + if (!frame.isValid()) + return {}; + QImage img = frame.toImage(); + if (img.isNull()) { + QVideoFrame mapped(frame); + if (mapped.map(QVideoFrame::ReadOnly)) { + // UVC webcams often deliver Format_Jpeg (MJPEG). Qt's bundled FFmpeg + // path can fail toImage() on confined Linux builds; QImage::fromData + // needs the qjpeg imageformat plugin (deploy.yml bundles it). + if (mapped.pixelFormat() == QVideoFrameFormat::Format_Jpeg + && mapped.planeCount() >= 1 && mapped.mappedBytes(0) > 0) { + img = QImage::fromData(mapped.bits(0), mapped.mappedBytes(0), + "JPEG"); + } + if (img.isNull()) + img = mapped.toImage(); + mapped.unmap(); + } + } + if (img.isNull()) + return {}; + return mocapFrameToRgb888(img); +} + +namespace { + +// USB webcams often default to 1080p; a lighter format starts streaming faster +// and is more reliable under Snap/PipeWire + the FFmpeg backend. +QCameraFormat pickCameraFormat(const QCameraDevice& device) +{ + const auto formats = device.videoFormats(); + if (formats.isEmpty()) + return {}; + auto score = [](const QCameraFormat& f) { + const int w = f.resolution().width(); + const int h = f.resolution().height(); + const int pixels = w * h; + int diff = qAbs(pixels - 640 * 480); + if (w == 1280 && h == 720) + diff = qMin(diff, 200000); + return diff; + }; + QCameraFormat best = formats.first(); + int bestScore = score(best); + for (const QCameraFormat& f : formats) { + const int s = score(f); + if (s < bestScore) { + best = f; + bestScore = s; + } + } + return best; +} + +} // namespace + // --------------------------------------------------------------------------- // ImageSequenceFrameSource // --------------------------------------------------------------------------- @@ -164,9 +224,8 @@ void FileFrameSource::stop() m_player->stop(); } -void FileFrameSource::handleVideoFrame() +void FileFrameSource::handleVideoFrame(const QVideoFrame& vf) { - const QVideoFrame vf = m_sink->videoFrame(); if (!vf.isValid()) return; const qint64 index = m_frameIndex++; @@ -177,7 +236,7 @@ void FileFrameSource::handleVideoFrame() if (!m_decimator.shouldEmit(t)) return; MocapFrame frame; - frame.image = mocapFrameToRgb888(vf.toImage()); + frame.image = mocapFrameFromVideoFrame(vf); frame.timeSec = t; frame.frameIndex = index; if (!frame.image.isNull()) @@ -190,10 +249,15 @@ void FileFrameSource::handleVideoFrame() QList CameraFrameSource::availableDevices() { + MocapCameraHints::ensureMultimediaBackendSafe(); QList out; const auto devices = QMediaDevices::videoInputs(); - for (const QCameraDevice& d : devices) + for (const QCameraDevice& d : devices) { + // Skip metadata-only nodes (e.g. /dev/video1 on dual-node UVC cams). + if (d.videoFormats().isEmpty()) + continue; out.append({QString::fromUtf8(d.id()), d.description()}); + } return out; } @@ -218,6 +282,7 @@ CameraFrameSource::~CameraFrameSource() bool CameraFrameSource::open(QString* error) { + MocapCameraHints::ensureMultimediaBackendSafe(); QCameraDevice device; const auto devices = QMediaDevices::videoInputs(); if (m_deviceId.isEmpty()) { @@ -248,9 +313,15 @@ bool CameraFrameSource::open(QString* error) m_session->setVideoSink(m_sink.get()); m_clock = std::make_unique(); - const auto formats = device.videoFormats(); - if (!formats.isEmpty()) - m_nativeFps = formats.first().maxFrameRate(); + const QCameraFormat format = pickCameraFormat(device); + if (!format.isNull()) { + m_camera->setCameraFormat(format); + m_nativeFps = format.maxFrameRate(); + } else { + const auto formats = device.videoFormats(); + if (!formats.isEmpty()) + m_nativeFps = formats.first().maxFrameRate(); + } connect(m_sink.get(), &QVideoSink::videoFrameChanged, this, &CameraFrameSource::handleVideoFrame); @@ -283,13 +354,12 @@ void CameraFrameSource::stop() m_camera->stop(); } -void CameraFrameSource::handleVideoFrame() +void CameraFrameSource::handleVideoFrame(const QVideoFrame& vf) { - const QVideoFrame vf = m_sink->videoFrame(); if (!vf.isValid()) return; MocapFrame frame; - frame.image = mocapFrameToRgb888(vf.toImage()); + frame.image = mocapFrameFromVideoFrame(vf); if (frame.image.isNull()) return; frame.timeSec = m_clock->isValid() ? m_clock->elapsed() / 1e3 : 0.0; diff --git a/src/Mocap/VideoFrameSource.h b/src/Mocap/VideoFrameSource.h index fd7f8f023..1c9002406 100644 --- a/src/Mocap/VideoFrameSource.h +++ b/src/Mocap/VideoFrameSource.h @@ -23,6 +23,7 @@ class QCamera; class QElapsedTimer; class QMediaCaptureSession; class QMediaPlayer; +class QVideoFrame; class QVideoSink; // One decoded frame. image is guaranteed Format_RGB888. @@ -105,6 +106,8 @@ class FrameMailbox { // Guarantees Format_RGB888 (the input contract of every mocap predictor). QImage mocapFrameToRgb888(const QImage& image); +// Decode a QVideoFrame to RGB888 (maps the buffer when toImage() alone fails). +QImage mocapFrameFromVideoFrame(const QVideoFrame& frame); class VideoFrameSource : public QObject { Q_OBJECT @@ -181,7 +184,7 @@ class FileFrameSource : public VideoFrameSource { double nativeFps() const override { return m_nativeFps; } private: - void handleVideoFrame(); + void handleVideoFrame(const QVideoFrame& frame); QString m_path; FrameDecimator m_decimator; @@ -216,7 +219,7 @@ class CameraFrameSource : public VideoFrameSource { double nativeFps() const override { return m_nativeFps; } private: - void handleVideoFrame(); + void handleVideoFrame(const QVideoFrame& frame); QString m_deviceId; std::unique_ptr m_camera; diff --git a/src/Mocap/VideoFrameSource_test.cpp b/src/Mocap/VideoFrameSource_test.cpp index e8161599c..af6e6511c 100644 --- a/src/Mocap/VideoFrameSource_test.cpp +++ b/src/Mocap/VideoFrameSource_test.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -197,4 +198,15 @@ TEST(MocapFrameToRgb888, ConvertsAndPassesThrough) EXPECT_EQ(same.format(), QImage::Format_RGB888); } +TEST(MocapFrameFromVideoFrame, ConvertsQVideoFrameFromQImage) +{ + QImage rgb(8, 8, QImage::Format_RGB888); + rgb.fill(Qt::blue); + const QVideoFrame vf(rgb); + ASSERT_TRUE(vf.isValid()); + const QImage out = mocapFrameFromVideoFrame(vf); + EXPECT_FALSE(out.isNull()); + EXPECT_EQ(out.format(), QImage::Format_RGB888); +} + #endif // ENABLE_MOCAP diff --git a/src/test_main.cpp b/src/test_main.cpp index 41c91be9c..d315f1bf4 100644 --- a/src/test_main.cpp +++ b/src/test_main.cpp @@ -15,6 +15,9 @@ #include #include "Manager.h" #include "TestHelpers.h" +#ifdef ENABLE_MOCAP +#include "Mocap/MocapCameraHints.h" +#endif #ifndef Q_OS_WIN #include @@ -106,6 +109,10 @@ int main(int argc, char **argv) qputenv(guard, "1"); } +#ifdef ENABLE_MOCAP + MocapCameraHints::ensureMultimediaBackendSafe(); +#endif + QApplication app(argc, argv); // Suppress Ogre log output (debug spam from Root, RenderSystem, plugins). From 7d452d4c85e90dd414108f71d8291a2c057d7608 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 3 Aug 2026 23:47:29 -0400 Subject: [PATCH 02/11] feat(onnx): add OnnxRuntimeSettings GPU preference and wire all predictors Centralize ONNX session options (CPU/CoreML/CUDA/DirectML) behind a QML singleton with AI Settings UI. Optional QTMESH_ONNX_GPU fetch on Linux x64, provider .so copy rules, CI stays on CPU package, and mocap predictors use the shared path so the app links cleanly. Co-authored-by: Cursor --- .github/workflows/deploy.yml | 11 + CMakeLists.txt | 4 + cmake/OnnxRuntime.cmake | 64 ++++-- qml/AISettingsDialog.qml | 38 +++- qml/CollapsibleSection.qml | 22 +- qml/ThemedComboBox.qml | 10 +- scripts/install-onnx-gpu-deps.sh | 38 ++++ src/CLIPipeline.cpp | 8 + src/CMakeLists.txt | 10 +- src/FaceRig/FaceLandmarkDetector.cpp | 9 +- src/ImageTo3D/BackgroundRemover.cpp | 9 +- src/ImageTo3D/MeshGenPredictor.cpp | 9 +- src/ImageTo3D/TripoSGPredictor.cpp | 19 +- src/LLMSettingsWidget.cpp | 57 ++++- src/LLMSettingsWidget.h | 5 + src/LLMSettingsWidget_test.cpp | 2 +- src/MeshSegmenter.cpp | 13 +- src/Mocap/FaceCapPredictor.cpp | 9 +- src/Mocap/PoseCapPredictor.cpp | 8 +- src/MotionGenerator.cpp | 8 +- src/MotionInbetween.cpp | 9 +- src/OnnxRuntimeSettings.cpp | 328 +++++++++++++++++++++++++++ src/OnnxRuntimeSettings.h | 91 ++++++++ src/OnnxRuntimeSettings_test.cpp | 32 +++ src/PbrMapSynth.cpp | 11 +- src/SkinTokensPredictor.cpp | 20 +- src/TextureUpscaler.cpp | 13 +- src/UniRigPredictor.cpp | 106 +++++---- src/UniRigPredictor.h | 7 + src/UniRigPredictor_test.cpp | 10 + src/main.cpp | 18 ++ src/mainwindow.cpp | 8 + 32 files changed, 834 insertions(+), 172 deletions(-) create mode 100755 scripts/install-onnx-gpu-deps.sh create mode 100644 src/OnnxRuntimeSettings.cpp create mode 100644 src/OnnxRuntimeSettings.h create mode 100644 src/OnnxRuntimeSettings_test.cpp diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a14da457e..d425a6dcd 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -783,6 +783,7 @@ jobs: -DCMAKE_CXX_FLAGS="-g" -DCMAKE_C_FLAGS="-g" \ -DENABLE_STABLE_DIFFUSION=ON \ -DENABLE_ONNX=ON \ + -DQTMESH_ONNX_GPU=OFF \ -DENABLE_MOCAP=ON \ -DENABLE_AUTO_UPDATER=OFF \ -DASSIMP_DIR=/usr/local/lib/cmake/assimp-${{ env.ASSIMP_DIR_VERSION }} \ @@ -916,6 +917,14 @@ jobs: ./pack-deb/usr/lib/qtmesheditor/plugins/multimedia/ 2>/dev/null || true fi + # MJPEG webcams deliver Format_Jpeg frames; decoding them via + # QImage::fromData("JPEG") requires the qjpeg imageformat plugin. + if [ -d "$QT_DIR/plugins/imageformats" ]; then + mkdir -p ./pack-deb/usr/lib/qtmesheditor/plugins/imageformats + cp -R "$QT_DIR/plugins/imageformats/"*.so \ + ./pack-deb/usr/lib/qtmesheditor/plugins/imageformats/ 2>/dev/null || true + fi + # Qt FFmpeg stub shims — libffmpegmediaplugin.so depends on these; when # they are absent the FFmpeg backend fails to load and # QMediaDevices::videoInputs() returns empty ("no camera available"). @@ -1124,6 +1133,7 @@ jobs: -DENABLE_SENTRY=OFF \ -DENABLE_PS1_RIP=ON \ -DENABLE_ONNX=ON \ + -DQTMESH_ONNX_GPU=OFF \ -DENABLE_MOCAP=ON - name: Run build-wrapper @@ -1944,6 +1954,7 @@ jobs: -DCMAKE_CXX_FLAGS="-g" -DCMAKE_C_FLAGS="-g" \ -DENABLE_STABLE_DIFFUSION=ON \ -DENABLE_ONNX=ON \ + -DQTMESH_ONNX_GPU=OFF \ -DENABLE_MOCAP=ON \ -DCMAKE_OSX_ARCHITECTURES="$(uname -m)" \ -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 \ diff --git a/CMakeLists.txt b/CMakeLists.txt index d12ec9e98..240934519 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -303,6 +303,10 @@ option(ENABLE_ONNX "Enable AI PBR map synthesis via ONNX Runtime" OFF) if(ENABLE_ONNX) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/OnnxRuntime.cmake) add_definitions(-DENABLE_ONNX) + if(QTMESH_ONNX_GPU) + add_definitions(-DQTMESH_ONNX_GPU_BUILD) + message(STATUS "ONNX Runtime GPU package enabled (CUDA EP on Linux x64 when cuDNN 9 is installed)") + endif() message(STATUS "AI PBR map synthesis enabled with ONNX Runtime") endif() ############################################################## diff --git a/cmake/OnnxRuntime.cmake b/cmake/OnnxRuntime.cmake index 153391500..7e2b3b693 100644 --- a/cmake/OnnxRuntime.cmake +++ b/cmake/OnnxRuntime.cmake @@ -8,9 +8,12 @@ # macOS uses the universal2 archive (covers arm64 + x86_64) so there is no # per-arch selection trap (the lesson from libsodium being built x86_64). CoreML # execution provider ships inside that archive; the CPU EP is always present. +# +# Linux x64 GPU: pass -DQTMESH_ONNX_GPU=ON (auto-defaults ON when nvidia-smi is +# found outside CI) to fetch onnxruntime-linux-x64-gpu-*.tgz. The CUDA provider +# .so must ship next to the binary AND cuDNN 9 + CUDA 12 must be on the system. # Windows MinGW is intentionally NOT wired here — the official Windows archive is -# MSVC-built and won't link under MinGW; that path degrades gracefully (the -# feature reports "rebuild with -DENABLE_ONNX"). See the #404 follow-up. +# MSVC-built and won't link under MinGW. if(TARGET qtmesh_onnx) return() @@ -20,23 +23,49 @@ set(QTMESH_ONNX_VERSION "1.20.1" CACHE STRING "ONNX Runtime release version") set(QTMESH_ONNX_BASE_URL "https://github.com/microsoft/onnxruntime/releases/download/v${QTMESH_ONNX_VERSION}") +# Default GPU package on local Linux x64 builds when an NVIDIA GPU is present. +set(_qtmesh_onnx_gpu_default OFF) +if(UNIX AND NOT APPLE AND NOT DEFINED ENV{CI}) + find_program(_QTMESH_NVIDIA_SMI nvidia-smi) + if(_QTMESH_NVIDIA_SMI) + set(_qtmesh_onnx_gpu_default ON) + endif() +endif() +option(QTMESH_ONNX_GPU + "Download GPU ONNX Runtime (CUDA on Linux x64; adds ~700 MB provider libs)" + ${_qtmesh_onnx_gpu_default}) + # Select the archive + its SHA256 for this platform. if(APPLE) set(_ort_archive "onnxruntime-osx-universal2-${QTMESH_ONNX_VERSION}.tgz") set(_ort_sha256 "da4349e01a7e997f5034563183c7183d069caadc1d95f499b560961787813efd") set(_ort_libname "libonnxruntime.${QTMESH_ONNX_VERSION}.dylib") + set(_ort_fetch_name "qtmesh_onnxruntime") elseif(UNIX) if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") set(_ort_archive "onnxruntime-linux-aarch64-${QTMESH_ONNX_VERSION}.tgz") set(_ort_sha256 "ae4fedbdc8c18d688c01306b4b50c63de3445cdf2dbd720e01a2fa3810b8106a") + set(_ort_fetch_name "qtmesh_onnxruntime") else() - set(_ort_archive "onnxruntime-linux-x64-${QTMESH_ONNX_VERSION}.tgz") - set(_ort_sha256 "67db4dc1561f1e3fd42e619575c82c601ef89849afc7ea85a003abbac1a1a105") + if(QTMESH_ONNX_GPU) + set(_ort_archive "onnxruntime-linux-x64-gpu-${QTMESH_ONNX_VERSION}.tgz") + set(_ort_sha256 "6bfb87c6ebe55367a94509b8ef062239e188dccf8d5caac8d6909b2344893bf0") + set(_ort_fetch_name "qtmesh_onnxruntime_gpu") + else() + set(_ort_archive "onnxruntime-linux-x64-${QTMESH_ONNX_VERSION}.tgz") + set(_ort_sha256 "67db4dc1561f1e3fd42e619575c82c601ef89849afc7ea85a003abbac1a1a105") + set(_ort_fetch_name "qtmesh_onnxruntime") + endif() endif() set(_ort_libname "libonnxruntime.so.${QTMESH_ONNX_VERSION}") elseif(WIN32 AND NOT MINGW) + if(QTMESH_ONNX_GPU) + message(WARNING "QTMESH_ONNX_GPU: Windows GPU package not wired in CMake yet; using CPU ONNX Runtime") + set(QTMESH_ONNX_GPU OFF CACHE BOOL "" FORCE) + endif() set(_ort_archive "onnxruntime-win-x64-${QTMESH_ONNX_VERSION}.zip") set(_ort_sha256 "78d447051e48bd2e1e778bba378bec4ece11191c9e538cf7b2c4a4565e8f5581") + set(_ort_fetch_name "qtmesh_onnxruntime") set(_ort_libname "onnxruntime.dll") else() message(FATAL_ERROR "ENABLE_ONNX: unsupported platform — no ONNX Runtime archive mapping. " @@ -45,19 +74,16 @@ endif() include(FetchContent) FetchContent_Declare( - qtmesh_onnxruntime + ${_ort_fetch_name} URL "${QTMESH_ONNX_BASE_URL}/${_ort_archive}" URL_HASH SHA256=${_ort_sha256} ) -FetchContent_MakeAvailable(qtmesh_onnxruntime) +FetchContent_MakeAvailable(${_ort_fetch_name}) -# The archive extracts to a single top-level dir with include/ and lib/. -set(QTMESH_ONNX_ROOT "${qtmesh_onnxruntime_SOURCE_DIR}") +# FetchContent sets _SOURCE_DIR +set(QTMESH_ONNX_ROOT "${${_ort_fetch_name}_SOURCE_DIR}") set(QTMESH_ONNX_INCLUDE_DIR "${QTMESH_ONNX_ROOT}/include") -# Resolve the actual shared-lib path. Prebuilt layouts vary slightly across -# platforms (versioned symlinks on *nix, lib/*.dll on Windows), so glob for it -# rather than hardcoding a single name. file(GLOB _ort_libs "${QTMESH_ONNX_ROOT}/lib/${_ort_libname}" "${QTMESH_ONNX_ROOT}/lib/libonnxruntime*.dylib" @@ -70,20 +96,18 @@ endif() list(GET _ort_libs 0 QTMESH_ONNX_RUNTIME_LIB) set(QTMESH_ONNX_RUNTIME_LIB "${QTMESH_ONNX_RUNTIME_LIB}" CACHE FILEPATH "Path to the ONNX Runtime shared library to ship next to the binary" FORCE) -# The lib dir holds the versioned shared lib PLUS its SONAME symlinks -# (libonnxruntime.so.1 / libonnxruntime.so) that the loader actually requests -# at runtime — copying only the resolved file leaves the binary unable to find -# libonnxruntime.so.1. Expose the dir so the POST_BUILD copies the whole set. get_filename_component(QTMESH_ONNX_LIB_DIR "${QTMESH_ONNX_RUNTIME_LIB}" DIRECTORY) set(QTMESH_ONNX_LIB_DIR "${QTMESH_ONNX_LIB_DIR}" - CACHE PATH "Directory of the ONNX Runtime shared library + its SONAME symlinks" FORCE) + CACHE PATH "Directory of the ONNX Runtime shared library + provider libs" FORCE) + +set(QTMESH_ONNX_GPU "${QTMESH_ONNX_GPU}" CACHE BOOL + "Using GPU ONNX Runtime package (CUDA/DirectML EPs)" FORCE) add_library(qtmesh_onnx SHARED IMPORTED GLOBAL) set_target_properties(qtmesh_onnx PROPERTIES IMPORTED_LOCATION "${QTMESH_ONNX_RUNTIME_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${QTMESH_ONNX_INCLUDE_DIR}") if(WIN32) - # On Windows the import library is needed for linking. file(GLOB _ort_implib "${QTMESH_ONNX_ROOT}/lib/onnxruntime.lib") if(_ort_implib) list(GET _ort_implib 0 _ort_implib0) @@ -91,4 +115,8 @@ if(WIN32) endif() endif() -message(STATUS "ONNX Runtime ${QTMESH_ONNX_VERSION}: ${QTMESH_ONNX_RUNTIME_LIB}") +if(QTMESH_ONNX_GPU) + message(STATUS "ONNX Runtime ${QTMESH_ONNX_VERSION} (GPU): ${QTMESH_ONNX_RUNTIME_LIB}") +else() + message(STATUS "ONNX Runtime ${QTMESH_ONNX_VERSION} (CPU): ${QTMESH_ONNX_RUNTIME_LIB}") +endif() diff --git a/qml/AISettingsDialog.qml b/qml/AISettingsDialog.qml index 5279072d4..31df437c2 100644 --- a/qml/AISettingsDialog.qml +++ b/qml/AISettingsDialog.qml @@ -622,6 +622,42 @@ Dialog { } } + GroupBox { + Layout.fillWidth: true + title: "ONNX Models" + visible: OnnxRuntimeSettings.onnxAvailable + + ColumnLayout { + anchors.fill: parent + spacing: 8 + + CheckBox { + id: onnxGpuCheckBox + text: "Prefer GPU for ONNX models (when available)" + checked: OnnxRuntimeSettings.preferGpu + onCheckedChanged: OnnxRuntimeSettings.preferGpu = checked + + contentItem: Text { + text: onnxGpuCheckBox.text + color: textColor + leftPadding: onnxGpuCheckBox.indicator.width + onnxGpuCheckBox.spacing + verticalAlignment: Text.AlignVCenter + wrapMode: Text.WordWrap + Layout.fillWidth: true + } + } + + Text { + Layout.fillWidth: true + text: OnnxRuntimeSettings.gpuProviderNote + font.pointSize: 9 + color: OnnxRuntimeSettings.gpuProviderReady ? "#2e7d32" + : (OnnxRuntimeSettings.preferGpu ? "#e65100" : Qt.darker(textColor, 1.3)) + wrapMode: Text.WordWrap + } + } + } + GroupBox { Layout.fillWidth: true title: "Models Directory" @@ -675,7 +711,7 @@ Dialog { Text { Layout.fillWidth: true - text: "Note: Settings changes will take effect when loading a new model." + text: "Note: LLM settings take effect when loading a new model. ONNX GPU preference applies immediately." font.pointSize: 9 font.italic: true color: Qt.darker(textColor, 1.5) diff --git a/qml/CollapsibleSection.qml b/qml/CollapsibleSection.qml index 6f4048001..9de62db22 100644 --- a/qml/CollapsibleSection.qml +++ b/qml/CollapsibleSection.qml @@ -11,6 +11,8 @@ Column { property bool sectionVisible: true default property alias content: contentLoader.sourceComponent + signal contentReady() + visible: sectionVisible width: parent ? parent.width : 200 @@ -58,7 +60,25 @@ Column { Loader { id: contentLoader width: parent.width - active: root.expanded + // Defer activation to the next event-loop turn. Synchronous Loader + // startup while a parent component is still finalizing (e.g. expanding + // a section during a binding cascade) can SIGSEGV — see PropertiesPanel + // Component.onCompleted comment. + active: loadActive visible: root.expanded + property bool loadActive: false + onLoaded: root.contentReady() + } + + onExpandedChanged: { + if (root.expanded) + Qt.callLater(function() { contentLoader.loadActive = true }) + else + contentLoader.loadActive = false + } + + Component.onCompleted: { + if (root.expanded) + Qt.callLater(function() { contentLoader.loadActive = true }) } } diff --git a/qml/ThemedComboBox.qml b/qml/ThemedComboBox.qml index 8a537a398..5fcf5fbd4 100644 --- a/qml/ThemedComboBox.qml +++ b/qml/ThemedComboBox.qml @@ -17,7 +17,15 @@ ComboBox { leftPadding: 6 rightPadding: 6 contentItem: Text { - text: modelData + text: { + if (control.textRole && model && typeof model === "object") + return model[control.textRole] ?? "" + if (typeof modelData === "string") + return modelData + if (control.textRole && modelData && typeof modelData === "object") + return modelData[control.textRole] ?? "" + return modelData !== undefined ? String(modelData) : "" + } color: ThemeManager.textColor font: control.font elide: Text.ElideRight diff --git a/scripts/install-onnx-gpu-deps.sh b/scripts/install-onnx-gpu-deps.sh new file mode 100755 index 000000000..dd1ce9f3b --- /dev/null +++ b/scripts/install-onnx-gpu-deps.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Install runtime deps for ONNX Runtime CUDA EP (Linux, ORT 1.20.x + CUDA 12). +# Downloads cuDNN 9 + cuBLAS into .cache/cuda-deps/ (no sudo). QtMeshEditor picks +# this up automatically via OnnxRuntimeSettings::prepareRuntimeEnvironment(). + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DEST="$ROOT/.cache/cuda-deps" + +if ! command -v nvidia-smi >/dev/null 2>&1; then + echo "No NVIDIA GPU driver (nvidia-smi) found." >&2 + exit 1 +fi + +echo "GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)" +mkdir -p "$DEST" + +if [[ -f "$DEST/nvidia/cudnn/lib/libcudnn.so.9" ]]; then + echo "cuDNN already present at $DEST/nvidia/cudnn/lib" +else + echo "Installing nvidia-cudnn-cu12 + nvidia-cublas-cu12 into $DEST ..." + python3 -m pip install \ + nvidia-cudnn-cu12 nvidia-cublas-cu12 \ + -t "$DEST" --no-cache-dir --upgrade +fi + +if [[ ! -f "$DEST/nvidia/cudnn/lib/libcudnn.so.9" ]]; then + echo "cuDNN install failed." >&2 + exit 1 +fi + +echo "Done. Rebuild with GPU ONNX Runtime if needed:" +echo " cmake . -B build_local -DENABLE_ONNX=ON -DQTMESH_ONNX_GPU=ON" +echo " cmake --build build_local --target QtMeshEditor -j4" +echo "" +echo "Enable AI Settings → Prefer GPU for ONNX models, then run UniRig." +echo "Debug: QTMESH_ONNX_DEBUG=1 ./build_local/bin/qtmesh rig model.obj --algo unirig -o out.fbx" diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index d04755ea5..83a294550 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -7,6 +7,9 @@ #include "SceneLightsIO.h" #include "SceneLightsCLI.h" #include "Mocap/MocapCLI.h" +#ifdef ENABLE_MOCAP +#include "Mocap/MocapCameraHints.h" +#endif #include "AnimationMerger.h" #include "MotionInbetween.h" #include "MotionLibrary.h" @@ -29,6 +32,7 @@ #include "VertexCacheOptimizer.h" #include "ExportOptimizer.h" #include "UvUnwrap.h" +#include "OnnxRuntimeSettings.h" #include "HDR/HdrBundledLibrary.h" #include "HDR/HDREnvironmentManager.h" #include "HDR/HdrMaterialScript.h" @@ -1552,10 +1556,14 @@ int CLIPipeline::run(int argc, char* argv[]) // This also lets us call _exit() after the subcommand returns, // skipping QApplication/Ogre static destructor teardown that // causes SIGSEGV on macOS (GL context cleanup race). +#ifdef ENABLE_MOCAP + MocapCameraHints::ensureMultimediaBackendSafe(); +#endif QApplication a(argc, argv); QCoreApplication::setOrganizationName("QtMeshEditor"); QCoreApplication::setApplicationName("QtMeshEditor"); QCoreApplication::setApplicationVersion(QTMESHEDITOR_VERSION); + OnnxRuntimeSettings::prepareRuntimeEnvironment(); // Redirect stdout to stderr so Ogre/Qt debug output doesn't // pollute the CLI pipeline output (JSON, info text, etc.) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 67bc2da24..a05e9b43b 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -57,6 +57,7 @@ MaterialHighlighter.cpp QMLMaterialHighlighter.cpp LLMWorker.cpp LLMManager.cpp +OnnxRuntimeSettings.cpp ModelDownloader.cpp LLMSettingsWidget.cpp MCPServer.cpp @@ -311,6 +312,7 @@ MaterialHighlighter.h QMLMaterialHighlighter.h LLMWorker.h LLMManager.h +OnnxRuntimeSettings.h ModelDownloader.h LLMSettingsWidget.h MCPServer.h @@ -780,7 +782,9 @@ if(ENABLE_ONNX) file(GLOB _ort_runtime_libs "${QTMESH_ONNX_LIB_DIR}/libonnxruntime.so*" "${QTMESH_ONNX_LIB_DIR}/libonnxruntime*.dylib" - "${QTMESH_ONNX_LIB_DIR}/onnxruntime.dll") + "${QTMESH_ONNX_LIB_DIR}/libonnxruntime_providers*.so" + "${QTMESH_ONNX_LIB_DIR}/onnxruntime.dll" + "${QTMESH_ONNX_LIB_DIR}/onnxruntime_providers_*.dll") foreach(_ortlib ${_ort_runtime_libs}) add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different @@ -945,7 +949,9 @@ if(BUILD_TESTS) file(GLOB _ort_test_libs "${QTMESH_ONNX_LIB_DIR}/libonnxruntime.so*" "${QTMESH_ONNX_LIB_DIR}/libonnxruntime*.dylib" - "${QTMESH_ONNX_LIB_DIR}/onnxruntime.dll") + "${QTMESH_ONNX_LIB_DIR}/libonnxruntime_providers*.so" + "${QTMESH_ONNX_LIB_DIR}/onnxruntime.dll" + "${QTMESH_ONNX_LIB_DIR}/onnxruntime_providers_*.dll") foreach(_ortlib ${_ort_test_libs}) add_custom_command(TARGET UnitTests POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different diff --git a/src/FaceRig/FaceLandmarkDetector.cpp b/src/FaceRig/FaceLandmarkDetector.cpp index 1ba874f20..cf65cb7af 100644 --- a/src/FaceRig/FaceLandmarkDetector.cpp +++ b/src/FaceRig/FaceLandmarkDetector.cpp @@ -2,6 +2,7 @@ #include "ArkitTemplate.h" // reuse its model dir + base-url convention #include "../ModelDownloader.h" +#include "../OnnxRuntimeSettings.h" #include "../SentryReporter.h" #include @@ -149,13 +150,7 @@ bool FaceLandmarkDetector::load(const QString& path) d->env = std::make_unique(ORT_LOGGING_LEVEL_WARNING, "qtmesh_facelmk"); Ort::SessionOptions so; - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); -#if defined(__APPLE__) - try { - std::unordered_map coreml; - so.AppendExecutionProvider("CoreML", coreml); - } catch (...) { /* CPU fallback */ } -#endif + OnnxRuntimeSettings::configureSessionOptions(so); #ifdef _WIN32 const std::wstring wp = p.toStdWString(); d->session = std::make_unique(*d->env, wp.c_str(), so); diff --git a/src/ImageTo3D/BackgroundRemover.cpp b/src/ImageTo3D/BackgroundRemover.cpp index de0efd88f..eec2f5daa 100644 --- a/src/ImageTo3D/BackgroundRemover.cpp +++ b/src/ImageTo3D/BackgroundRemover.cpp @@ -10,6 +10,7 @@ #ifdef ENABLE_ONNX #include "ModelDownloader.h" +#include "OnnxRuntimeSettings.h" #include #include #include @@ -132,13 +133,7 @@ BackgroundRemover::Result BackgroundRemover::removeBackground(const QImage& imag try { Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_rembg"); Ort::SessionOptions so; - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); -#ifdef __APPLE__ - try { - std::unordered_map coremlOpts; - so.AppendExecutionProvider("CoreML", coremlOpts); - } catch (const Ort::Exception&) {} -#endif + OnnxRuntimeSettings::configureSessionOptions(so); #ifdef _WIN32 Ort::Session session(env, modelPath.toStdWString().c_str(), so); #else diff --git a/src/ImageTo3D/MeshGenPredictor.cpp b/src/ImageTo3D/MeshGenPredictor.cpp index 52c3202f1..533456b54 100644 --- a/src/ImageTo3D/MeshGenPredictor.cpp +++ b/src/ImageTo3D/MeshGenPredictor.cpp @@ -5,6 +5,7 @@ #include "MeshGenBaker.h" // xatlas unwrap + diffuse texture bake #include "PbrMapSynth.h" // toNCHW (image → planar [0,1]) #include "BackgroundRemover.h" +#include "OnnxRuntimeSettings.h" #include "TripoSGPredictor.h" // Backend::TripoSG dispatch #include @@ -271,13 +272,7 @@ MeshGenPredictor::Result MeshGenPredictor::predict(const QImage& image, try { Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_triposr"); Ort::SessionOptions so; - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); -#ifdef __APPLE__ - try { - std::unordered_map coremlOpts; - so.AppendExecutionProvider("CoreML", coremlOpts); - } catch (const Ort::Exception&) {} -#endif + OnnxRuntimeSettings::configureSessionOptions(so); Ort::Session encoder = openSession(env, so, encoderModelPath); Ort::Session decoder = openSession(env, so, decoderModelPath); Ort::AllocatorWithDefaultOptions alloc; diff --git a/src/ImageTo3D/TripoSGPredictor.cpp b/src/ImageTo3D/TripoSGPredictor.cpp index 252e3ab99..a9038c560 100644 --- a/src/ImageTo3D/TripoSGPredictor.cpp +++ b/src/ImageTo3D/TripoSGPredictor.cpp @@ -2,6 +2,7 @@ #include "MarchingCubes.h" #include "MeshRefine.h" +#include "OnnxRuntimeSettings.h" #include "PbrMapSynth.h" // toNCHW (image → planar [0,1]) #include @@ -270,20 +271,16 @@ MeshGenPredictor::Result TripoSGPredictor::predict( // which is invoked ~2000×/run — the one graph where GPU dispatch // clearly pays and the compile cost is trivial. Ort::SessionOptions cpuOnly; - cpuOnly.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + OnnxRuntimeSettings::SessionConfig cpuCfg; + cpuCfg.appendGpu = false; + OnnxRuntimeSettings::configureSessionOptions(cpuOnly, cpuCfg); Ort::SessionOptions gpu; - gpu.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + OnnxRuntimeSettings::SessionConfig gpuCfg; + gpuCfg.coreMlStyle = OnnxRuntimeSettings::CoreMlStyle::MlProgram; + OnnxRuntimeSettings::configureSessionOptions(gpu, gpuCfg); bool gpuAvailable = false; #ifdef __APPLE__ - try { - // Legacy "NeuralNetwork" format maps almost none of these ops — - // MLProgram is what actually reaches the M-series GPU/ANE. - std::unordered_map coreml; - coreml["ModelFormat"] = "MLProgram"; - coreml["MLComputeUnits"] = "ALL"; - gpu.AppendExecutionProvider("CoreML", coreml); - gpuAvailable = true; - } catch (const Ort::Exception&) {} + gpuAvailable = OnnxRuntimeSettings::instance()->preferGpu(); #endif const bool ditOnGpu = gpuAvailable && qEnvironmentVariableIntValue("QTMESH_TRIPOSG_COREML_DIT") == 1; diff --git a/src/LLMSettingsWidget.cpp b/src/LLMSettingsWidget.cpp index a889fc363..f0400b3f6 100644 --- a/src/LLMSettingsWidget.cpp +++ b/src/LLMSettingsWidget.cpp @@ -79,7 +79,7 @@ void LLMSettingsWidget::setupUI() setupAIModelCatalogTab(aiModelsTab); m_tabWidget->addTab(modelsTab, "LLM Models"); - m_tabWidget->addTab(settingsTab, "LLM Settings"); + m_tabWidget->addTab(settingsTab, "Settings"); m_tabWidget->addTab(downloadTab, "LLM Download"); m_tabWidget->addTab(aiModelsTab, "QtMeshEditor Models"); @@ -222,6 +222,25 @@ void LLMSettingsWidget::setupSettingsTab(QWidget *parent) layout->addWidget(inferenceGroup); + if (OnnxRuntimeSettings::instance()->onnxAvailable()) { + QGroupBox *onnxGroup = new QGroupBox("ONNX Models", parent); + QVBoxLayout *onnxLayout = new QVBoxLayout(onnxGroup); + + m_onnxPreferGpuCheckBox = + new QCheckBox("Prefer GPU for ONNX models (when available)", onnxGroup); + m_onnxPreferGpuCheckBox->setToolTip( + "UniRig, SkinTokens, PBR synthesis, image-to-3D, and other ONNX " + "features. Takes effect on the next run (no Apply needed)."); + onnxLayout->addWidget(m_onnxPreferGpuCheckBox); + + m_onnxGpuNoteLabel = new QLabel(onnxGroup); + m_onnxGpuNoteLabel->setWordWrap(true); + m_onnxGpuNoteLabel->setStyleSheet("color: gray; font-size: 11px;"); + onnxLayout->addWidget(m_onnxGpuNoteLabel); + + layout->addWidget(onnxGroup); + } + // Hardware settings QGroupBox *hardwareGroup = new QGroupBox("Hardware Settings", parent); QFormLayout *hwFormLayout = new QFormLayout(hardwareGroup); @@ -266,6 +285,10 @@ void LLMSettingsWidget::setupSettingsTab(QWidget *parent) connect(m_topPSpinBox, QOverload::of(&QDoubleSpinBox::valueChanged), this, &LLMSettingsWidget::onSettingsChanged); connect(m_topKSpinBox, QOverload::of(&QSpinBox::valueChanged), this, &LLMSettingsWidget::onSettingsChanged); connect(m_repeatPenaltySpinBox, QOverload::of(&QDoubleSpinBox::valueChanged), this, &LLMSettingsWidget::onSettingsChanged); + if (m_onnxPreferGpuCheckBox) { + connect(m_onnxPreferGpuCheckBox, &QCheckBox::toggled, + this, &LLMSettingsWidget::onOnnxPreferGpuToggled); + } } void LLMSettingsWidget::setupDownloadTab(QWidget *parent) @@ -436,6 +459,17 @@ void LLMSettingsWidget::loadCurrentSettings() m_topKSpinBox->setValue(settings.topK); m_repeatPenaltySpinBox->setValue(static_cast(settings.repeatPenalty)); + if (m_onnxPreferGpuCheckBox) { + OnnxRuntimeSettings* ort = OnnxRuntimeSettings::instance(); + ort->refreshGpuProviderStatus(); + ort->loadSettings(); + m_onnxPreferGpuCheckBox->blockSignals(true); + m_onnxPreferGpuCheckBox->setChecked(ort->preferGpu()); + m_onnxPreferGpuCheckBox->blockSignals(false); + if (m_onnxGpuNoteLabel) + m_onnxGpuNoteLabel->setText(ort->gpuProviderNote()); + } + m_applyButton->setEnabled(false); } @@ -649,10 +683,22 @@ void LLMSettingsWidget::onApplySettings() settings.repeatPenalty = static_cast(m_repeatPenaltySpinBox->value()); LLMManager::instance()->setSettings(settings); + m_applyButton->setEnabled(false); QMessageBox::information(this, "Settings Applied", - "Settings have been saved. They will take effect on the next model load."); + "LLM settings have been saved. They will take effect on the next model load."); +} + +void LLMSettingsWidget::onOnnxPreferGpuToggled(bool checked) +{ + OnnxRuntimeSettings* ort = OnnxRuntimeSettings::instance(); + ort->setPreferGpu(checked); + if (m_onnxGpuNoteLabel) + m_onnxGpuNoteLabel->setText(ort->gpuProviderNote()); + SentryReporter::addBreadcrumb( + "ui.action", + QStringLiteral("AI settings: ONNX prefer GPU %1").arg(checked ? "on" : "off")); } void LLMSettingsWidget::onResetDefaults() @@ -667,6 +713,13 @@ void LLMSettingsWidget::onResetDefaults() m_topKSpinBox->setValue(defaults.topK); m_repeatPenaltySpinBox->setValue(static_cast(defaults.repeatPenalty)); + if (m_onnxPreferGpuCheckBox) { + m_onnxPreferGpuCheckBox->blockSignals(true); + m_onnxPreferGpuCheckBox->setChecked(OnnxRuntimeSettings::defaultPreferGpu()); + m_onnxPreferGpuCheckBox->blockSignals(false); + onOnnxPreferGpuToggled(m_onnxPreferGpuCheckBox->isChecked()); + } + m_applyButton->setEnabled(true); } diff --git a/src/LLMSettingsWidget.h b/src/LLMSettingsWidget.h index 455762252..325afd861 100644 --- a/src/LLMSettingsWidget.h +++ b/src/LLMSettingsWidget.h @@ -13,9 +13,11 @@ #include #include #include +#include #include "LLMManager.h" #include "AIModelCatalog.h" #include "ModelDownloader.h" +#include "OnnxRuntimeSettings.h" #ifdef ENABLE_STABLE_DIFFUSION #include "SDManager.h" #endif @@ -48,6 +50,7 @@ private slots: void onSettingsChanged(); void onApplySettings(); void onResetDefaults(); + void onOnnxPreferGpuToggled(bool checked); #ifdef ENABLE_STABLE_DIFFUSION void onSDLoadModelClicked(); @@ -117,6 +120,8 @@ private slots: QDoubleSpinBox *m_topPSpinBox; QSpinBox *m_topKSpinBox; QDoubleSpinBox *m_repeatPenaltySpinBox; + QCheckBox *m_onnxPreferGpuCheckBox = nullptr; + QLabel *m_onnxGpuNoteLabel = nullptr; QPushButton *m_applyButton; QPushButton *m_resetButton; diff --git a/src/LLMSettingsWidget_test.cpp b/src/LLMSettingsWidget_test.cpp index ef432e41d..cea3ae596 100644 --- a/src/LLMSettingsWidget_test.cpp +++ b/src/LLMSettingsWidget_test.cpp @@ -47,7 +47,7 @@ TEST_F(LLMSettingsWidgetTest, TabNames) QTabWidget* tabWidget = widget.findChild(); ASSERT_NE(tabWidget, nullptr); EXPECT_EQ(tabWidget->tabText(0), "LLM Models"); - EXPECT_EQ(tabWidget->tabText(1), "LLM Settings"); + EXPECT_EQ(tabWidget->tabText(1), "Settings"); EXPECT_EQ(tabWidget->tabText(2), "LLM Download"); EXPECT_EQ(tabWidget->tabText(3), "QtMeshEditor Models"); #ifdef ENABLE_STABLE_DIFFUSION diff --git a/src/MeshSegmenter.cpp b/src/MeshSegmenter.cpp index 7c1903700..7b260cb54 100644 --- a/src/MeshSegmenter.cpp +++ b/src/MeshSegmenter.cpp @@ -1,6 +1,7 @@ #include "MeshSegmenter.h" #include "ModelDownloader.h" +#include "OnnxRuntimeSettings.h" #include #include @@ -1314,12 +1315,8 @@ MeshSegmenter::Result MeshSegmenter::predict(const float* positions, int vertexC Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_segment"); Ort::SessionOptions so; + OnnxRuntimeSettings::configureSessionOptions(so); so.SetIntraOpNumThreads(1); - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); -#ifdef __APPLE__ - try { std::unordered_map c; so.AppendExecutionProvider("CoreML", c); } - catch (const Ort::Exception&) {} -#endif #ifdef _WIN32 const std::wstring wpath = modelPath.toStdWString(); Ort::Session session(env, wpath.c_str(), so); @@ -1469,12 +1466,8 @@ MeshSegmenter::Category MeshSegmenter::classifyCategory(const float* positions, Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_segment_cls"); Ort::SessionOptions so; + OnnxRuntimeSettings::configureSessionOptions(so); so.SetIntraOpNumThreads(1); - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); -#ifdef __APPLE__ - try { std::unordered_map c; so.AppendExecutionProvider("CoreML", c); } - catch (const Ort::Exception&) {} -#endif #ifdef _WIN32 const std::wstring wpath = classifierPath.toStdWString(); Ort::Session session(env, wpath.c_str(), so); diff --git a/src/Mocap/FaceCapPredictor.cpp b/src/Mocap/FaceCapPredictor.cpp index 0d93790d9..6130e122c 100644 --- a/src/Mocap/FaceCapPredictor.cpp +++ b/src/Mocap/FaceCapPredictor.cpp @@ -6,6 +6,7 @@ #include "FaceCapGeom.h" #include "FaceCapPose.h" #include "../ModelDownloader.h" +#include "../OnnxRuntimeSettings.h" #include #include @@ -177,14 +178,8 @@ struct FaceCapPredictor::Impl { std::unique_ptr openSession(const QString& path) { Ort::SessionOptions so; + OnnxRuntimeSettings::configureSessionOptions(so); so.SetIntraOpNumThreads(2); -#ifdef Q_OS_MACOS - // CoreML EP where it helps; CPU fallback is always registered. - try { - std::unordered_map opts; - so.AppendExecutionProvider("CoreML", opts); - } catch (const Ort::Exception&) {} -#endif #ifdef Q_OS_WIN const std::wstring wpath = path.toStdWString(); return std::make_unique(env, wpath.c_str(), so); diff --git a/src/Mocap/PoseCapPredictor.cpp b/src/Mocap/PoseCapPredictor.cpp index ea24e8730..9c5c36e0b 100644 --- a/src/Mocap/PoseCapPredictor.cpp +++ b/src/Mocap/PoseCapPredictor.cpp @@ -4,6 +4,7 @@ #include "FaceCapGeom.h" #include "../ModelDownloader.h" +#include "../OnnxRuntimeSettings.h" #include #include @@ -167,13 +168,8 @@ struct PoseCapPredictor::Impl { std::unique_ptr openSession(const QString& path) { Ort::SessionOptions so; + OnnxRuntimeSettings::configureSessionOptions(so); so.SetIntraOpNumThreads(2); -#ifdef Q_OS_MACOS - try { - std::unordered_map opts; - so.AppendExecutionProvider("CoreML", opts); - } catch (const Ort::Exception&) {} -#endif #ifdef Q_OS_WIN const std::wstring wpath = path.toStdWString(); return std::make_unique(env, wpath.c_str(), so); diff --git a/src/MotionGenerator.cpp b/src/MotionGenerator.cpp index eafff638d..8e862d8a5 100644 --- a/src/MotionGenerator.cpp +++ b/src/MotionGenerator.cpp @@ -1,6 +1,7 @@ #include "MotionGenerator.h" #include "ModelDownloader.h" +#include "OnnxRuntimeSettings.h" #include #include @@ -240,13 +241,8 @@ MotionGenerator::Result MotionGenerator::generate( // ---- run the ONNX model ---- Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_t2m"); Ort::SessionOptions so; + OnnxRuntimeSettings::configureSessionOptions(so); so.SetIntraOpNumThreads(1); -#ifdef __APPLE__ - try { - std::unordered_map opt; - so.AppendExecutionProvider("CoreML", opt); - } catch (const Ort::Exception&) { /* CPU fallback */ } -#endif #ifdef _WIN32 const std::wstring wpath = modelPathArg.toStdWString(); Ort::Session session(env, wpath.c_str(), so); diff --git a/src/MotionInbetween.cpp b/src/MotionInbetween.cpp index ecf23d341..92e065c9b 100644 --- a/src/MotionInbetween.cpp +++ b/src/MotionInbetween.cpp @@ -1,6 +1,7 @@ #include "MotionInbetween.h" #include "ModelDownloader.h" +#include "OnnxRuntimeSettings.h" #include #include @@ -428,14 +429,8 @@ MotionInbetween::Result MotionInbetween::predict( try { Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_inbetween"); Ort::SessionOptions so; + OnnxRuntimeSettings::configureSessionOptions(so); so.SetIntraOpNumThreads(1); - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); -#ifdef __APPLE__ - try { - std::unordered_map coremlOpts; - so.AppendExecutionProvider("CoreML", coremlOpts); - } catch (const Ort::Exception&) { /* CPU EP fallback */ } -#endif #ifdef _WIN32 const std::wstring wpath = modelPath.toStdWString(); Ort::Session session(env, wpath.c_str(), so); diff --git a/src/OnnxRuntimeSettings.cpp b/src/OnnxRuntimeSettings.cpp new file mode 100644 index 000000000..8b259d9d0 --- /dev/null +++ b/src/OnnxRuntimeSettings.cpp @@ -0,0 +1,328 @@ +#include "OnnxRuntimeSettings.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef ENABLE_ONNX +#include +#endif + +namespace { +constexpr const char* kPreferGpuKey = "ai/onnxPreferGpu"; + +#ifdef QTMESH_ONNX_GPU_BUILD +void addCudaDepsTree(const QString& root, QStringList& paths) +{ + const QString cudnn = QDir::cleanPath(root + QStringLiteral("/nvidia/cudnn/lib")); + const QString cublas = QDir::cleanPath(root + QStringLiteral("/nvidia/cublas/lib")); + if (QFileInfo::exists(cudnn + QStringLiteral("/libcudnn.so.9")) + && !paths.contains(cudnn)) + paths << cudnn; + if (QFileInfo(cublas).isDir() && !paths.contains(cublas)) + paths << cublas; +} + +QStringList cudnnSearchPaths() +{ + QStringList paths; + auto addIfCudnn = [&](const QString& dir) { + const QString d = QDir::cleanPath(dir); + if (QFileInfo::exists(d + QStringLiteral("/libcudnn.so.9")) + && !paths.contains(d)) + paths << d; + }; + + const QByteArray env = qgetenv("QTMESH_CUDNN_LIB_DIR"); + for (const QByteArray& part : env.split(':')) { + if (!part.isEmpty()) + addIfCudnn(QString::fromUtf8(part)); + } + + const QString appDir = QCoreApplication::applicationDirPath(); + addCudaDepsTree(appDir + QStringLiteral("/cuda-deps"), paths); + addCudaDepsTree(appDir + QStringLiteral("/../cuda-deps"), paths); + + QDir repo(appDir); + if (repo.cdUp() && repo.cdUp()) { + const QString root = repo.absolutePath(); + addCudaDepsTree(root + QStringLiteral("/.cache/cuda-deps"), paths); + addCudaDepsTree(root + QStringLiteral("/cuda-deps"), paths); + } + + addIfCudnn(QStringLiteral("/usr/lib/x86_64-linux-gnu")); + addIfCudnn(QStringLiteral("/usr/local/cuda/lib64")); + return paths; +} + +void prependLdLibraryPath(const QStringList& dirs) +{ + if (dirs.isEmpty()) + return; + QStringList merged = dirs; + const QByteArray old = qgetenv("LD_LIBRARY_PATH"); + if (!old.isEmpty()) + merged << QString::fromUtf8(old).split(':', + Qt::SkipEmptyParts); + qputenv("LD_LIBRARY_PATH", merged.join(':').toUtf8()); + if (qEnvironmentVariableIsSet("QTMESH_ONNX_DEBUG")) { + fprintf(stderr, "[onnx] LD_LIBRARY_PATH=%s\n", + qgetenv("LD_LIBRARY_PATH").constData()); + } +} + +bool cudaProviderLibraryLoads() +{ + // Do NOT dlopen libonnxruntime_providers_cuda.so here — it must be loaded + // by libonnxruntime.so (standalone load fails: undefined Provider_GetHost). + for (const QString& dir : cudnnSearchPaths()) { + const QString cudnn = dir + QStringLiteral("/libcudnn.so.9"); + if (!QFileInfo::exists(cudnn)) + continue; + QLibrary lib(cudnn); + if (lib.load()) { + lib.unload(); + return true; + } + if (qEnvironmentVariableIsSet("QTMESH_ONNX_DEBUG")) { + fprintf(stderr, "[onnx] libcudnn load failed (%s): %s\n", + dir.toUtf8().constData(), + lib.errorString().toUtf8().constData()); + } + } + return false; +} +#endif + +} // namespace + +void OnnxRuntimeSettings::prepareRuntimeEnvironment() +{ +#ifdef QTMESH_ONNX_GPU_BUILD + static bool done = false; + if (done) + return; + done = true; + prependLdLibraryPath(cudnnSearchPaths()); +#endif +} + +OnnxRuntimeSettings::OnnxRuntimeSettings(QObject* parent) + : QObject(parent) +{ + loadSettings(); + prepareRuntimeEnvironment(); + refreshGpuProviderStatus(); +} + +OnnxRuntimeSettings* OnnxRuntimeSettings::instance() +{ + static OnnxRuntimeSettings inst; + return &inst; +} + +OnnxRuntimeSettings* OnnxRuntimeSettings::qmlInstance(QQmlEngine* engine, + QJSEngine* scriptEngine) +{ + Q_UNUSED(engine) + Q_UNUSED(scriptEngine) + return instance(); +} + +bool OnnxRuntimeSettings::defaultPreferGpu() +{ +#if defined(__APPLE__) || defined(QTMESH_ONNX_GPU_BUILD) + return true; +#else + return false; +#endif +} + +bool OnnxRuntimeSettings::onnxAvailable() const +{ +#ifdef ENABLE_ONNX + return true; +#else + return false; +#endif +} + +bool OnnxRuntimeSettings::preferGpu() const +{ + if (qEnvironmentVariableIsSet("QTMESH_ONNX_PREFER_GPU")) + return qEnvironmentVariableIntValue("QTMESH_ONNX_PREFER_GPU") != 0; + return m_preferGpu; +} + +void OnnxRuntimeSettings::setPreferGpu(bool value) +{ + if (m_preferGpu == value) + return; + m_preferGpu = value; + saveSettings(); + updateGpuProviderNoteLocked(); + emit settingsChanged(); +} + +bool OnnxRuntimeSettings::gpuProviderBundled() const +{ + return m_gpuProviderBundled; +} + +bool OnnxRuntimeSettings::gpuProviderReady() const +{ + return m_gpuProviderReady; +} + +QString OnnxRuntimeSettings::gpuProviderNote() const +{ + return m_gpuProviderNote; +} + +void OnnxRuntimeSettings::refreshGpuProviderStatus() +{ +#ifdef QTMESH_ONNX_GPU_BUILD + prepareRuntimeEnvironment(); + const QString cudaSo = + QCoreApplication::applicationDirPath() + + QStringLiteral("/libonnxruntime_providers_cuda.so"); + m_gpuProviderBundled = QFileInfo::exists(cudaSo); + m_gpuProviderReady = m_gpuProviderBundled && cudaProviderLibraryLoads(); +#else + m_gpuProviderBundled = false; + m_gpuProviderReady = false; +#endif + updateGpuProviderNoteLocked(); + emit gpuStatusChanged(); + emit settingsChanged(); +} + +void OnnxRuntimeSettings::updateGpuProviderNoteLocked() +{ +#ifndef ENABLE_ONNX + m_gpuProviderNote = tr("ONNX is not enabled in this build."); +#elif defined(__APPLE__) + if (!preferGpu()) { + m_gpuProviderNote = tr("CPU only."); + return; + } + m_gpuProviderNote = tr("Uses CoreML (GPU/ANE) when available; falls back to CPU."); +#elif defined(_WIN32) && !defined(__MINGW32__) + if (!preferGpu()) { + m_gpuProviderNote = tr("CPU only."); + return; + } + m_gpuProviderNote = + tr("Attempts DirectML when available; rebuild with -DQTMESH_ONNX_GPU=ON for the GPU package."); +#elif defined(QTMESH_ONNX_GPU_BUILD) + if (!m_gpuProviderBundled) { + m_gpuProviderNote = + tr("GPU ONNX Runtime was enabled at build time but libonnxruntime_providers_cuda.so " + "is missing next to the executable — rebuild QtMeshEditor."); + return; + } + if (!m_gpuProviderReady) { + m_gpuProviderNote = + tr("CUDA provider found but cuDNN 9 is missing — run " + "scripts/install-onnx-gpu-deps.sh (installs to .cache/cuda-deps). " + "Until then UniRig runs on CPU."); + return; + } + m_gpuProviderNote = preferGpu() + ? tr("CUDA execution provider ready (NVIDIA GPU).") + : tr("CUDA available but disabled — enable “Prefer GPU” above."); +#else + if (!preferGpu()) { + m_gpuProviderNote = tr("CPU only."); + return; + } + m_gpuProviderNote = + tr("This build bundles CPU-only ONNX Runtime. Rebuild with " + "-DQTMESH_ONNX_GPU=ON (auto-detected when nvidia-smi is present) to use your NVIDIA GPU."); +#endif +} + +void OnnxRuntimeSettings::loadSettings() +{ + QSettings settings; + m_preferGpu = settings.value(kPreferGpuKey, defaultPreferGpu()).toBool(); + updateGpuProviderNoteLocked(); +} + +void OnnxRuntimeSettings::saveSettings() +{ + QSettings settings; + settings.setValue(kPreferGpuKey, m_preferGpu); +} + +#ifdef ENABLE_ONNX +void OnnxRuntimeSettings::configureSessionOptions(Ort::SessionOptions& so) +{ + configureSessionOptions(so, SessionConfig{}); +} + +void OnnxRuntimeSettings::configureSessionOptions(Ort::SessionOptions& so, + const SessionConfig& cfg) +{ + so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + if (cfg.reserveUiThreadCore) { + const unsigned hc = std::thread::hardware_concurrency(); + so.SetIntraOpNumThreads(hc > 1 ? static_cast(hc - 1) : 1); + } + if (!cfg.allowSpinning) + so.AddConfigEntry("session.intra_op.allow_spinning", "0"); + if (cfg.appendGpu && instance()->preferGpu()) { + if (!tryAppendGpuExecutionProvider(so, cfg.coreMlStyle) + && qEnvironmentVariableIsSet("QTMESH_ONNX_DEBUG")) { + fprintf(stderr, + "[onnx] preferGpu=1 but no GPU execution provider was registered " + "(see AI Settings → ONNX note)\n"); + } + } +} + +bool OnnxRuntimeSettings::tryAppendGpuExecutionProvider(Ort::SessionOptions& so, + CoreMlStyle style) +{ + prepareRuntimeEnvironment(); +#ifdef __APPLE__ + try { + std::unordered_map opts; + if (style == CoreMlStyle::MlProgram) { + opts["ModelFormat"] = "MLProgram"; + opts["MLComputeUnits"] = "ALL"; + } + so.AppendExecutionProvider("CoreML", opts); + return true; + } catch (const Ort::Exception& e) { + if (qEnvironmentVariableIsSet("QTMESH_ONNX_DEBUG")) + fprintf(stderr, "[onnx] CoreML EP failed: %s\n", e.what()); + } +#elif defined(_WIN32) && !defined(__MINGW32__) + try { + so.AppendExecutionProvider("DML", {}); + return true; + } catch (const Ort::Exception& e) { + if (qEnvironmentVariableIsSet("QTMESH_ONNX_DEBUG")) + fprintf(stderr, "[onnx] DirectML EP failed: %s\n", e.what()); + } +#elif defined(__linux__) + try { + OrtCUDAProviderOptions cuda{}; + cuda.device_id = 0; + so.AppendExecutionProvider_CUDA(cuda); + if (qEnvironmentVariableIsSet("QTMESH_ONNX_DEBUG")) + fprintf(stderr, "[onnx] CUDA execution provider registered\n"); + return true; + } catch (const Ort::Exception& e) { + if (qEnvironmentVariableIsSet("QTMESH_ONNX_DEBUG")) + fprintf(stderr, "[onnx] CUDA EP failed: %s\n", e.what()); + } +#endif + return false; +} +#endif // ENABLE_ONNX diff --git a/src/OnnxRuntimeSettings.h b/src/OnnxRuntimeSettings.h new file mode 100644 index 000000000..a9ecb95e8 --- /dev/null +++ b/src/OnnxRuntimeSettings.h @@ -0,0 +1,91 @@ +#ifndef ONNXRUNTIMESETTINGS_H +#define ONNXRUNTIMESETTINGS_H + +#include +#include +#include + +#ifdef ENABLE_ONNX +#include +#endif + +// Central ONNX Runtime session preferences (issue #408 follow-up): thread pool +// sizing, spinning, and optional GPU execution providers. Persisted under +// QSettings ai/onnxPreferGpu and surfaced in AI Settings → Settings tab. +class OnnxRuntimeSettings : public QObject +{ + Q_OBJECT + QML_ELEMENT + QML_SINGLETON + + Q_PROPERTY(bool onnxAvailable READ onnxAvailable CONSTANT) + Q_PROPERTY(bool preferGpu READ preferGpu WRITE setPreferGpu NOTIFY settingsChanged) + Q_PROPERTY(bool gpuProviderBundled READ gpuProviderBundled NOTIFY gpuStatusChanged) + Q_PROPERTY(bool gpuProviderReady READ gpuProviderReady NOTIFY gpuStatusChanged) + Q_PROPERTY(QString gpuProviderNote READ gpuProviderNote NOTIFY settingsChanged) + +public: + static OnnxRuntimeSettings* instance(); + static OnnxRuntimeSettings* qmlInstance(QQmlEngine* engine, QJSEngine* scriptEngine); + + bool onnxAvailable() const; + + bool preferGpu() const; + void setPreferGpu(bool value); + + QString gpuProviderNote() const; + + // True when this binary was built against the GPU ONNX Runtime package AND + // the CUDA provider .so was copied next to the executable. + bool gpuProviderBundled() const; + // True when bundled GPU EP exists and runtime deps (cuDNN 9, CUDA 12) load. + bool gpuProviderReady() const; + + Q_INVOKABLE void refreshGpuProviderStatus(); + +#ifdef ENABLE_ONNX + enum class CoreMlStyle { Legacy, MlProgram }; + + struct SessionConfig { + bool reserveUiThreadCore = true; + bool allowSpinning = false; + bool appendGpu = true; + CoreMlStyle coreMlStyle = CoreMlStyle::Legacy; + }; + + // Apply graph optimization, thread count, spinning, and (when preferGpu()) + // the best available GPU EP for this platform/build. + static void configureSessionOptions(Ort::SessionOptions& so); + static void configureSessionOptions(Ort::SessionOptions& so, + const SessionConfig& cfg); + + // Best-effort GPU EP append; returns true when one was registered. + static bool tryAppendGpuExecutionProvider(Ort::SessionOptions& so, + CoreMlStyle style = CoreMlStyle::Legacy); +#endif + + static bool defaultPreferGpu(); + + // Must run once before the first Ort::Session (main + CLI entry). On Linux + // GPU builds this prepends cuDNN/cuBLAS dirs to LD_LIBRARY_PATH. + static void prepareRuntimeEnvironment(); + + Q_INVOKABLE void loadSettings(); + Q_INVOKABLE void saveSettings(); + +signals: + void settingsChanged(); + void gpuStatusChanged(); + +private: + explicit OnnxRuntimeSettings(QObject* parent = nullptr); + + void updateGpuProviderNoteLocked(); + + bool m_preferGpu = defaultPreferGpu(); + bool m_gpuProviderBundled = false; + bool m_gpuProviderReady = false; + QString m_gpuProviderNote; +}; + +#endif // ONNXRUNTIMESETTINGS_H diff --git a/src/OnnxRuntimeSettings_test.cpp b/src/OnnxRuntimeSettings_test.cpp new file mode 100644 index 000000000..c3a9f3431 --- /dev/null +++ b/src/OnnxRuntimeSettings_test.cpp @@ -0,0 +1,32 @@ +#include + +#include "OnnxRuntimeSettings.h" + +#include +#include + +TEST(OnnxRuntimeSettings, PreferGpuPersists) +{ + QSettings settings; + settings.remove("ai/onnxPreferGpu"); + + OnnxRuntimeSettings* ort = OnnxRuntimeSettings::instance(); + ort->loadSettings(); + + const bool initial = ort->preferGpu(); + ort->setPreferGpu(!initial); + EXPECT_EQ(ort->preferGpu(), !initial); + + OnnxRuntimeSettings* reloaded = OnnxRuntimeSettings::instance(); + reloaded->loadSettings(); + EXPECT_EQ(reloaded->preferGpu(), !initial); + + ort->setPreferGpu(initial); +} + +TEST(OnnxRuntimeSettings, GpuProviderNoteNonEmpty) +{ + OnnxRuntimeSettings::instance()->refreshGpuProviderStatus(); + const QString note = OnnxRuntimeSettings::instance()->gpuProviderNote(); + EXPECT_FALSE(note.isEmpty()); +} diff --git a/src/PbrMapSynth.cpp b/src/PbrMapSynth.cpp index fd8826081..08ad1f467 100644 --- a/src/PbrMapSynth.cpp +++ b/src/PbrMapSynth.cpp @@ -1,4 +1,5 @@ #include "PbrMapSynth.h" +#include "OnnxRuntimeSettings.h" #include #include @@ -298,16 +299,8 @@ std::vector runTiledModel(const QImage& albedoIn, const QString& modelPat try { Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_pbr"); Ort::SessionOptions so; + OnnxRuntimeSettings::configureSessionOptions(so); so.SetIntraOpNumThreads(1); - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); -#ifdef __APPLE__ - // CoreML EP — best-effort; fall back to CPU if it can't be appended. - try { - std::unordered_map coremlOpts; - so.AppendExecutionProvider("CoreML", coremlOpts); - } catch (const Ort::Exception&) { - } -#endif #ifdef _WIN32 std::wstring wpath = modelPath.toStdWString(); Ort::Session session(env, wpath.c_str(), so); diff --git a/src/SkinTokensPredictor.cpp b/src/SkinTokensPredictor.cpp index e1dba0606..ca418c875 100644 --- a/src/SkinTokensPredictor.cpp +++ b/src/SkinTokensPredictor.cpp @@ -1,5 +1,6 @@ #include "SkinTokensPredictor.h" #include "ModelDownloader.h" +#include "OnnxRuntimeSettings.h" #include #include @@ -19,7 +20,6 @@ #include #include #include -#include #ifdef ENABLE_ONNX #include @@ -594,13 +594,7 @@ SkinTokensPredictor::Result SkinTokensPredictor::predict( try { Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_skintokens"); Ort::SessionOptions so; - // hardware_concurrency() may legally return 0 — guard before the -1 - // (0u - 1 underflows to a UINT_MAX-sized thread pool request). - const unsigned hc = std::thread::hardware_concurrency(); - so.SetIntraOpNumThreads(hc > 1 ? static_cast(hc - 1) : 1); - // Don't busy-spin the pool between ops — spinning starves the GUI - // render loop even though the compute runs on a worker thread. - so.AddConfigEntry("session.intra_op.allow_spinning", "0"); + OnnxRuntimeSettings::configureSessionOptions(so); auto openSession = [&](const char* file) -> Ort::Session { const QString p = modelDir() + QLatin1Char('/') + QLatin1String(file); @@ -705,17 +699,17 @@ SkinTokensPredictor::Result SkinTokensPredictor::predict( "SkinTokens: VAE conditioning failed.")); dbg("stage 3: embed skeleton"); - // ── (3) embed the skeleton ids ────────────────────────────── + // ── (3) embed the skeleton ids (reuse the embed session for decode) ─ std::vector skelEmbeds; + Ort::Session embed = openSession("embed.onnx"); { - Ort::Session s = openSession("embed.onnx"); std::vector ids = skelIds; const int64_t shp[2] = { 1, int64_t(ids.size()) }; std::vector ins; ins.push_back(Ort::Value::CreateTensor( mem, ids.data(), ids.size(), shp, 2)); std::vector eshape; - skelEmbeds = runSimple(s, ins, eshape); + skelEmbeds = runSimple(embed, ins, eshape); if (skelEmbeds.empty() || eshape.size() != 3 || eshape[2] != hidden) return failResult(QStringLiteral( @@ -724,10 +718,8 @@ SkinTokensPredictor::Result SkinTokensPredictor::predict( dbg("stage 4: decoder open"); // ── (4) decoder prefix + greedy skin decode ───────────────── - Ort::Session dec = openSession("decoder.onnx"); + Ort::Session dec = openSession("decoder.onnx"); dbg("stage 4a: decoder session created"); - Ort::Session embed = openSession("embed.onnx"); - dbg("stage 4a2: embed session created"); const size_t decIn = dec.GetInputCount(); std::vector dInHold; diff --git a/src/TextureUpscaler.cpp b/src/TextureUpscaler.cpp index 89da46ed2..7819c8919 100644 --- a/src/TextureUpscaler.cpp +++ b/src/TextureUpscaler.cpp @@ -1,4 +1,5 @@ #include "TextureUpscaler.h" +#include "OnnxRuntimeSettings.h" #include "PbrMapSynth.h" // reuse toNCHW / nchwToRgb #include @@ -148,20 +149,10 @@ Result upscale(const QImage& srcIn, const QString& modelPath, const Options& opt try { Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_upscale"); Ort::SessionOptions so; - // Real-ESRGAN is CPU-heavy; use all cores for intra-op parallelism - // (was pinned to 1, which made large textures take hours). Leave one - // core free so the UI/host stays responsive. ONNX Runtime treats 0 as - // "let ORT decide", but we cap explicitly for predictability. + OnnxRuntimeSettings::configureSessionOptions(so); const unsigned hw = std::thread::hardware_concurrency(); const int threads = (hw > 1) ? static_cast(hw - 1) : 1; so.SetIntraOpNumThreads(threads); - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); -#ifdef __APPLE__ - try { - std::unordered_map coreml; - so.AppendExecutionProvider("CoreML", coreml); - } catch (const Ort::Exception&) {} -#endif #ifdef _WIN32 std::wstring wp = modelPath.toStdWString(); Ort::Session session(env, wp.c_str(), so); diff --git a/src/UniRigPredictor.cpp b/src/UniRigPredictor.cpp index 5e58d9fb0..e3a0e6a1a 100644 --- a/src/UniRigPredictor.cpp +++ b/src/UniRigPredictor.cpp @@ -1,5 +1,6 @@ #include "UniRigPredictor.h" #include "ModelDownloader.h" +#include "OnnxRuntimeSettings.h" #include #include @@ -559,6 +560,17 @@ bool UniRigPredictor::modelsPresent() && QFileInfo::exists(embedModelPath()); } +int UniRigPredictor::sampleBudgetForMesh(int vertexCount, int requested) +{ + if (requested > 0) + return std::clamp(requested, 4096, kNumSamples); + if (vertexCount <= 50000) + return kNumSamples; + if (vertexCount <= 200000) + return kNumSamples / 2; + return kNumSamples / 4; +} + QString UniRigPredictor::ensureModelBlocking() { #ifndef ENABLE_ONNX @@ -826,31 +838,20 @@ UniRigPredictor::Result UniRigPredictor::predict( nverts[3*i+2] = static_cast(p[2]); } - // --- (1b) Surface-sample up to kNumSamples points + normals -------------- - SampledCloud cloud = sampleSurface(nverts, vertexCount, indices, indexCount, kNumSamples); + // --- (1b) Surface-sample up to the encoder budget ----------------------- + const int sampleBudget = + sampleBudgetForMesh(vertexCount, opts.numSamples); + SampledCloud cloud = + sampleSurface(nverts, vertexCount, indices, indexCount, sampleBudget); if (cloud.count < 1) return failResult(QStringLiteral("UniRig: failed to sample mesh surface.")); try { Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_unirig"); - Ort::SessionOptions so; - so.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); - // Leave one core for the UI and stop ORT's pool from busy-spinning — - // the default (all cores, spin-wait) starves the render loop for the - // whole multi-minute decode and the viewport appears frozen. - // hardware_concurrency() may legally return 0 — guard before the -1 - // (0u - 1 underflows to a UINT_MAX-sized thread pool request). - const unsigned hc = std::thread::hardware_concurrency(); - so.SetIntraOpNumThreads(hc > 1 ? static_cast(hc - 1) : 1); - so.AddConfigEntry("session.intra_op.allow_spinning", "0"); -#ifdef __APPLE__ - try { - std::unordered_map coremlOpts; - so.AppendExecutionProvider("CoreML", coremlOpts); - } catch (const Ort::Exception&) {} -#endif + Ort::AllocatorWithDefaultOptions alloc; + Ort::MemoryInfo mem = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); - auto openSession = [&](const QString& path) -> Ort::Session { + auto openSession = [&](Ort::SessionOptions& so, const QString& path) -> Ort::Session { #ifdef _WIN32 std::wstring wpath = path.toStdWString(); return Ort::Session(env, wpath.c_str(), so); @@ -860,29 +861,23 @@ UniRigPredictor::Result UniRigPredictor::predict( #endif }; - Ort::Session encoder = openSession(encoderModelPath); - Ort::Session decoder = openSession(decoderModelPath); - Ort::Session embed = openSession(embedModelPath); - Ort::AllocatorWithDefaultOptions alloc; - Ort::MemoryInfo mem = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); - - // embed.onnx: input_ids[1,S] (int64) -> token_embeds[1,S,hidden]. Turns - // the seed tokens + each generated token into the inputs_embeds the - // decoder consumes (the exported decoder takes embeddings, not ids — - // UniRig conditions the LM by prepending the encoder latents as embeds). - auto embedTokens = [&](const std::vector& ids) -> std::vector { - const int64_t shp[2] = {1, static_cast(ids.size())}; - Ort::Value in = Ort::Value::CreateTensor( - mem, const_cast(ids.data()), ids.size(), shp, 2); - auto inName = embed.GetInputNameAllocated(0, alloc); - auto outName = embed.GetOutputNameAllocated(0, alloc); - const char* inN[] = { inName.get() }; - const char* outN[] = { outName.get() }; - auto out = embed.Run(Ort::RunOptions{nullptr}, inN, &in, 1, outN, 1); - const float* d = out[0].GetTensorData(); - const size_t cnt = (size_t)out[0].GetTensorTypeAndShapeInfo().GetElementCount(); - return std::vector(d, d + cnt); - }; + // The three ONNX files total ~1.4 GB. Open the encoder alone first and + // release it before loading the decoder + embed pair — holding all three + // at once was enough to OOM on large meshes when SkinTokens also runs. + std::vector latents; + int64_t numLatents = 0; + int64_t hidden = 0; + { + Ort::SessionOptions encSo; + OnnxRuntimeSettings::configureSessionOptions(encSo); + Ort::Session encoder = openSession(encSo, encoderModelPath); + if (qEnvironmentVariableIsSet("QTMESH_ONNX_DEBUG")) { + const auto eps = Ort::GetAvailableProviders(); + fprintf(stderr, "[unirig] ORT available providers:"); + for (const auto& ep : eps) + fprintf(stderr, " %s", ep.c_str()); + fprintf(stderr, "\n"); + } // ===================================================================== // (2) ENCODER: pc[1,N,3] + feats[1,N,3] (normals) → latents prefix. @@ -943,7 +938,6 @@ UniRigPredictor::Result UniRigPredictor::predict( // the encoder graph's trailing nn.Linear). We feed it as the decoder's // `inputs_embeds` prefix. const float* latentData = nullptr; - int64_t numLatents = 0, hidden = 0; for (size_t i = 0; i < encOuts.size(); ++i) { if (!encOuts[i].IsTensor()) continue; auto info = encOuts[i].GetTensorTypeAndShapeInfo(); @@ -961,8 +955,32 @@ UniRigPredictor::Result UniRigPredictor::predict( "UniRig: encoder produced no latent prefix.")); // Own the latent bytes (encOuts is reused/invalidated as we proceed). - std::vector latents( + latents.assign( latentData, latentData + static_cast(numLatents) * hidden); + } // encoder session released + + Ort::SessionOptions decSo; + OnnxRuntimeSettings::configureSessionOptions(decSo); + Ort::Session decoder = openSession(decSo, decoderModelPath); + Ort::Session embed = openSession(decSo, embedModelPath); + + // embed.onnx: input_ids[1,S] (int64) -> token_embeds[1,S,hidden]. Turns + // the seed tokens + each generated token into the inputs_embeds the + // decoder consumes (the exported decoder takes embeddings, not ids — + // UniRig conditions the LM by prepending the encoder latents as embeds). + auto embedTokens = [&](const std::vector& ids) -> std::vector { + const int64_t shp[2] = {1, static_cast(ids.size())}; + Ort::Value in = Ort::Value::CreateTensor( + mem, const_cast(ids.data()), ids.size(), shp, 2); + auto inName = embed.GetInputNameAllocated(0, alloc); + auto outName = embed.GetOutputNameAllocated(0, alloc); + const char* inN[] = { inName.get() }; + const char* outN[] = { outName.get() }; + auto out = embed.Run(Ort::RunOptions{nullptr}, inN, &in, 1, outN, 1); + const float* d = out[0].GetTensorData(); + const size_t cnt = (size_t)out[0].GetTensorTypeAndShapeInfo().GetElementCount(); + return std::vector(d, d + cnt); + }; // ===================================================================== // (3) GREEDY CONSTRAINED AUTOREGRESSIVE DECODE with a manual KV-cache. diff --git a/src/UniRigPredictor.h b/src/UniRigPredictor.h index 9cf0757d3..863a7d9d8 100644 --- a/src/UniRigPredictor.h +++ b/src/UniRigPredictor.h @@ -60,6 +60,9 @@ class UniRigPredictor { // Coarse cap on emitted joints. UniRig's decode terminates on the EOS // token; this is a safety bound (decode also stops at 2048 tokens). int maxJoints = 256; + // Surface sample count for the encoder (0 = auto — full budget on small + // meshes, scaled down on very large ones to reduce encoder peak RAM). + int numSamples = 0; // Up axis: 0=X, 1=Y, 2=Z (default +Y). UniRig is trained +Y-up; a // non-Y up axis is rotated into +Y for inference and back out after. int upAxis = 1; @@ -97,6 +100,10 @@ class UniRigPredictor { // True when all three model files already exist on disk (no download needed). static bool modelsPresent(); + // Encoder surface-sample budget. `requested` > 0 clamps to [4096, 65536]; + // 0 picks an automatic budget that scales down on very large meshes. + static int sampleBudgetForMesh(int vertexCount, int requested = 0); + // Ensure ALL THREE models exist on disk, downloading whichever is missing on // first use (blocks via a local event loop, like // AIAssistManager::ensureModelBlocking). Returns the encoder path on diff --git a/src/UniRigPredictor_test.cpp b/src/UniRigPredictor_test.cpp index 92c6d1d68..53100805c 100644 --- a/src/UniRigPredictor_test.cpp +++ b/src/UniRigPredictor_test.cpp @@ -451,6 +451,16 @@ TEST(UniRigPredictor, LabelsAnatomicallyResolveCanonicalJoints) EXPECT_EQ(rh, 9) << "right (+X) hand index " << rh; } +TEST(UniRigPredictor, SampleBudgetScalesForLargeMeshes) +{ + EXPECT_EQ(UniRigPredictor::sampleBudgetForMesh(1000), 65536); + EXPECT_EQ(UniRigPredictor::sampleBudgetForMesh(100000), 32768); + EXPECT_EQ(UniRigPredictor::sampleBudgetForMesh(500000), 16384); + EXPECT_EQ(UniRigPredictor::sampleBudgetForMesh(1000, 12000), 12000); + EXPECT_EQ(UniRigPredictor::sampleBudgetForMesh(1000, 1000), 4096); + EXPECT_EQ(UniRigPredictor::sampleBudgetForMesh(1000, 999999), 65536); +} + TEST(UniRigPredictor, EnsureModelBlockingHonoursNoDownloadGuard) { // With QTMESH_UNIRIG_NO_DOWNLOAD set (and no model on disk) ensureModelBlocking diff --git a/src/main.cpp b/src/main.cpp index 0eab80313..eeacd6691 100755 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,6 +21,7 @@ #include "MaterialEditorQML.h" #include "QMLMaterialHighlighter.h" #include "LLMManager.h" +#include "OnnxRuntimeSettings.h" #include "AIModelCatalog.h" #ifdef ENABLE_STABLE_DIFFUSION #include "SDManager.h" @@ -33,6 +34,9 @@ #include "CLIPipeline.h" #include "AppConsoleLog.h" #include "AppLaunchHandler.h" +#ifdef ENABLE_MOCAP +#include "Mocap/MocapCameraHints.h" +#endif #ifdef ENABLE_AUTO_UPDATER #include "updater/UpdaterController.h" #include "updater/UpdaterTelemetry.h" @@ -120,6 +124,7 @@ int main(int argc, char *argv[]) QCoreApplication::setOrganizationDomain("none"); QCoreApplication::setApplicationName("QtMeshEditor"); QCoreApplication::setApplicationVersion(QTMESHEDITOR_VERSION); + OnnxRuntimeSettings::prepareRuntimeEnvironment(); // Initialize Sentry using stored consent (no dialog in headless mode) SentryReporter::initialize(); @@ -156,10 +161,18 @@ int main(int argc, char *argv[]) // initialises, so they belong here, ahead of QApplication. qputenv("QSG_RHI_BACKEND", "software"); qputenv("QT_QUICK_BACKEND", "software"); +#ifdef ENABLE_MOCAP + // Before QApplication: Qt Multimedia's FFmpeg backend probes VA-API during + // integration init; on some Linux/NVIDIA setups that crashes inside + // QMediaDevices::videoInputs() when the Performance Capture panel expands. + MocapCameraHints::ensureMultimediaBackendSafe(); +#endif QQuickWindow::setGraphicsApi(QSGRendererInterface::Software); QApplication a(argc, argv); + OnnxRuntimeSettings::prepareRuntimeEnvironment(); + // Capture qDebug/qWarning/etc. from the rest of startup into the in-app console // (MainWindow attaches and drains the backlog when its console exists). AppConsoleLog::install(); @@ -228,6 +241,11 @@ int main(int argc, char *argv[]) return LLMManager::qmlInstance(engine, scriptEngine); }); + qmlRegisterSingletonType("MaterialEditorQML", 1, 0, "OnnxRuntimeSettings", + [](QQmlEngine *engine, QJSEngine *scriptEngine) -> QObject* { + return OnnxRuntimeSettings::qmlInstance(engine, scriptEngine); + }); + // Register ModelDownloader singleton for QML qmlRegisterSingletonType("MaterialEditorQML", 1, 0, "ModelDownloader", [](QQmlEngine *engine, QJSEngine *scriptEngine) -> QObject* { diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index 11ce620c1..15bd5d575 100755 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -101,6 +101,7 @@ #include "HDR/HdrViewportController.h" #include "HDR/HdrBundledLibrary.h" #include "LLMManager.h" +#include "OnnxRuntimeSettings.h" #ifdef ENABLE_ONNX #include "AIAssistManager.h" #endif @@ -5633,6 +5634,13 @@ void MainWindow::on_actionMaterial_Editor_triggered() return LLMManager::qmlInstance(engine, scriptEngine); }); + qmlRegisterSingletonType("MaterialEditorQML", 1, 0, "OnnxRuntimeSettings", + [](QQmlEngine *engine, QJSEngine *scriptEngine) -> QObject* { + Q_UNUSED(engine) + Q_UNUSED(scriptEngine) + return OnnxRuntimeSettings::qmlInstance(engine, scriptEngine); + }); + // Register ModelDownloader singleton for QML qmlRegisterSingletonType("MaterialEditorQML", 1, 0, "ModelDownloader", [](QQmlEngine *engine, QJSEngine *scriptEngine) -> QObject* { From c3267bb9a35e705b2af4121c04b4991866964068 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 3 Aug 2026 23:49:22 -0400 Subject: [PATCH 03/11] fix(mocap): invert head yaw for mirrored webcam preview FaceCap head rotation needs yaw correction on selfie/webcam feeds while body retargeting uses landmark directions without L/R swapping. Co-authored-by: Cursor --- docs/MOCAP.md | 5 +++-- src/Mocap/MocapController.cpp | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/MOCAP.md b/docs/MOCAP.md index ba4111451..694aadf61 100644 --- a/docs/MOCAP.md +++ b/docs/MOCAP.md @@ -135,8 +135,9 @@ startup (software decode only; fine for live webcam preview). - Single person per frame; the highest-scoring detection wins. - Head pose is camera-relative — walking around the camera reads as head - rotation. Keep the camera static. Up/down (pitch) is corrected for - Mixamo-style rigs. + rotation. Keep the camera static. Up/down (pitch) and left/right (yaw) are + corrected for Mixamo-style rigs and mirrored webcam previews (body uses + landmark directions separately; no mirror-L/R toggle). - Body retargeting uses MediaPipe landmark directions (same geometry as the PoseIK debug overlay) to aim skeleton bones — no mirror-L/R toggle. - Body root is locked (no root motion); some foot slide is expected. Live diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index 375a6a331..c0f36cd4c 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -1013,6 +1013,9 @@ void MocapController::onSample(const FaceSample& sample, sample.headRotation[1], sample.headRotation[2]); Ogre::Quaternion delta = current * d->neutral.Inverse(); delta = MocapPoseFix::invertCameraPitchDelta(delta); + // Webcam preview is mirrored; FaceCap head yaw is opposite the rig + // unless corrected. Body uses landmark directions (no L/R swap). + delta = MocapPoseFix::invertCameraYawDelta(delta); const Ogre::Quaternion local = d->headBindWorld.Inverse() * delta * d->headBindWorld; Ogre::SkeletonInstance* skel = entity->getSkeleton(); From f10e91eb81dd9884e7d491ac27c60a6a6510f05b Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 3 Aug 2026 23:51:02 -0400 Subject: [PATCH 04/11] chore(mocap): generalize Mixamo-specific comments in retarget path Use vendor-neutral wording in live mocap, BodyRetargeter, docs, and debug UI while keeping factual mixamorig examples where they describe bone naming. Co-authored-by: Cursor --- docs/MOCAP.md | 4 ++-- qml/PropertiesPanel.qml | 4 ++-- src/AnimationMerger.cpp | 6 +++--- src/AnimationMerger_test.cpp | 6 +++--- src/Mocap/MocapController.cpp | 6 +++--- src/Mocap/MocapPoseFix.h | 7 +++---- src/Mocap/PoseIKSolver.cpp | 4 ++-- 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/docs/MOCAP.md b/docs/MOCAP.md index 694aadf61..462aeee1e 100644 --- a/docs/MOCAP.md +++ b/docs/MOCAP.md @@ -57,7 +57,7 @@ take as a clip (status line shows the result; Ctrl+Z discards it). - **Body capture** needs a **humanoid skeleton** resolving at least half of the 22 canonical roles (hips/spine/neck/head, both arms, both legs — - Mixamo and most generic naming conventions resolve). Unrigged meshes: run + standard humanoid bone names resolve). Unrigged meshes: run `qtmesh rig --skeleton humanoid --skin` first. The root stays locked to the standing pose (v1 accepts some foot slide). Body limbs calibrate on the first visible frame too — start preview with arms in a natural rest @@ -136,7 +136,7 @@ startup (software decode only; fine for live webcam preview). - Single person per frame; the highest-scoring detection wins. - Head pose is camera-relative — walking around the camera reads as head rotation. Keep the camera static. Up/down (pitch) and left/right (yaw) are - corrected for Mixamo-style rigs and mirrored webcam previews (body uses + corrected for typical humanoid rigs and mirrored webcam previews (body uses landmark directions separately; no mirror-L/R toggle). - Body retargeting uses MediaPipe landmark directions (same geometry as the PoseIK debug overlay) to aim skeleton bones — no mirror-L/R toggle. diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 6fa3fa6f8..239348c58 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -2461,8 +2461,8 @@ Rectangle { font.pixelSize: 9 text: "Debug (beside character): cyan = MediaPipe landmarks, " + "yellow = PoseIK 22-bone FK. If cyan matches you but " - + "Mixamo does not, retarget is wrong; if cyan is wrong, " - + "capture or lighting is wrong." + + "the skinned mesh does not, retarget is wrong; if cyan " + + "is wrong, capture or lighting is wrong." } // Video-file source — the path for macOS where the camera is diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 6c8dd686a..b62bad8a2 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1574,7 +1574,7 @@ BodyRetargeter::BodyRetargeter(Ogre::Skeleton* skel, bool yaw180) MotionInbetween::canonicalIndexForBone(QString::fromStdString( skel->getBone(static_cast(i))->getName())); // Pose-ik mocap: anatomical name→role only (NO CMU handedness swap — that - // swap is for BVH/library clips and mirrors live limb motion on Mixamo). + // swap is for BVH/library clips and would mirror live limb motion). d->tb = readTargetBindFrame(skel, d->boneToCanon); d->canonDup.assign(static_cast(d->Jc), 0); @@ -1641,8 +1641,8 @@ BodyRetargeter::BodyRetargeter(Ogre::Skeleton* skel, bool yaw180) d->haveAnyStand = true; } } - // Rig bone rests identity? (Mixamo yes → Mc heuristic valid; UniRig/template - // no → skip Mc, use raw delta on the standing pose.) + // Rig bone rests identity? (common FBX humanoids → Mc heuristic valid; + // UniRig/template no → skip Mc, use raw delta on the standing pose.) for (int i = 0; i < d->nBones && d->restsAreIdentity; ++i) { Ogre::Bone* b = skel->getBone(static_cast(i)); if (!b->getInitialOrientation().equals(Ogre::Quaternion::IDENTITY, diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index 35b0df953..e47f5602b 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -964,7 +964,7 @@ TEST_F(AnimationMergerTest, ExtractCanonicalClipsRejectsNonHumanoid) TEST_F(AnimationMergerTest, ExtractCanonicalClipsSamplesWorldFrame) { - // Minimal humanoid: Mixamo-style names resolve hip/head/lhip/rhip roles. + // Minimal humanoid: common bone names resolve hip/head/lhip/rhip roles. auto skel = Ogre::SkeletonManager::getSingleton().create( "extract_skel", Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); auto* hips = skel->createBone("Hips", 0); @@ -1479,7 +1479,7 @@ TEST_F(AnimationMergerTest, TwistTransportCarriesBoneRoll) TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) { - // Rig WITH clavicles (Mixamo "Shoulder" → collar roles 6/10). + // Rig WITH clavicles ("Shoulder" bone name → collar roles 6/10). auto skelRes = Ogre::SkeletonManager::getSingleton().create( "twist_collar_skel", Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); @@ -1578,7 +1578,7 @@ TEST_F(AnimationMergerTest, VerticalDescentLowersRootDescentOnly) }; // A FULL humanoid bone set — applyMotionClip rejects a rig that resolves // fewer than ~half of the 22 canonical roles ("not a humanoid rig"), so the - // minimal 9-bone skeleton isn't enough. Mixamo-style names map onto the + // minimal 9-bone skeleton isn't enough. Standard humanoid names map onto the // canonical roles (hip/spine/chest/neck/head, collar/shoulder/elbow/hand, // upleg/leg/foot per side). // Hips at world Y=1.0; each leg chain reaches down to Y=0 → leg length 1.0. diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index c0f36cd4c..497f9eb5a 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -301,7 +301,7 @@ struct MocapController::Impl { // AnimationStateSet or manual bones are dirty. Mocap samples arrive on the Qt // event loop (~30 Hz) while the render loop runs faster — without a per-frame // refresh the mesh stays frozen in bind pose even though Bone::setOrientation - // succeeded (Mixamo: debug overlay moves, skin does not). + // succeeded (debug overlay can move while skin does not). struct SkinningFrameListener : public Ogre::FrameListener { Impl* impl = nullptr; @@ -1029,8 +1029,8 @@ void MocapController::onSample(const FaceSample& sample, } } - // live drive — body (independent of face confidence). PoseIK quats are - // aligned onto Mixamo via BodyRetargeter — same math as applyMotionClip. + // live drive — body (independent of face confidence). Landmark directions + // retarget onto the selection skeleton via BodyRetargeter. if (body.valid && d->bodyRetargeter && d->bodyRetargeter->valid() && entity->hasSkeleton()) { std::array, 22> canonQuats{}; diff --git a/src/Mocap/MocapPoseFix.h b/src/Mocap/MocapPoseFix.h index 07f12cdf4..301a18d29 100644 --- a/src/Mocap/MocapPoseFix.h +++ b/src/Mocap/MocapPoseFix.h @@ -62,9 +62,8 @@ inline Ogre::Quaternion kPoseToSkeletonYawPi() return Ogre::Quaternion(Ogre::Degree(180), Ogre::Vector3::UNIT_Y); } -// Mixamo Head (and similar rigs) often need camera-frame pitch inverted while -// yaw stays correct. For unit quaternions, negating the X component mirrors -// rotation about +X (pitch) while leaving yaw (Y) unchanged to first order. +// Humanoid head bones often need camera-frame pitch inverted while yaw is +// corrected separately for mirrored webcam previews. inline Ogre::Quaternion invertCameraPitchDelta(const Ogre::Quaternion& delta) { return Ogre::Quaternion(delta.w, -delta.x, delta.y, delta.z); @@ -77,7 +76,7 @@ inline Ogre::Quaternion invertCameraYawDelta(const Ogre::Quaternion& delta) } // Legacy 180°-yaw bridge for CMU library clips whose bind frame differs. -// Live PoseIK mocap does NOT use these — PoseIK and Mixamo both face +Z with +// Live PoseIK mocap does NOT use these — canonical +Z-facing rigs with // anatomical L/R bone names; a yaw flip made the body drive backward. inline Ogre::Vector3 poseDirectionToSkeleton(const Ogre::Vector3& v) { diff --git a/src/Mocap/PoseIKSolver.cpp b/src/Mocap/PoseIKSolver.cpp index 2a2dd5cde..58a64c495 100644 --- a/src/Mocap/PoseIKSolver.cpp +++ b/src/Mocap/PoseIKSolver.cpp @@ -230,8 +230,8 @@ FrameResult Solver::solveFrame(const float* world, const float* visibility, // Canonicalize MediaPipe's frame (+x subject-left, +y down, +z toward camera) // into the CMU/canonical rig frame (+Y up, +Z forward, LEFT at +X). Flip Y - // and keep Z (subject faces +Z); do NOT negate X — (-x,-y,+z) mirrored L/R - // on Mixamo-style rigs whose left bones already sit at +X. + // and keep Z (subject faces +Z); do NOT negate X — (-x,-y,+z) would swap + // anatomical left/right on rigs whose left bones already sit at +X. Vec3 p[kLandmarkCount]; std::array, kLandmarkCount> canon{}; canonicalizeMediaPipeWorld(world, canon); From a21a840e511e9073e2ebd19bd6e7e222dfc4bcc7 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 4 Aug 2026 08:59:47 -0400 Subject: [PATCH 05/11] fix(mocap): address CI link errors and P1 review items Wire OnnxRuntimeSettings and mocap debug sources into qtmesh_test_common, propagate parent world transforms for unmapped bones, bake body recordings via landmark retarget, and clean up skinning listener/head manual-control edge cases. Co-authored-by: Cursor --- src/AnimationMerger.cpp | 11 +- src/Mocap/MocapBodyDriveDebug.cpp | 3 + src/Mocap/MocapController.cpp | 44 ++++-- src/Mocap/MocapPoseDebugOverlay.cpp | 5 +- src/Mocap/MocapPoseFix.h | 5 +- src/Mocap/MocapRecorder.cpp | 174 +++++++++++++++++++++++- src/Mocap/MocapRecorder.h | 9 ++ src/commands/RecordMocapClipCommand.cpp | 18 ++- src/commands/RecordMocapClipCommand.h | 6 + tests/CMakeLists.txt | 3 + 10 files changed, 251 insertions(+), 27 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index b62bad8a2..c4706d211 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1791,11 +1791,6 @@ BodyRetargeter::evaluateFrame( const Ogre::Quaternion CtInv = tb.Ct.Inverse(); std::vector W(static_cast(nBones)); for (int i : tb.order) { - const int c = d->boneToCanon[static_cast(i)]; - if (c < 0 || c >= Jc) - continue; - if (skipRolesMask & (1u << static_cast(c))) - continue; const Ogre::Quaternion base = (d->haveStand[static_cast(i)] ? d->standLocal[static_cast(i)] @@ -1804,6 +1799,12 @@ BodyRetargeter::evaluateFrame( const Ogre::Quaternion Wp = (pi >= 0) ? W[static_cast(pi)] : Ogre::Quaternion::IDENTITY; + const int c = d->boneToCanon[static_cast(i)]; + if (c < 0 || c >= Jc + || (skipRolesMask & (1u << static_cast(c)))) { + W[static_cast(i)] = Wp * base; + continue; + } Ogre::Quaternion local; if (c == 0) { local = base; diff --git a/src/Mocap/MocapBodyDriveDebug.cpp b/src/Mocap/MocapBodyDriveDebug.cpp index 91b995173..13ce62d7c 100644 --- a/src/Mocap/MocapBodyDriveDebug.cpp +++ b/src/Mocap/MocapBodyDriveDebug.cpp @@ -10,11 +10,14 @@ #include #include +#include #include #include #include #include +#include + namespace { using Vec3 = std::array; diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index 497f9eb5a..06d52b692 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -142,6 +142,14 @@ struct MocapMetaTypeRegistrar { }; const MocapMetaTypeRegistrar mocapMetaTypeRegistrar; +void unregisterSkinningListener(MocapController::Impl* impl) +{ + if (!impl || !impl->skinningListener) + return; + Ogre::Root::getSingleton().removeFrameListener(impl->skinningListener.get()); + impl->skinningListener->impl = nullptr; +} + struct BodyDriveBone { int role = -1; std::string boneName; @@ -732,8 +740,12 @@ bool MocapController::beginPreviewWithLiveSource( d->bodyRetargeter.reset(); d->bodyBones.clear(); } else { + const std::string headBoneStd = + headBoneDrive ? d->headBone.toStdString() : std::string{}; for (unsigned short i = 0; i < skel->getNumBones(); ++i) { Ogre::Bone* bone = skel->getBone(i); + if (headBoneDrive && bone->getName() == headBoneStd) + continue; BodyManualBoneSnapshot snap; snap.boneName = bone->getName(); snap.bindLocal = bone->getOrientation(); @@ -795,6 +807,7 @@ bool MocapController::beginPreviewWithLiveSource( QString error; if (!d->camera->open(&error)) { d->camera.reset(); + unregisterSkinningListener(d.get()); restoreEntityState(); setStatusMessage(error); emit errorOccurred(error); @@ -814,6 +827,7 @@ bool MocapController::beginPreviewWithLiveSource( delete d->worker; d->worker = nullptr; d->camera.reset(); + unregisterSkinningListener(d.get()); restoreEntityState(); setStatusMessage(msg); emit errorOccurred(msg); @@ -1065,7 +1079,9 @@ void MocapController::onSample(const FaceSample& sample, d->bodyNeutralReady = d->bodyRetargeter->hasNeutralReference(); const uint32_t skipHead = - (d->headEnabled && !d->headBone.isEmpty()) ? (1u << 5) : 0u; + (d->headEnabled && !d->headBone.isEmpty()) + ? (1u << static_cast(PoseIK::Head)) + : 0u; const auto locals = d->bodyRetargeter->evaluateFrame( canonQuats, body.resolvedMask, skipHead, body.world.data(), body.visibility.data()); @@ -1173,23 +1189,24 @@ void MocapController::stopRecording() GamificationManager::Surface::Gui); } - // body clip: convert the buffered live frames to the [frame][22] world-quat - // stream recordBody expects (identity for roles unresolved that frame), and - // push a SEPARATE undo command so face + body each undo cleanly. + // body clip: bake the buffered live frames with landmark-direction retarget + // (same BodyRetargeter path as preview) into a separate undo command. if (d->bodyRetargeter && d->bodyTake.size() >= 2) { - std::vector>> clipQuats; - clipQuats.reserve(d->bodyTake.size()); + std::vector valid; + valid.reserve(d->bodyTake.size()); for (const auto& bf : d->bodyTake) { - if (!bf.valid) - continue; - clipQuats.emplace_back(bf.quats.begin(), bf.quats.end()); + if (bf.valid) + valid.push_back(bf); } - if (clipQuats.size() >= 2) { + if (valid.size() >= 2) { MocapRecorder::BodyRecordOptions bopts; bopts.clipName = d->clipName + QStringLiteral("_Body"); - bopts.algorithmUsed = QStringLiteral("pose-ik"); + bopts.algorithmUsed = QStringLiteral("pose-ik-landmarks"); + if (d->headEnabled && !d->headBone.isEmpty()) + bopts.skipRolesMask = + (1u << static_cast(PoseIK::Head)); const int fps = 30; - auto* bcmd = new RecordBodyClipCommand(d->entityName, clipQuats, + auto* bcmd = new RecordBodyClipCommand(d->entityName, std::move(valid), fps, bopts); UndoManager::getSingleton()->push(bcmd); const auto& br = bcmd->report(); @@ -1299,8 +1316,7 @@ void MocapController::stopPreview() d->poseDebugOverlay.detach(); if (d->skinningListener) { - Ogre::Root::getSingleton().removeFrameListener(d->skinningListener.get()); - d->skinningListener->impl = nullptr; + unregisterSkinningListener(d.get()); } restoreEntityState(); diff --git a/src/Mocap/MocapPoseDebugOverlay.cpp b/src/Mocap/MocapPoseDebugOverlay.cpp index 7da6a268c..6c6b10a57 100644 --- a/src/Mocap/MocapPoseDebugOverlay.cpp +++ b/src/Mocap/MocapPoseDebugOverlay.cpp @@ -195,7 +195,10 @@ void MocapPoseDebugOverlay::ensureMaterial() kMatName, Ogre::ResourceGroupManager::INTERNAL_RESOURCE_GROUP_NAME); if (!mat->getNumTechniques()) mat->createTechnique(); - Ogre::Pass* pass = mat->getTechnique(0)->getPass(0); + Ogre::Technique* tech = mat->getTechnique(0); + if (!tech->getNumPasses()) + tech->createPass(); + Ogre::Pass* pass = tech->getPass(0); pass->setLightingEnabled(false); pass->setVertexColourTracking(Ogre::TVC_DIFFUSE); pass->setCullingMode(Ogre::CULL_NONE); diff --git a/src/Mocap/MocapPoseFix.h b/src/Mocap/MocapPoseFix.h index 301a18d29..6e13cfc43 100644 --- a/src/Mocap/MocapPoseFix.h +++ b/src/Mocap/MocapPoseFix.h @@ -62,8 +62,9 @@ inline Ogre::Quaternion kPoseToSkeletonYawPi() return Ogre::Quaternion(Ogre::Degree(180), Ogre::Vector3::UNIT_Y); } -// Humanoid head bones often need camera-frame pitch inverted while yaw is -// corrected separately for mirrored webcam previews. +// Humanoid head bones often need camera-frame pitch inverted; for unit +// quaternions, negating X mirrors rotation about +X (pitch) while leaving +// yaw (Y) unchanged to first order. Yaw is corrected separately below. inline Ogre::Quaternion invertCameraPitchDelta(const Ogre::Quaternion& delta) { return Ogre::Quaternion(delta.w, -delta.x, delta.y, delta.z); diff --git a/src/Mocap/MocapRecorder.cpp b/src/Mocap/MocapRecorder.cpp index c8ca585b7..7b8105eff 100644 --- a/src/Mocap/MocapRecorder.cpp +++ b/src/Mocap/MocapRecorder.cpp @@ -336,10 +336,10 @@ BodyRecordReport recordBody( // existing clip on a retarget failure. The replace is deferred to the // moment the take is known good. - // VALIDATION HARNESS (QTMESH_MOCAP_USE_RETARGETER=1): bake the clip with - // the SHARED BodyRetargeter — the EXACT math the LIVE preview runs — so a - // rendered clip validates the live path headlessly (the live drive can't - // be render-captured directly). Default path stays applyMotionClip. + // VALIDATION HARNESS (QTMESH_MOCAP_USE_RETARGETER=1): bake via BodyRetargeter + // using quaternion deltas only (no landmark directions) — useful for + // headless regression of the fallback retarget path, not the live landmark + // drive (use recordBodyLive for that). if (qEnvironmentVariableIntValue("QTMESH_MOCAP_USE_RETARGETER")) { const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); BodyRetargeter rt(skel.get(), yaw180); @@ -420,6 +420,172 @@ BodyRecordReport recordBody( return report; } +namespace { + +constexpr uint32_t kTorsoResolvedMask = + (1u << 0) | (1u << 1) | (1u << 2); // hip, abdomen, chest + +std::array, 22> frameQuats(const BodyLiveFrame& frame) +{ + std::array, 22> q{}; + for (int r = 0; r < PoseIK::kCanonicalRoles; ++r) + q[static_cast(r)] = frame.quats[r]; + return q; +} + +BodyRecordReport bakeRetargeterClip( + Ogre::Entity* entity, + Ogre::Skeleton* skel, + const std::string& clip, + int fps, + const BodyRecordOptions& options, + const std::vector& frames, + bool useLandmarks, + uint32_t skipRolesMask) +{ + BodyRecordReport report; + report.clipName = options.clipName; + report.algorithmUsed = options.algorithmUsed; + report.fallbackReason = options.fallbackReason; + report.framesProcessed = static_cast(frames.size()); + + const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + BodyRetargeter rt(skel, yaw180); + if (!rt.valid()) { + report.error = QStringLiteral("retargeter: not a humanoid rig"); + return report; + } + if (skel->hasAnimation(clip)) + skel->removeAnimation(clip); + skel->reset(true); + std::map bindLocal; + for (unsigned short i = 0; i < skel->getNumBones(); ++i) + bindLocal[i] = skel->getBone(i)->getOrientation(); + + bool neutralSet = false; + for (const auto& frame : frames) { + if (!frame.valid) + continue; + if ((frame.resolvedMask & kTorsoResolvedMask) != kTorsoResolvedMask) + continue; + const auto q = frameQuats(frame); + if (useLandmarks) + rt.setNeutralReference(q, frame.resolvedMask, frame.world.data(), + frame.visibility.data()); + else + rt.setNeutralReference(q, frame.resolvedMask); + neutralSet = true; + break; + } + if (!neutralSet) { + for (const auto& frame : frames) { + if (!frame.valid) + continue; + const auto q = frameQuats(frame); + if (useLandmarks) + rt.setNeutralReference(q, frame.resolvedMask, frame.world.data(), + frame.visibility.data()); + else + rt.setNeutralReference(q, frame.resolvedMask); + neutralSet = true; + break; + } + } + if (!neutralSet) { + report.error = QStringLiteral("no confident body frames to calibrate"); + return report; + } + + const double dt = 1.0 / static_cast(fps); + Ogre::Animation* anim = skel->createAnimation( + clip, static_cast(dt * (frames.size() - 1))); + anim->setRotationInterpolationMode(Ogre::Animation::RIM_LINEAR); + std::map tracks; + for (size_t f = 0; f < frames.size(); ++f) { + const auto& frame = frames[f]; + if (!frame.valid) + continue; + const auto q = frameQuats(frame); + const float* world = useLandmarks ? frame.world.data() : nullptr; + const float* vis = useLandmarks ? frame.visibility.data() : nullptr; + const auto locals = rt.evaluateFrame(q, frame.resolvedMask, skipRolesMask, + world, vis); + for (const auto& [handle, local] : locals) { + auto it = tracks.find(handle); + if (it == tracks.end()) + it = tracks.emplace( + handle, anim->createNodeTrack(handle, skel->getBone(handle))) + .first; + auto* kf = it->second->createNodeKeyFrame( + static_cast(dt * f)); + kf->setRotation(bindLocal[handle].Inverse() * local); + } + } + report.tracksWritten = static_cast(tracks.size()); + report.rolesResolved = static_cast(tracks.size()); + report.clipLength = anim->getLength(); + entity->refreshAvailableAnimationState(); + return report; +} + +} // namespace + +BodyRecordReport recordBodyLive( + Ogre::Entity* entity, + const std::vector& frames, int fps, + const BodyRecordOptions& options) +{ + BodyRecordReport report; + report.clipName = options.clipName; + report.algorithmUsed = options.algorithmUsed; + report.fallbackReason = options.fallbackReason; + + if (!entity) { + report.error = QStringLiteral("no entity"); + return report; + } + if (!entity->hasSkeleton() || !entity->getMesh() + || !entity->getMesh()->getSkeleton()) { + report.error = QStringLiteral( + "the mesh is not skinned — body capture retargets onto a humanoid " + "skeleton (rig one first: qtmesh rig --skeleton humanoid --skin)"); + return report; + } + + std::vector valid; + valid.reserve(frames.size()); + for (const auto& frame : frames) { + if (frame.valid) + valid.push_back(frame); + } + report.framesProcessed = static_cast(valid.size()); + if (valid.size() < 2 || fps <= 0) { + report.error = QStringLiteral("need at least 2 pose frames"); + return report; + } + + Ogre::SkeletonPtr skel = entity->getMesh()->getSkeleton(); + const std::string clip = options.clipName.toStdString(); + if (skel->hasAnimation(clip) && !options.replaceExisting) { + report.error = QStringLiteral( + "animation '%1' already exists (pass replace)").arg(options.clipName); + return report; + } + + report = bakeRetargeterClip(entity, skel.get(), clip, fps, options, valid, + /*useLandmarks=*/true, options.skipRolesMask); + if (report.ok()) { + SentryReporter::addBreadcrumb( + "ai.assist.mocap_body", + QStringLiteral("recorded '%1' via BodyRetargeter+landmarks: %2 frames, " + "%3 tracks") + .arg(options.clipName) + .arg(report.framesProcessed) + .arg(report.tracksWritten)); + } + return report; +} + } // namespace MocapRecorder #endif // ENABLE_MOCAP diff --git a/src/Mocap/MocapRecorder.h b/src/Mocap/MocapRecorder.h index eaf6664d4..1fb1dbe44 100644 --- a/src/Mocap/MocapRecorder.h +++ b/src/Mocap/MocapRecorder.h @@ -26,6 +26,7 @@ #include "FaceCapMapper.h" #include "FaceCapPredictor.h" +#include "MocapLiveTypes.h" #include #include @@ -80,6 +81,7 @@ struct BodyRecordOptions { bool replaceExisting = true; QString algorithmUsed = QStringLiteral("pose-ik"); // for the report QString fallbackReason; // why not sam3dbody + uint32_t skipRolesMask = 0; // e.g. Head role when head bone is driven separately }; struct BodyRecordReport { @@ -105,6 +107,13 @@ BodyRecordReport recordBody( const std::vector>>& clipQuats, int fps, const BodyRecordOptions& options = {}); +// Same retarget path as the live preview: BodyRetargeter + landmark directions. +// `frames` must be time-ascending BodyLiveFrame samples (world + visibility kept). +BodyRecordReport recordBodyLive( + Ogre::Entity* entity, + const std::vector& frames, int fps, + const BodyRecordOptions& options = {}); + // Head-bone resolution (exposed for the GUI gate + tests): the first bone of // the entity's skeleton whose name resolves to the canonical Head role, or // empty when the entity is not skinned / has no such bone. diff --git a/src/commands/RecordMocapClipCommand.cpp b/src/commands/RecordMocapClipCommand.cpp index 9635be02e..1eb8a70a7 100644 --- a/src/commands/RecordMocapClipCommand.cpp +++ b/src/commands/RecordMocapClipCommand.cpp @@ -208,6 +208,18 @@ RecordBodyClipCommand::RecordBodyClipCommand( setText(QObject::tr("Record body capture '%1'").arg(m_options.clipName)); } +RecordBodyClipCommand::RecordBodyClipCommand( + std::string entityName, std::vector liveFrames, int fps, + MocapRecorder::BodyRecordOptions options, QUndoCommand* parent) + : QUndoCommand(parent), + m_entityName(std::move(entityName)), + m_liveFrames(std::move(liveFrames)), + m_fps(fps), + m_options(std::move(options)) +{ + setText(QObject::tr("Record body capture '%1'").arg(m_options.clipName)); +} + RecordBodyClipCommand::~RecordBodyClipCommand() = default; void RecordBodyClipCommand::redo() @@ -238,7 +250,11 @@ void RecordBodyClipCommand::redo() m_snapshotTaken = true; } - m_report = MocapRecorder::recordBody(entity, m_clipQuats, m_fps, m_options); + m_report = !m_liveFrames.empty() + ? MocapRecorder::recordBodyLive( + entity, m_liveFrames, m_fps, m_options) + : MocapRecorder::recordBody( + entity, m_clipQuats, m_fps, m_options); } void RecordBodyClipCommand::undo() diff --git a/src/commands/RecordMocapClipCommand.h b/src/commands/RecordMocapClipCommand.h index b734dd38b..e775bdc59 100644 --- a/src/commands/RecordMocapClipCommand.h +++ b/src/commands/RecordMocapClipCommand.h @@ -20,6 +20,7 @@ #include "../Mocap/FaceCapMapper.h" #include "../Mocap/MocapRecorder.h" +#include "../Mocap/MocapLiveTypes.h" #include #include @@ -60,6 +61,10 @@ class RecordBodyClipCommand : public QUndoCommand std::vector>> clipQuats, int fps, MocapRecorder::BodyRecordOptions options, QUndoCommand* parent = nullptr); + RecordBodyClipCommand(std::string entityName, + std::vector liveFrames, + int fps, MocapRecorder::BodyRecordOptions options, + QUndoCommand* parent = nullptr); ~RecordBodyClipCommand() override; void undo() override; @@ -72,6 +77,7 @@ class RecordBodyClipCommand : public QUndoCommand std::string m_entityName; std::vector>> m_clipQuats; + std::vector m_liveFrames; int m_fps; MocapRecorder::BodyRecordOptions m_options; MocapRecorder::BodyRecordReport m_report; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3e6ea1242..db03199fd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -70,6 +70,7 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/LLMManager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/LLMWorker.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/LLMSettingsWidget.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/OnnxRuntimeSettings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ModelDownloader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/SDManager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/SDWorker.cpp @@ -105,6 +106,8 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/Mocap/PoseIKSolver.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/Mocap/MocapCLI.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/Mocap/MocapController.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/Mocap/MocapPoseDebugOverlay.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/Mocap/MocapBodyDriveDebug.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/commands/RecordMocapClipCommand.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MCPServer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MCPSettingsDialog.cpp From bb65a476b8f1a4f27b38678e5ca65e88cf2382b1 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 4 Aug 2026 09:35:48 -0400 Subject: [PATCH 06/11] fix(mocap): make skinning listener cleanup an Impl method Free function in an anonymous namespace could not access private Impl; move unregisterSkinningListener onto Impl so MocapController call sites compile. Co-authored-by: Cursor --- src/Mocap/MocapController.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index 06d52b692..e41251787 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -142,14 +142,6 @@ struct MocapMetaTypeRegistrar { }; const MocapMetaTypeRegistrar mocapMetaTypeRegistrar; -void unregisterSkinningListener(MocapController::Impl* impl) -{ - if (!impl || !impl->skinningListener) - return; - Ogre::Root::getSingleton().removeFrameListener(impl->skinningListener.get()); - impl->skinningListener->impl = nullptr; -} - struct BodyDriveBone { int role = -1; std::string boneName; @@ -341,6 +333,14 @@ struct MocapController::Impl { return scene->hasEntity(entityName) ? scene->getEntity(entityName) : nullptr; } + + void unregisterSkinningListener() + { + if (!skinningListener) + return; + Ogre::Root::getSingleton().removeFrameListener(skinningListener.get()); + skinningListener->impl = nullptr; + } }; MocapController::MocapController(QObject* parent) @@ -807,7 +807,7 @@ bool MocapController::beginPreviewWithLiveSource( QString error; if (!d->camera->open(&error)) { d->camera.reset(); - unregisterSkinningListener(d.get()); + d->unregisterSkinningListener(); restoreEntityState(); setStatusMessage(error); emit errorOccurred(error); @@ -827,7 +827,7 @@ bool MocapController::beginPreviewWithLiveSource( delete d->worker; d->worker = nullptr; d->camera.reset(); - unregisterSkinningListener(d.get()); + d->unregisterSkinningListener(); restoreEntityState(); setStatusMessage(msg); emit errorOccurred(msg); @@ -1316,7 +1316,7 @@ void MocapController::stopPreview() d->poseDebugOverlay.detach(); if (d->skinningListener) { - unregisterSkinningListener(d.get()); + d->unregisterSkinningListener(); } restoreEntityState(); From 545b89cea41c6ff652a047230af594be97f11fd5 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 4 Aug 2026 09:41:00 -0400 Subject: [PATCH 07/11] fix(mocap): restore body drive after head-skip regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review fix skipped the head bone entirely in the body setup loop, which also skipped zeroing its animation blend mask — idle clips could fight manual drive. Still exclude head from bodyManualRestore (preserve headWasManuallyControlled) but zero blend masks on all bones; track skinning listener registration. Co-authored-by: Cursor --- src/AnimationMerger.cpp | 5 +-- src/Mocap/MocapController.cpp | 63 ++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index c4706d211..d11cc5701 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1800,11 +1800,12 @@ BodyRetargeter::evaluateFrame( (pi >= 0) ? W[static_cast(pi)] : Ogre::Quaternion::IDENTITY; const int c = d->boneToCanon[static_cast(i)]; - if (c < 0 || c >= Jc - || (skipRolesMask & (1u << static_cast(c)))) { + if (c < 0 || c >= Jc) { W[static_cast(i)] = Wp * base; continue; } + if (skipRolesMask & (1u << static_cast(c))) + continue; Ogre::Quaternion local; if (c == 0) { local = base; diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index e41251787..491a8a7a1 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -322,6 +322,7 @@ struct MocapController::Impl { }; std::unique_ptr skinningListener; + bool skinningListenerRegistered = false; bool addedSoftwareAnimRequest = false; Ogre::Entity* entity() const @@ -336,10 +337,22 @@ struct MocapController::Impl { void unregisterSkinningListener() { - if (!skinningListener) + if (!skinningListener || !skinningListenerRegistered) return; Ogre::Root::getSingleton().removeFrameListener(skinningListener.get()); skinningListener->impl = nullptr; + skinningListenerRegistered = false; + } + + void registerSkinningListener() + { + if (!skinningListener) + skinningListener = std::make_unique(); + skinningListener->impl = this; + if (!skinningListenerRegistered) { + Ogre::Root::getSingleton().addFrameListener(skinningListener.get()); + skinningListenerRegistered = true; + } } }; @@ -744,26 +757,28 @@ bool MocapController::beginPreviewWithLiveSource( headBoneDrive ? d->headBone.toStdString() : std::string{}; for (unsigned short i = 0; i < skel->getNumBones(); ++i) { Ogre::Bone* bone = skel->getBone(i); - if (headBoneDrive && bone->getName() == headBoneStd) - continue; - BodyManualBoneSnapshot snap; - snap.boneName = bone->getName(); - snap.bindLocal = bone->getOrientation(); - snap.wasManuallyControlled = bone->isManuallyControlled(); - d->bodyManualRestore.push_back(std::move(snap)); - bone->setManuallyControlled(true); - const int role = MotionInbetween::canonicalIndexForBone( - QString::fromStdString(bone->getName())); - if (role >= 0) { - BodyDriveBone bb; - bb.role = role; - bb.boneName = bone->getName(); - bb.bindLocal = bone->getOrientation(); - bb.wasManuallyControlled = snap.wasManuallyControlled; - d->bodyBones.push_back(std::move(bb)); + const bool isHeadBone = + headBoneDrive && bone->getName() == headBoneStd; + if (!isHeadBone) { + BodyManualBoneSnapshot snap; + snap.boneName = bone->getName(); + snap.bindLocal = bone->getOrientation(); + snap.wasManuallyControlled = bone->isManuallyControlled(); + d->bodyManualRestore.push_back(std::move(snap)); + bone->setManuallyControlled(true); + const int role = MotionInbetween::canonicalIndexForBone( + QString::fromStdString(bone->getName())); + if (role >= 0) { + BodyDriveBone bb; + bb.role = role; + bb.boneName = bone->getName(); + bb.bindLocal = bone->getOrientation(); + bb.wasManuallyControlled = snap.wasManuallyControlled; + d->bodyBones.push_back(std::move(bb)); + } } - // Animation tracks still write manually-controlled bones unless - // the blend mask is zero (TransformOperator bone-drag pattern). + // Zero animation influence on every bone (incl. head) so idle + // clips cannot fight manual mocap drive. if (auto* states = entity->getAllAnimationStates()) { const auto nBones = static_cast(skel->getNumBones()); for (const auto& [animName, st] : states->getAnimationStates()) { @@ -783,10 +798,7 @@ bool MocapController::beginPreviewWithLiveSource( d->bodyBones.clear(); } - if (!d->skinningListener) - d->skinningListener = std::make_unique(); - d->skinningListener->impl = d.get(); - Ogre::Root::getSingleton().addFrameListener(d->skinningListener.get()); + d->registerSkinningListener(); if (!d->addedSoftwareAnimRequest) { entity->addSoftwareAnimationRequest(true); d->addedSoftwareAnimRequest = true; @@ -1315,9 +1327,8 @@ void MocapController::stopPreview() d->bodyRetargeter.reset(); d->poseDebugOverlay.detach(); - if (d->skinningListener) { + if (d->skinningListener) d->unregisterSkinningListener(); - } restoreEntityState(); From c217684f6ef597fb2d1eb88a4426ace273738f15 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 4 Aug 2026 13:55:09 -0400 Subject: [PATCH 08/11] feat(mocap): show body calibration hint during live preview warmup Expose bodyCalibrationHint on MocapController and surface it in the Performance Capture panel while torso neutral is being captured. Co-authored-by: Cursor --- qml/PropertiesPanel.qml | 14 ++++++++++++++ src/Mocap/MocapController.cpp | 12 ++++++++++++ src/Mocap/MocapController.h | 3 +++ 3 files changed, 29 insertions(+) diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index 239348c58..dc639e091 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -2501,7 +2501,9 @@ Rectangle { } // camera preview + HUD + // live HUD (webcam thumbnail + detection dots) Rectangle { + id: mocapPreviewHud width: parent.width - 16 height: visible ? 140 : 0 visible: mocapReady && MocapController.previewDataUrl !== "" @@ -2548,6 +2550,18 @@ Rectangle { } } + Text { + width: parent.width - 16 + visible: mocapReady && previewing + && MocapController.bodyEnabled + && MocapController.bodyCalibrationHint !== "" + wrapMode: Text.Wrap + color: PropertiesPanelController.highlightColor + font.pixelSize: 10 + font.italic: true + text: MocapController.bodyCalibrationHint + } + // channel summary Text { width: parent.width - 16 diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index 491a8a7a1..d5caf34cd 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -99,6 +99,7 @@ QStringList MocapController::unmatchedChannels() const { return {}; } bool MocapController::headAvailable() const { return false; } bool MocapController::bodyAvailable() const { return false; } bool MocapController::bodyDetected() const { return false; } +QString MocapController::bodyCalibrationHint() const { return {}; } bool MocapController::faceEnabled() const { return false; } void MocapController::setFaceEnabled(bool) {} bool MocapController::headEnabled() const { return false; } @@ -409,6 +410,17 @@ QStringList MocapController::unmatchedChannels() const bool MocapController::headAvailable() const { return !d->headBone.isEmpty(); } bool MocapController::bodyAvailable() const { return d->bodyRigOk; } bool MocapController::bodyDetected() const { return d->bodyDetected; } +QString MocapController::bodyCalibrationHint() const +{ + if (d->state < Previewing || !d->bodyEnabled || !d->bodyRetargeter) + return {}; + if (d->bodyNeutralReady) + return {}; + if (!d->bodyDetected) + return tr("Stand in frame so your hips and shoulders are visible…"); + return tr("Calibrating body drive — face the camera with arms relaxed at " + "your sides…"); +} bool MocapController::faceEnabled() const { return d->faceEnabled; } void MocapController::setFaceEnabled(bool on) { diff --git a/src/Mocap/MocapController.h b/src/Mocap/MocapController.h index 3531ed87f..2003b2d59 100644 --- a/src/Mocap/MocapController.h +++ b/src/Mocap/MocapController.h @@ -62,6 +62,8 @@ class MocapController : public QObject // body: only offerable when the selection is a humanoid rig Q_PROPERTY(bool bodyAvailable READ bodyAvailable NOTIFY mappingChanged) Q_PROPERTY(bool bodyDetected READ bodyDetected NOTIFY liveStatsChanged) + Q_PROPERTY(QString bodyCalibrationHint READ bodyCalibrationHint + NOTIFY liveStatsChanged) // channel enables (writable from the QML Face/Head/Body checkboxes) Q_PROPERTY(bool faceEnabled READ faceEnabled WRITE setFaceEnabled NOTIFY channelsChanged) @@ -99,6 +101,7 @@ class MocapController : public QObject bool headAvailable() const; bool bodyAvailable() const; bool bodyDetected() const; + QString bodyCalibrationHint() const; bool faceEnabled() const; void setFaceEnabled(bool on); bool headEnabled() const; From a4903c732797f8c44c5752ce4e1bb08ad652be46 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 4 Aug 2026 16:58:28 -0400 Subject: [PATCH 09/11] fix(mocap): address review feedback on retarget, restore, and FK dedup Propagate parent world transforms for skipped/unmapped bones, snapshot head and body manual state before skeleton reset, share PoseIK FK via MocapPoseIkFk, and make ONNX runtime env prep thread-safe. Co-authored-by: Cursor --- qml/AISettingsDialog.qml | 2 +- src/AnimationMerger.cpp | 5 +- src/Mocap/MocapBodyDriveDebug.cpp | 104 +-------------------- src/Mocap/MocapController.cpp | 54 +++++++---- src/Mocap/MocapPoseDebugOverlay.cpp | 93 +------------------ src/Mocap/MocapPoseIkFk.h | 139 ++++++++++++++++++++++++++++ src/OnnxRuntimeSettings.cpp | 8 +- 7 files changed, 188 insertions(+), 217 deletions(-) create mode 100644 src/Mocap/MocapPoseIkFk.h diff --git a/qml/AISettingsDialog.qml b/qml/AISettingsDialog.qml index 31df437c2..8c7e51970 100644 --- a/qml/AISettingsDialog.qml +++ b/qml/AISettingsDialog.qml @@ -711,7 +711,7 @@ Dialog { Text { Layout.fillWidth: true - text: "Note: LLM settings take effect when loading a new model. ONNX GPU preference applies immediately." + text: "Note: LLM settings take effect when loading a new model. ONNX GPU preference applies to newly created ONNX sessions." font.pointSize: 9 font.italic: true color: Qt.darker(textColor, 1.5) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index d11cc5701..c4706d211 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1800,12 +1800,11 @@ BodyRetargeter::evaluateFrame( (pi >= 0) ? W[static_cast(pi)] : Ogre::Quaternion::IDENTITY; const int c = d->boneToCanon[static_cast(i)]; - if (c < 0 || c >= Jc) { + if (c < 0 || c >= Jc + || (skipRolesMask & (1u << static_cast(c)))) { W[static_cast(i)] = Wp * base; continue; } - if (skipRolesMask & (1u << static_cast(c))) - continue; Ogre::Quaternion local; if (c == 0) { local = base; diff --git a/src/Mocap/MocapBodyDriveDebug.cpp b/src/Mocap/MocapBodyDriveDebug.cpp index 13ce62d7c..5614e8c15 100644 --- a/src/Mocap/MocapBodyDriveDebug.cpp +++ b/src/Mocap/MocapBodyDriveDebug.cpp @@ -4,6 +4,7 @@ #include "../AnimationMerger.h" #include "../MotionInbetween.h" +#include "MocapPoseIkFk.h" #include "PoseIKSolver.h" #include @@ -16,11 +17,12 @@ #include #include +#include #include namespace { -using Vec3 = std::array; +using Vec3 = MocapPoseIkFk::Vec3; Vec3 sub(const Vec3& a, const Vec3& b) { @@ -49,104 +51,6 @@ Ogre::Quaternion quatFromArray(const std::array& q) return Ogre::Quaternion(q[3], q[0], q[1], q[2]); } -int effectiveParentRole(int role, uint32_t resolvedMask) -{ - int p = MotionInbetween::canonicalParentOf(role); - while (p >= 0 && !(resolvedMask & (1u << static_cast(p)))) - p = MotionInbetween::canonicalParentOf(p); - return p; -} - -Ogre::Quaternion localArtic( - const std::array, PoseIK::kCanonicalRoles>& quats, - int role, uint32_t resolvedMask) -{ - const int ep = effectiveParentRole(role, resolvedMask); - const Ogre::Quaternion oq = quatFromArray(quats[static_cast(role)]); - if (ep < 0) - return oq; - const Ogre::Quaternion op = quatFromArray(quats[static_cast(ep)]); - return op.Inverse() * oq; -} - -void fkPoseIkJoints( - const std::array, PoseIK::kCanonicalRoles>& quats, - uint32_t resolvedMask, - const std::array, PoseIK::kLandmarkCount>& canonLmPts, - std::array& out) -{ - out.fill({0.f, 0.f, 0.f}); - const Vec3 hip = { - (canonLmPts[23][0] + canonLmPts[24][0]) * 0.5f, - (canonLmPts[23][1] + canonLmPts[24][1]) * 0.5f, - (canonLmPts[23][2] + canonLmPts[24][2]) * 0.5f}; - out[static_cast(PoseIK::Hip)] = hip; - - struct BoneSeg { - int role; - int fromLm; - int toLm; - }; - static const BoneSeg segs[] = { - {PoseIK::Abdomen, 23, 11}, {PoseIK::Chest, 11, 12}, - {PoseIK::Neck, 12, 0}, {PoseIK::Head, 0, 8}, - {PoseIK::RShoulder, 12, 14}, {PoseIK::RElbow, 14, 16}, - {PoseIK::RHand, 16, 16}, - {PoseIK::LShoulder, 11, 13}, {PoseIK::LElbow, 13, 15}, - {PoseIK::LHand, 15, 15}, - {PoseIK::RHip, 24, 26}, {PoseIK::RKnee, 26, 28}, - {PoseIK::RFoot, 28, 32}, - {PoseIK::LHip, 23, 25}, {PoseIK::LKnee, 25, 27}, - {PoseIK::LFoot, 27, 31}, - }; - - std::array restOffset{}; - for (const BoneSeg& s : segs) { - Vec3 dir = sub(canonLmPts[static_cast(s.toLm)], - canonLmPts[static_cast(s.fromLm)]); - const float d = len(dir); - if (d < 1e-5f) - dir = {0.f, 0.12f, 0.f}; - else - dir = norm(dir); - restOffset[static_cast(s.role)] = { - dir[0] * std::max(d, 0.05f), - dir[1] * std::max(d, 0.05f), - dir[2] * std::max(d, 0.05f)}; - } - - std::array worldRot{}; - worldRot.fill(Ogre::Quaternion::IDENTITY); - - for (int role = 0; role < PoseIK::kCanonicalRoles; ++role) { - if (!(resolvedMask & (1u << static_cast(role)))) - continue; - const int parent = MotionInbetween::canonicalParentOf(role); - const Ogre::Quaternion local = localArtic(quats, role, resolvedMask); - if (parent >= 0 && (resolvedMask & (1u << static_cast(parent)))) { - worldRot[static_cast(role)] = - worldRot[static_cast(parent)] * local; - const Ogre::Vector3 off( - restOffset[static_cast(role)][0], - restOffset[static_cast(role)][1], - restOffset[static_cast(role)][2]); - const Ogre::Vector3 w = - worldRot[static_cast(parent)] * off; - out[static_cast(role)] = { - out[static_cast(parent)][0] + w.x, - out[static_cast(parent)][1] + w.y, - out[static_cast(parent)][2] + w.z}; - } else if (role == PoseIK::Hip) { - worldRot[0] = quatFromArray(quats[0]); - } else { - out[static_cast(role)] = { - hip[0] + restOffset[static_cast(role)][0], - hip[1] + restOffset[static_cast(role)][1], - hip[2] + restOffset[static_cast(role)][2]}; - } - } -} - const char* roleName(int role) { static const char* names[] = { @@ -207,7 +111,7 @@ void logFrame( const float scale = (entityH > 1e-3f ? entityH : 1.8f) / skelH; std::array fk{}; - fkPoseIkJoints(body.quats, body.resolvedMask, canon, fk); + MocapPoseIkFk::fkPoseIkJoints(body.quats, body.resolvedMask, canon, fk); skel->_updateTransforms(); diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index d5caf34cd..504134e5f 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -255,7 +255,9 @@ struct MocapController::Impl { Ogre::Quaternion neutral = Ogre::Quaternion::IDENTITY; Ogre::Quaternion headBindWorld = Ogre::Quaternion::IDENTITY; Ogre::Quaternion headBindLocal = Ogre::Quaternion::IDENTITY; + Ogre::Quaternion headRestoreLocal = Ogre::Quaternion::IDENTITY; bool headWasManuallyControlled = false; + bool savedHeadSnapshot = false; QHash savedWeights; // mesh target -> weight QStringList savedEnabledAnimations; bool savedSkipAnimStateUpdate = false; @@ -470,9 +472,12 @@ void MocapController::setShowPoseDebug(bool on) return; d->showPoseDebug = on; if (on && d->state != Idle) { - if (Ogre::Entity* entity = d->entity()) - d->poseDebugOverlay.attach(Manager::getSingleton()->getSceneMgr(), - entity->getParentSceneNode()); + if (Ogre::Entity* entity = d->entity()) { + auto* mgr = Manager::getSingletonPtr(); + if (mgr && mgr->getSceneMgr()) + d->poseDebugOverlay.attach(mgr->getSceneMgr(), + entity->getParentSceneNode()); + } } else { d->poseDebugOverlay.detach(); } @@ -739,13 +744,34 @@ bool MocapController::beginPreviewWithLiveSource( // PoseIK's head role is coarse and fights the face solve. const bool headBoneDrive = d->headEnabled && !d->headBone.isEmpty(); + const std::string headBoneStd = + headBoneDrive ? d->headBone.toStdString() : std::string{}; if (entity->hasSkeleton()) { Ogre::SkeletonInstance* skel = entity->getSkeleton(); + // Snapshot manual-bone state BEFORE reset(true) — reset clears manual + // orientations, so post-reset snapshots cannot be restored faithfully. + d->bodyManualRestore.clear(); + d->savedHeadSnapshot = false; + for (unsigned short i = 0; i < skel->getNumBones(); ++i) { + Ogre::Bone* bone = skel->getBone(i); + if (headBoneDrive && bone->getName() == headBoneStd) { + d->headWasManuallyControlled = bone->isManuallyControlled(); + d->headRestoreLocal = bone->getOrientation(); + d->savedHeadSnapshot = true; + continue; + } + if (bodyDrivable) { + BodyManualBoneSnapshot snap; + snap.boneName = bone->getName(); + snap.bindLocal = bone->getOrientation(); + snap.wasManuallyControlled = bone->isManuallyControlled(); + d->bodyManualRestore.push_back(std::move(snap)); + } + } skel->reset(true); skel->_updateTransforms(); if (headBoneDrive) { - Ogre::Bone* bone = skel->getBone(d->headBone.toStdString()); - d->headWasManuallyControlled = bone->isManuallyControlled(); + Ogre::Bone* bone = skel->getBone(headBoneStd); d->headBindLocal = bone->getOrientation(); d->headBindWorld = bone->_getDerivedOrientation(); bone->setManuallyControlled(true); @@ -754,7 +780,6 @@ bool MocapController::beginPreviewWithLiveSource( // body drive setup: BodyRetargeter + manual bone control for live drive. d->bodyRetargeter.reset(); - d->bodyManualRestore.clear(); d->bodyBones.clear(); d->bodyAnimMaskRestore.clear(); if (bodyDrivable && entity->hasSkeleton()) { @@ -765,19 +790,13 @@ bool MocapController::beginPreviewWithLiveSource( d->bodyRetargeter.reset(); d->bodyBones.clear(); } else { - const std::string headBoneStd = - headBoneDrive ? d->headBone.toStdString() : std::string{}; for (unsigned short i = 0; i < skel->getNumBones(); ++i) { Ogre::Bone* bone = skel->getBone(i); const bool isHeadBone = headBoneDrive && bone->getName() == headBoneStd; - if (!isHeadBone) { - BodyManualBoneSnapshot snap; - snap.boneName = bone->getName(); - snap.bindLocal = bone->getOrientation(); - snap.wasManuallyControlled = bone->isManuallyControlled(); - d->bodyManualRestore.push_back(std::move(snap)); + if (!isHeadBone) bone->setManuallyControlled(true); + if (!isHeadBone) { const int role = MotionInbetween::canonicalIndexForBone( QString::fromStdString(bone->getName())); if (role >= 0) { @@ -785,7 +804,7 @@ bool MocapController::beginPreviewWithLiveSource( bb.role = role; bb.boneName = bone->getName(); bb.bindLocal = bone->getOrientation(); - bb.wasManuallyControlled = snap.wasManuallyControlled; + bb.wasManuallyControlled = bone->isManuallyControlled(); d->bodyBones.push_back(std::move(bb)); } } @@ -1271,10 +1290,11 @@ void MocapController::restoreEntityState() } if (entity->hasSkeleton()) { Ogre::SkeletonInstance* skel = entity->getSkeleton(); - if (!d->headBone.isEmpty()) { + if (d->savedHeadSnapshot && !d->headBone.isEmpty()) { Ogre::Bone* bone = skel->getBone(d->headBone.toStdString()); - bone->setOrientation(d->headBindLocal); + bone->setOrientation(d->headRestoreLocal); bone->setManuallyControlled(d->headWasManuallyControlled); + d->savedHeadSnapshot = false; } for (const auto& snap : d->bodyManualRestore) { Ogre::Bone* bone = skel->getBone(snap.boneName); diff --git a/src/Mocap/MocapPoseDebugOverlay.cpp b/src/Mocap/MocapPoseDebugOverlay.cpp index 6c6b10a57..75a5fddf2 100644 --- a/src/Mocap/MocapPoseDebugOverlay.cpp +++ b/src/Mocap/MocapPoseDebugOverlay.cpp @@ -3,6 +3,7 @@ #include "MocapPoseDebugOverlay.h" #include "MocapLiveTypes.h" +#include "MocapPoseIkFk.h" #include "../MotionInbetween.h" #include @@ -64,25 +65,6 @@ Ogre::Quaternion quatFromArray(const std::array& q) return Ogre::Quaternion(q[3], q[0], q[1], q[2]); } -int effectiveParentRole(int role, uint32_t resolvedMask) -{ - int p = MotionInbetween::canonicalParentOf(role); - while (p >= 0 && !(resolvedMask & (1u << static_cast(p)))) - p = MotionInbetween::canonicalParentOf(p); - return p; -} - -Ogre::Quaternion localArtic( - const std::array, PoseIK::kCanonicalRoles>& src, - int role, uint32_t resolvedMask) -{ - const int ep = effectiveParentRole(role, resolvedMask); - if (ep >= 0) - return quatFromArray(src[static_cast(ep)]).Inverse() - * quatFromArray(src[static_cast(role)]); - return quatFromArray(src[static_cast(role)]); -} - const int kLmEdges[][2] = { {0, 1}, {1, 2}, {2, 3}, {3, 7}, {0, 4}, {4, 5}, {5, 6}, {6, 8}, {9, 10}, {11, 12}, {11, 13}, {13, 15}, {15, 17}, {15, 19}, @@ -102,77 +84,6 @@ float skeletonHeight(const std::array, PoseIK::kLandmarkCou return h > 1e-4f ? h : 1.65f; } -void fkPoseIkJoints( - const std::array, PoseIK::kCanonicalRoles>& quats, - uint32_t resolvedMask, - const std::array, PoseIK::kLandmarkCount>& canonLmPts, - std::array& out) -{ - out.fill({0.f, 0.f, 0.f}); - const Vec3 hip = mid(canonLmPts[23], canonLmPts[24]); - out[static_cast(PoseIK::Hip)] = hip; - - struct BoneSeg { - int role; - int fromLm; - int toLm; - }; - static const BoneSeg segs[] = { - {PoseIK::Abdomen, 23, 11}, - {PoseIK::Chest, 11, 12}, - {PoseIK::Neck, 12, 0}, - {PoseIK::Head, 0, 8}, - {PoseIK::RShoulder, 12, 14}, - {PoseIK::RElbow, 14, 16}, - {PoseIK::RHand, 16, 16}, - {PoseIK::LShoulder, 11, 13}, - {PoseIK::LElbow, 13, 15}, - {PoseIK::LHand, 15, 15}, - {PoseIK::RHip, 24, 26}, - {PoseIK::RKnee, 26, 28}, - {PoseIK::RFoot, 28, 32}, - {PoseIK::LHip, 23, 25}, - {PoseIK::LKnee, 25, 27}, - {PoseIK::LFoot, 27, 31}, - }; - - std::array restOffset{}; - for (const BoneSeg& s : segs) { - Vec3 dir = sub(canonLmPts[static_cast(s.toLm)], - canonLmPts[static_cast(s.fromLm)]); - const float d = len(dir); - if (d < 1e-5f) - dir = {0.f, 0.12f, 0.f}; - else - dir = mul(dir, 1.f / d); - restOffset[static_cast(s.role)] = mul(dir, std::max(d, 0.05f)); - } - - std::array worldRot{}; - worldRot.fill(Ogre::Quaternion::IDENTITY); - - for (int role = 0; role < PoseIK::kCanonicalRoles; ++role) { - if (!(resolvedMask & (1u << static_cast(role)))) - continue; - const int parent = MotionInbetween::canonicalParentOf(role); - const Ogre::Quaternion local = localArtic(quats, role, resolvedMask); - if (parent >= 0 && (resolvedMask & (1u << static_cast(parent)))) { - worldRot[static_cast(role)] = - worldRot[static_cast(parent)] * local; - out[static_cast(role)] = - add(out[static_cast(parent)], - mulVec3(worldRot[static_cast(parent)], - restOffset[static_cast(role)])); - } else if (role == PoseIK::Hip) { - worldRot[0] = quatFromArray(quats[0]); - } else { - worldRot[static_cast(role)] = quatFromArray(quats[role]); - out[static_cast(role)] = - add(hip, restOffset[static_cast(role)]); - } - } -} - void appendLines(Ogre::ManualObject* mo, const std::vector>& segs, float scale, const Ogre::ColourValue& colour) @@ -276,7 +187,7 @@ void MocapPoseDebugOverlay::update(const BodyLiveFrame& body, float entityHeight lmLines.emplace_back(canonLm(canon, e[0]), canonLm(canon, e[1])); std::array joints{}; - fkPoseIkJoints(body.quats, body.resolvedMask, canon, joints); + MocapPoseIkFk::fkPoseIkJoints(body.quats, body.resolvedMask, canon, joints); std::vector> ikLines; for (int role = 0; role < PoseIK::kCanonicalRoles; ++role) { diff --git a/src/Mocap/MocapPoseIkFk.h b/src/Mocap/MocapPoseIkFk.h new file mode 100644 index 000000000..ddfc3d6d3 --- /dev/null +++ b/src/Mocap/MocapPoseIkFk.h @@ -0,0 +1,139 @@ +#ifndef MOCAPPOSEIKFK_H +#define MOCAPPOSEIKFK_H + +#ifdef ENABLE_MOCAP + +#include "../MotionInbetween.h" +#include "PoseIKSolver.h" + +#include +#include + +#include +#include +#include + +namespace MocapPoseIkFk { + +using Vec3 = std::array; + +inline Vec3 sub(const Vec3& a, const Vec3& b) +{ + return {a[0] - b[0], a[1] - b[1], a[2] - b[2]}; +} +inline Vec3 add(const Vec3& a, const Vec3& b) +{ + return {a[0] + b[0], a[1] + b[1], a[2] + b[2]}; +} +inline Vec3 mul(const Vec3& a, float s) +{ + return {a[0] * s, a[1] * s, a[2] * s}; +} +inline Vec3 mid(const Vec3& a, const Vec3& b) +{ + return mul(add(a, b), 0.5f); +} +inline float len(const Vec3& a) +{ + return std::sqrt(a[0] * a[0] + a[1] * a[1] + a[2] * a[2]); +} +inline Vec3 mulVec3(const Ogre::Quaternion& q, const Vec3& v) +{ + const Ogre::Vector3 r = q * Ogre::Vector3(v[0], v[1], v[2]); + return {r.x, r.y, r.z}; +} +inline Ogre::Quaternion quatFromArray(const std::array& q) +{ + return Ogre::Quaternion(q[3], q[0], q[1], q[2]); +} + +inline int effectiveParentRole(int role, uint32_t resolvedMask) +{ + int p = MotionInbetween::canonicalParentOf(role); + while (p >= 0 && !(resolvedMask & (1u << static_cast(p)))) + p = MotionInbetween::canonicalParentOf(p); + return p; +} + +inline Ogre::Quaternion localArtic( + const std::array, PoseIK::kCanonicalRoles>& src, + int role, uint32_t resolvedMask) +{ + const int ep = effectiveParentRole(role, resolvedMask); + if (ep >= 0) + return quatFromArray(src[static_cast(ep)]).Inverse() + * quatFromArray(src[static_cast(role)]); + return quatFromArray(src[static_cast(role)]); +} + +// PoseIK canonical FK used by the debug overlay and body-drive diagnostics. +inline void fkPoseIkJoints( + const std::array, PoseIK::kCanonicalRoles>& quats, + uint32_t resolvedMask, + const std::array, PoseIK::kLandmarkCount>& canonLmPts, + std::array& out) +{ + out.fill({0.f, 0.f, 0.f}); + const Vec3 hip = mid(canonLmPts[23], canonLmPts[24]); + out[static_cast(PoseIK::Hip)] = hip; + + struct BoneSeg { + int role; + int fromLm; + int toLm; + }; + static const BoneSeg segs[] = { + {PoseIK::Abdomen, 23, 11}, {PoseIK::Chest, 11, 12}, + {PoseIK::Neck, 12, 0}, {PoseIK::Head, 0, 8}, + {PoseIK::RShoulder, 12, 14}, {PoseIK::RElbow, 14, 16}, + {PoseIK::RHand, 16, 16}, + {PoseIK::LShoulder, 11, 13}, {PoseIK::LElbow, 13, 15}, + {PoseIK::LHand, 15, 15}, + {PoseIK::RHip, 24, 26}, {PoseIK::RKnee, 26, 28}, + {PoseIK::RFoot, 28, 32}, + {PoseIK::LHip, 23, 25}, {PoseIK::LKnee, 25, 27}, + {PoseIK::LFoot, 27, 31}, + }; + + std::array restOffset{}; + for (const BoneSeg& s : segs) { + Vec3 dir = sub(canonLmPts[static_cast(s.toLm)], + canonLmPts[static_cast(s.fromLm)]); + const float d = len(dir); + if (d < 1e-5f) + dir = {0.f, 0.12f, 0.f}; + else + dir = mul(dir, 1.f / d); + restOffset[static_cast(s.role)] = mul(dir, std::max(d, 0.05f)); + } + + std::array worldRot{}; + worldRot.fill(Ogre::Quaternion::IDENTITY); + + for (int role = 0; role < PoseIK::kCanonicalRoles; ++role) { + if (!(resolvedMask & (1u << static_cast(role)))) + continue; + const int parent = MotionInbetween::canonicalParentOf(role); + const Ogre::Quaternion local = localArtic(quats, role, resolvedMask); + if (parent >= 0 && (resolvedMask & (1u << static_cast(parent)))) { + worldRot[static_cast(role)] = + worldRot[static_cast(parent)] * local; + out[static_cast(role)] = + add(out[static_cast(parent)], + mulVec3(worldRot[static_cast(parent)], + restOffset[static_cast(role)])); + } else if (role == PoseIK::Hip) { + worldRot[0] = quatFromArray(quats[0]); + } else { + worldRot[static_cast(role)] = + quatFromArray(quats[static_cast(role)]); + out[static_cast(role)] = + add(hip, restOffset[static_cast(role)]); + } + } +} + +} // namespace MocapPoseIkFk + +#endif // ENABLE_MOCAP +#endif // MOCAPPOSEIKFK_H diff --git a/src/OnnxRuntimeSettings.cpp b/src/OnnxRuntimeSettings.cpp index 8b259d9d0..f7aa687c9 100644 --- a/src/OnnxRuntimeSettings.cpp +++ b/src/OnnxRuntimeSettings.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -103,11 +104,8 @@ bool cudaProviderLibraryLoads() void OnnxRuntimeSettings::prepareRuntimeEnvironment() { #ifdef QTMESH_ONNX_GPU_BUILD - static bool done = false; - if (done) - return; - done = true; - prependLdLibraryPath(cudnnSearchPaths()); + static std::once_flag once; + std::call_once(once, []() { prependLdLibraryPath(cudnnSearchPaths()); }); #endif } From 57a44db9420bac4c9899787777ae062c023997b8 Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 4 Aug 2026 18:12:27 -0400 Subject: [PATCH 10/11] fix(mocap): use measured live fps and defer clip removal on bake failure Body recordings now derive keyframe timing from the measured preview rate (fallback 30 fps), and bakeRetargeterClip only removes an existing clip after neutral calibration succeeds. Co-authored-by: Cursor --- src/Mocap/MocapController.cpp | 5 +++- src/Mocap/MocapRecorder.cpp | 43 ++++++++++++++++------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index 504134e5f..6e10f3bb2 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -1248,7 +1248,10 @@ void MocapController::stopRecording() if (d->headEnabled && !d->headBone.isEmpty()) bopts.skipRolesMask = (1u << static_cast(PoseIK::Head)); - const int fps = 30; + const int fps = + (d->liveFps >= 5.0) + ? std::max(1, static_cast(std::lround(d->liveFps))) + : 30; auto* bcmd = new RecordBodyClipCommand(d->entityName, std::move(valid), fps, bopts); UndoManager::getSingleton()->push(bcmd); diff --git a/src/Mocap/MocapRecorder.cpp b/src/Mocap/MocapRecorder.cpp index 7b8105eff..739e3e68e 100644 --- a/src/Mocap/MocapRecorder.cpp +++ b/src/Mocap/MocapRecorder.cpp @@ -455,47 +455,44 @@ BodyRecordReport bakeRetargeterClip( report.error = QStringLiteral("retargeter: not a humanoid rig"); return report; } - if (skel->hasAnimation(clip)) - skel->removeAnimation(clip); - skel->reset(true); - std::map bindLocal; - for (unsigned short i = 0; i < skel->getNumBones(); ++i) - bindLocal[i] = skel->getBone(i)->getOrientation(); - bool neutralSet = false; + const BodyLiveFrame* neutralFrame = nullptr; for (const auto& frame : frames) { if (!frame.valid) continue; if ((frame.resolvedMask & kTorsoResolvedMask) != kTorsoResolvedMask) continue; - const auto q = frameQuats(frame); - if (useLandmarks) - rt.setNeutralReference(q, frame.resolvedMask, frame.world.data(), - frame.visibility.data()); - else - rt.setNeutralReference(q, frame.resolvedMask); - neutralSet = true; + neutralFrame = &frame; break; } - if (!neutralSet) { + if (!neutralFrame) { for (const auto& frame : frames) { if (!frame.valid) continue; - const auto q = frameQuats(frame); - if (useLandmarks) - rt.setNeutralReference(q, frame.resolvedMask, frame.world.data(), - frame.visibility.data()); - else - rt.setNeutralReference(q, frame.resolvedMask); - neutralSet = true; + neutralFrame = &frame; break; } } - if (!neutralSet) { + if (!neutralFrame) { report.error = QStringLiteral("no confident body frames to calibrate"); return report; } + if (skel->hasAnimation(clip)) + skel->removeAnimation(clip); + skel->reset(true); + std::map bindLocal; + for (unsigned short i = 0; i < skel->getNumBones(); ++i) + bindLocal[i] = skel->getBone(i)->getOrientation(); + + const auto neutralQuats = frameQuats(*neutralFrame); + if (useLandmarks) + rt.setNeutralReference(neutralQuats, neutralFrame->resolvedMask, + neutralFrame->world.data(), + neutralFrame->visibility.data()); + else + rt.setNeutralReference(neutralQuats, neutralFrame->resolvedMask); + const double dt = 1.0 / static_cast(fps); Ogre::Animation* anim = skel->createAnimation( clip, static_cast(dt * (frames.size() - 1))); From cb61e2c74e2b5cae61f1aeae34fa9a69f194eeae Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 4 Aug 2026 19:02:04 -0400 Subject: [PATCH 11/11] fix(mocap,onnx): address remaining PR review feedback Gate head-yaw correction on mirrored webcam preview, bake body clips with sample timestamps, dedupe MJPEG decode with a test, guard CollapsibleSection deferred loads, force CPU ONNX on aarch64, and init OnnxRuntimeSettings early. Co-authored-by: Cursor --- cmake/OnnxRuntime.cmake | 4 ++++ qml/CollapsibleSection.qml | 23 ++++++++++++++++++----- scripts/install-onnx-gpu-deps.sh | 5 +++++ src/CLIPipeline.cpp | 1 + src/Mocap/MocapController.cpp | 11 ++++++++--- src/Mocap/MocapLiveTypes.h | 1 + src/Mocap/MocapRecorder.cpp | 15 +++++++++++++-- src/Mocap/VideoFrameSource.cpp | 16 ++++++++++++++-- src/Mocap/VideoFrameSource.h | 2 ++ src/Mocap/VideoFrameSource_test.cpp | 17 +++++++++++++++++ src/OnnxRuntimeSettings_test.cpp | 20 ++++++++++++++++++++ src/main.cpp | 2 ++ 12 files changed, 105 insertions(+), 12 deletions(-) diff --git a/cmake/OnnxRuntime.cmake b/cmake/OnnxRuntime.cmake index 7e2b3b693..d76d76315 100644 --- a/cmake/OnnxRuntime.cmake +++ b/cmake/OnnxRuntime.cmake @@ -43,6 +43,10 @@ if(APPLE) set(_ort_fetch_name "qtmesh_onnxruntime") elseif(UNIX) if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + if(QTMESH_ONNX_GPU) + message(STATUS "QTMESH_ONNX_GPU: no GPU ONNX archive for Linux aarch64; using CPU ONNX Runtime") + set(QTMESH_ONNX_GPU OFF CACHE BOOL "" FORCE) + endif() set(_ort_archive "onnxruntime-linux-aarch64-${QTMESH_ONNX_VERSION}.tgz") set(_ort_sha256 "ae4fedbdc8c18d688c01306b4b50c63de3445cdf2dbd720e01a2fa3810b8106a") set(_ort_fetch_name "qtmesh_onnxruntime") diff --git a/qml/CollapsibleSection.qml b/qml/CollapsibleSection.qml index 9de62db22..dfed7ceea 100644 --- a/qml/CollapsibleSection.qml +++ b/qml/CollapsibleSection.qml @@ -70,15 +70,28 @@ Column { onLoaded: root.contentReady() } + property int _loadGeneration: 0 + onExpandedChanged: { - if (root.expanded) - Qt.callLater(function() { contentLoader.loadActive = true }) - else + if (root.expanded) { + const gen = ++root._loadGeneration + Qt.callLater(function() { + if (root.expanded && gen === root._loadGeneration) + contentLoader.loadActive = true + }) + } else { + ++root._loadGeneration contentLoader.loadActive = false + } } Component.onCompleted: { - if (root.expanded) - Qt.callLater(function() { contentLoader.loadActive = true }) + if (root.expanded) { + const gen = ++root._loadGeneration + Qt.callLater(function() { + if (root.expanded && gen === root._loadGeneration) + contentLoader.loadActive = true + }) + } } } diff --git a/scripts/install-onnx-gpu-deps.sh b/scripts/install-onnx-gpu-deps.sh index dd1ce9f3b..a0e65d2c1 100755 --- a/scripts/install-onnx-gpu-deps.sh +++ b/scripts/install-onnx-gpu-deps.sh @@ -30,6 +30,11 @@ if [[ ! -f "$DEST/nvidia/cudnn/lib/libcudnn.so.9" ]]; then exit 1 fi +if [[ ! -f "$DEST/nvidia/cublas/lib/libcublas.so.12" ]]; then + echo "cuBLAS install failed (expected under $DEST/nvidia/cublas/lib)." >&2 + exit 1 +fi + echo "Done. Rebuild with GPU ONNX Runtime if needed:" echo " cmake . -B build_local -DENABLE_ONNX=ON -DQTMESH_ONNX_GPU=ON" echo " cmake --build build_local --target QtMeshEditor -j4" diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 83a294550..821c21e47 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -1564,6 +1564,7 @@ int CLIPipeline::run(int argc, char* argv[]) QCoreApplication::setApplicationName("QtMeshEditor"); QCoreApplication::setApplicationVersion(QTMESHEDITOR_VERSION); OnnxRuntimeSettings::prepareRuntimeEnvironment(); + (void)OnnxRuntimeSettings::instance(); // Redirect stdout to stderr so Ogre/Qt debug output doesn't // pollute the CLI pipeline output (JSON, info text, etc.) diff --git a/src/Mocap/MocapController.cpp b/src/Mocap/MocapController.cpp index 6e10f3bb2..f705c3b6b 100644 --- a/src/Mocap/MocapController.cpp +++ b/src/Mocap/MocapController.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -212,6 +213,7 @@ public slots: fr.quats[r] = roleFilters[r].filter(fr.quats[r], frame.timeSec); body.valid = true; + body.timeSec = frame.timeSec; body.quats = fr.quats; body.resolvedMask = fr.resolvedMask; body.world = ps.world; @@ -258,6 +260,7 @@ struct MocapController::Impl { Ogre::Quaternion headRestoreLocal = Ogre::Quaternion::IDENTITY; bool headWasManuallyControlled = false; bool savedHeadSnapshot = false; + bool mirroredLivePreview = false; QHash savedWeights; // mesh target -> weight QStringList savedEnabledAnimations; bool savedSkipAnimStateUpdate = false; @@ -681,6 +684,8 @@ bool MocapController::beginPreviewWithLiveSource( return false; } d->camera = std::move(source); + d->mirroredLivePreview = + (dynamic_cast(d->camera.get()) != nullptr); // Determine what the selection can be driven with BEFORE downloading any // models — a ~30 MB face-model fetch is wasted if the mesh has no ARKit @@ -1070,9 +1075,9 @@ void MocapController::onSample(const FaceSample& sample, sample.headRotation[1], sample.headRotation[2]); Ogre::Quaternion delta = current * d->neutral.Inverse(); delta = MocapPoseFix::invertCameraPitchDelta(delta); - // Webcam preview is mirrored; FaceCap head yaw is opposite the rig - // unless corrected. Body uses landmark directions (no L/R swap). - delta = MocapPoseFix::invertCameraYawDelta(delta); + // Selfie/webcam preview is mirrored; video-file playback is not. + if (d->mirroredLivePreview) + delta = MocapPoseFix::invertCameraYawDelta(delta); const Ogre::Quaternion local = d->headBindWorld.Inverse() * delta * d->headBindWorld; Ogre::SkeletonInstance* skel = entity->getSkeleton(); diff --git a/src/Mocap/MocapLiveTypes.h b/src/Mocap/MocapLiveTypes.h index 464298fb5..62127bcd6 100644 --- a/src/Mocap/MocapLiveTypes.h +++ b/src/Mocap/MocapLiveTypes.h @@ -11,6 +11,7 @@ // One body frame marshalled from the inference worker to the main thread. struct BodyLiveFrame { bool valid = false; + double timeSec = 0.0; std::array, PoseIK::kCanonicalRoles> quats; uint32_t resolvedMask = 0; std::array world{}; diff --git a/src/Mocap/MocapRecorder.cpp b/src/Mocap/MocapRecorder.cpp index 739e3e68e..10fc9e22a 100644 --- a/src/Mocap/MocapRecorder.cpp +++ b/src/Mocap/MocapRecorder.cpp @@ -494,10 +494,17 @@ BodyRecordReport bakeRetargeterClip( rt.setNeutralReference(neutralQuats, neutralFrame->resolvedMask); const double dt = 1.0 / static_cast(fps); + const double t0 = frames.front().timeSec; + const double tLast = frames.back().timeSec; + const bool useSampleTimes = (tLast - t0) > 1e-6; + const double clipDuration = + useSampleTimes ? (tLast - t0) + : dt * static_cast(frames.size() > 1 ? frames.size() - 1 : 0); Ogre::Animation* anim = skel->createAnimation( - clip, static_cast(dt * (frames.size() - 1))); + clip, static_cast(clipDuration)); anim->setRotationInterpolationMode(Ogre::Animation::RIM_LINEAR); std::map tracks; + size_t frameIndex = 0; for (size_t f = 0; f < frames.size(); ++f) { const auto& frame = frames[f]; if (!frame.valid) @@ -507,6 +514,10 @@ BodyRecordReport bakeRetargeterClip( const float* vis = useLandmarks ? frame.visibility.data() : nullptr; const auto locals = rt.evaluateFrame(q, frame.resolvedMask, skipRolesMask, world, vis); + const double keyTime = + useSampleTimes ? (frame.timeSec - t0) + : dt * static_cast(frameIndex); + ++frameIndex; for (const auto& [handle, local] : locals) { auto it = tracks.find(handle); if (it == tracks.end()) @@ -514,7 +525,7 @@ BodyRecordReport bakeRetargeterClip( handle, anim->createNodeTrack(handle, skel->getBone(handle))) .first; auto* kf = it->second->createNodeKeyFrame( - static_cast(dt * f)); + static_cast(keyTime)); kf->setRotation(bindLocal[handle].Inverse() * local); } } diff --git a/src/Mocap/VideoFrameSource.cpp b/src/Mocap/VideoFrameSource.cpp index a4e3eaea0..2d1aca479 100644 --- a/src/Mocap/VideoFrameSource.cpp +++ b/src/Mocap/VideoFrameSource.cpp @@ -32,6 +32,17 @@ QImage mocapFrameToRgb888(const QImage& image) return image.convertToFormat(QImage::Format_RGB888); } +QImage mocapFrameFromJpegBytes(const QByteArray& jpeg) +{ + if (jpeg.isEmpty()) + return {}; + QImage img = QImage::fromData( + reinterpret_cast(jpeg.constData()), jpeg.size(), "JPEG"); + if (img.isNull()) + return {}; + return mocapFrameToRgb888(img); +} + QImage mocapFrameFromVideoFrame(const QVideoFrame& frame) { if (!frame.isValid()) @@ -45,8 +56,9 @@ QImage mocapFrameFromVideoFrame(const QVideoFrame& frame) // needs the qjpeg imageformat plugin (deploy.yml bundles it). if (mapped.pixelFormat() == QVideoFrameFormat::Format_Jpeg && mapped.planeCount() >= 1 && mapped.mappedBytes(0) > 0) { - img = QImage::fromData(mapped.bits(0), mapped.mappedBytes(0), - "JPEG"); + img = mocapFrameFromJpegBytes(QByteArray( + reinterpret_cast(mapped.bits(0)), + static_cast(mapped.mappedBytes(0)))); } if (img.isNull()) img = mapped.toImage(); diff --git a/src/Mocap/VideoFrameSource.h b/src/Mocap/VideoFrameSource.h index 1c9002406..28f4aa9e9 100644 --- a/src/Mocap/VideoFrameSource.h +++ b/src/Mocap/VideoFrameSource.h @@ -108,6 +108,8 @@ class FrameMailbox { QImage mocapFrameToRgb888(const QImage& image); // Decode a QVideoFrame to RGB888 (maps the buffer when toImage() alone fails). QImage mocapFrameFromVideoFrame(const QVideoFrame& frame); +// UVC MJPEG fallback when QVideoFrame::toImage() fails (needs qjpeg plugin). +QImage mocapFrameFromJpegBytes(const QByteArray& jpeg); class VideoFrameSource : public QObject { Q_OBJECT diff --git a/src/Mocap/VideoFrameSource_test.cpp b/src/Mocap/VideoFrameSource_test.cpp index af6e6511c..8e1466846 100644 --- a/src/Mocap/VideoFrameSource_test.cpp +++ b/src/Mocap/VideoFrameSource_test.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -209,4 +210,20 @@ TEST(MocapFrameFromVideoFrame, ConvertsQVideoFrameFromQImage) EXPECT_EQ(out.format(), QImage::Format_RGB888); } +TEST(MocapFrameFromJpegBytes, DecodesValidJpegPayload) +{ + QImage src(8, 8, QImage::Format_RGB888); + src.fill(QColor(255, 128, 64)); + QByteArray jpeg; + { + QBuffer buf(&jpeg); + buf.open(QIODevice::WriteOnly); + ASSERT_TRUE(src.save(&buf, "JPEG")); + } + const QImage out = mocapFrameFromJpegBytes(jpeg); + EXPECT_FALSE(out.isNull()); + EXPECT_EQ(out.format(), QImage::Format_RGB888); + EXPECT_EQ(out.size(), src.size()); +} + #endif // ENABLE_MOCAP diff --git a/src/OnnxRuntimeSettings_test.cpp b/src/OnnxRuntimeSettings_test.cpp index c3a9f3431..760a08015 100644 --- a/src/OnnxRuntimeSettings_test.cpp +++ b/src/OnnxRuntimeSettings_test.cpp @@ -5,8 +5,28 @@ #include #include +namespace { +struct ScopedEnvUnset { + const char* key; + QByteArray saved; + explicit ScopedEnvUnset(const char* k) : key(k) + { + saved = qgetenv(key); + qunsetenv(key); + } + ~ScopedEnvUnset() + { + if (saved.isNull()) + qunsetenv(key); + else + qputenv(key, saved); + } +}; +} // namespace + TEST(OnnxRuntimeSettings, PreferGpuPersists) { + ScopedEnvUnset guard("QTMESH_ONNX_PREFER_GPU"); QSettings settings; settings.remove("ai/onnxPreferGpu"); diff --git a/src/main.cpp b/src/main.cpp index eeacd6691..d5ebaf679 100755 --- a/src/main.cpp +++ b/src/main.cpp @@ -125,6 +125,7 @@ int main(int argc, char *argv[]) QCoreApplication::setApplicationName("QtMeshEditor"); QCoreApplication::setApplicationVersion(QTMESHEDITOR_VERSION); OnnxRuntimeSettings::prepareRuntimeEnvironment(); + (void)OnnxRuntimeSettings::instance(); // Initialize Sentry using stored consent (no dialog in headless mode) SentryReporter::initialize(); @@ -172,6 +173,7 @@ int main(int argc, char *argv[]) QApplication a(argc, argv); OnnxRuntimeSettings::prepareRuntimeEnvironment(); + (void)OnnxRuntimeSettings::instance(); // Capture qDebug/qWarning/etc. from the rest of startup into the in-app console // (MainWindow attaches and drains the backlog when its console exists).