Mocap Slice B (#871): ENABLE_MOCAP build infra + VideoFrameSource (Qt Multimedia)

.github/workflows/deploy.yml

@@ -958,6 +958,8 @@ jobs:
         host: 'linux'
         target: 'desktop'
         arch: 'linux_gcc_64'
+        # qtmultimedia: performance capture (ENABLE_MOCAP, epic #869)
+        modules: 'qtmultimedia'
 
     - name: change folder permissions
       run: |
@@ -1074,7 +1076,8 @@ jobs:
             -DBUILD_QT_MESH_EDITOR=OFF \
             -DENABLE_SENTRY=OFF \
             -DENABLE_PS1_RIP=ON \
-            -DENABLE_ONNX=ON
+            -DENABLE_ONNX=ON \
+            -DENABLE_MOCAP=ON
 
     - name: Run build-wrapper
       env:

CMakeLists.txt

@@ -324,6 +324,29 @@ else()
     message(STATUS "Alembic vertex-animation import disabled (VAT_POSE playback still available)")
 endif()
 ##############################################################
+#  Performance capture — video/webcam mocap (epic #869)
+##############################################################
+# Slice B (#871): brings in Qt Multimedia (camera + video decode) and the
+# src/Mocap/ frame-source layer. Default OFF: Qt Multimedia is a new runtime
+# dependency (FFmpeg backend) that packaging has to carry per platform. The
+# ONNX predictors (Slices C/E) additionally require ENABLE_ONNX; requiring it
+# here keeps a single "mocap build" configuration instead of a half-working
+# one. Non-mocap builds print "rebuild with -DENABLE_MOCAP" on every surface.
+option(ENABLE_MOCAP "Enable performance capture (video/webcam mocap)" OFF)
+
+if(ENABLE_MOCAP)
+    if(NOT ENABLE_ONNX)
+        message(FATAL_ERROR "ENABLE_MOCAP requires ENABLE_ONNX (the face/pose "
+                "predictors run on ONNX Runtime). Configure with "
+                "-DENABLE_ONNX=ON -DENABLE_MOCAP=ON.")
+    endif()
+    find_package(Qt6 REQUIRED COMPONENTS Multimedia)
+    add_definitions(-DENABLE_MOCAP)
+    message(STATUS "Performance capture enabled (Qt Multimedia ${Qt6Multimedia_VERSION})")
+else()
+    message(STATUS "Performance capture disabled (rebuild with -DENABLE_MOCAP=ON -DENABLE_ONNX=ON)")
+endif()
+##############################################################
 #  PS1 runtime geometry extraction (experimental)
 ##############################################################
 option(ENABLE_PS1_RIP "Enable experimental PS1 runtime geometry extraction" OFF)

src/CMakeLists.txt

@@ -161,6 +161,7 @@ NodeAnimationManager.cpp
 PoseLibrary.cpp
 VertexAnimationManager.cpp
 AlembicImporter.cpp
+Mocap/VideoFrameSource.cpp
 ApplyAtlas.cpp
 EmbeddedTextureCache.cpp
 NormalMapGenerator.cpp
@@ -753,6 +754,11 @@ if(ENABLE_PS1_RIP AND TARGET Qt6::Gamepad)
     target_link_libraries(${CMAKE_PROJECT_NAME} Qt6::Gamepad)
 endif()
 
+# Performance capture (epic #869): Qt Multimedia for the frame sources
+if(ENABLE_MOCAP)
+    target_link_libraries(${CMAKE_PROJECT_NAME} Qt6::Multimedia)
+endif()
+
 if(ENABLE_AUTO_UPDATER AND TARGET qtmesh-relauncher)
     add_dependencies(${CMAKE_PROJECT_NAME} qtmesh-relauncher)
     if(APPLE)
@@ -898,6 +904,11 @@ if(BUILD_TESTS)
 
     ADD_DEPENDENCIES(UnitTests ui)
 
+    # Performance capture (epic #869): Qt Multimedia for the frame sources
+    if(ENABLE_MOCAP)
+        target_link_libraries(UnitTests Qt6::Multimedia)
+    endif()
+
     if(ENABLE_PS1_RIP)
         add_dependencies(UnitTests qtmesh_ps1core_stub)
         if(TARGET qtmesh_ps1core_libretro)

src/Info.plist.in

@@ -43,6 +43,9 @@
 	<key>NSSupportsAutomaticGraphicsSwitching</key>
 	<true/>
 
+	<key>NSCameraUsageDescription</key>
+	<string>QtMeshEditor uses the camera for live performance capture.</string>
+
 	<key>CFBundleDocumentTypes</key>
 	<array>
 		<dict>

src/Mocap/VideoFrameSource.cpp

@@ -0,0 +1,282 @@
+#ifdef ENABLE_MOCAP
+
+#include "VideoFrameSource.h"
+
+#include <QCamera>
+#include <QCameraDevice>
+#include <QElapsedTimer>
+#include <QFileInfo>
+#include <QMediaCaptureSession>
+#include <QMediaDevices>
+#include <QMediaMetaData>
+#include <QMediaPlayer>
+#include <QUrl>
+#include <QVideoFrame>
+#include <QVideoSink>
+
+namespace {
+// Queued frameReady connections (capture thread -> worker) need the metatype.
+struct MocapMetaTypeRegistrar {
+    MocapMetaTypeRegistrar() { qRegisterMetaType<MocapFrame>("MocapFrame"); }
+};
+const MocapMetaTypeRegistrar registrar;
+}  // namespace
+
+QImage mocapFrameToRgb888(const QImage& image)
+{
+    if (image.format() == QImage::Format_RGB888)
+        return image;
+    return image.convertToFormat(QImage::Format_RGB888);
+}
+
+// ---------------------------------------------------------------------------
+// ImageSequenceFrameSource
+// ---------------------------------------------------------------------------
+
+ImageSequenceFrameSource::ImageSequenceFrameSource(const QStringList& imagePaths,
+                                                   double fps, double targetFps,
+                                                   QObject* parent)
+    : VideoFrameSource(parent),
+      m_paths(imagePaths),
+      m_fps(fps > 0.0 ? fps : 30.0),
+      m_decimator(targetFps)
+{
+}
+
+bool ImageSequenceFrameSource::open(QString* error)
+{
+    if (m_paths.isEmpty()) {
+        if (error) *error = tr("image sequence is empty");
+        return false;
+    }
+    for (const QString& p : m_paths) {
+        if (!QFileInfo::exists(p)) {
+            if (error) *error = tr("image not found: %1").arg(p);
+            return false;
+        }
+    }
+    return true;
+}
+
+void ImageSequenceFrameSource::start()
+{
+    m_stopped = false;
+    m_decimator.reset();
+    for (qint64 i = 0; i < m_paths.size() && !m_stopped; ++i) {
+        const double t = static_cast<double>(i) / m_fps;
+        if (!m_decimator.shouldEmit(t))
+            continue;
+        QImage img(m_paths.at(i));
+        if (img.isNull()) {
+            emit errorOccurred(tr("failed to load image: %1").arg(m_paths.at(i)));
+            return;
+        }
+        MocapFrame frame;
+        frame.image = mocapFrameToRgb888(img);
+        frame.timeSec = t;
+        frame.frameIndex = i;
+        emit frameReady(frame);
+    }
+    if (!m_stopped)
+        emit finished();
+}
+
+void ImageSequenceFrameSource::stop()
+{
+    m_stopped = true;
+}
+
+// ---------------------------------------------------------------------------
+// FileFrameSource
+// ---------------------------------------------------------------------------
+
+FileFrameSource::FileFrameSource(const QString& filePath, double targetFps,
+                                 QObject* parent)
+    : VideoFrameSource(parent), m_path(filePath), m_decimator(targetFps)
+{
+}
+
+FileFrameSource::~FileFrameSource()
+{
+    stop();
+}
+
+bool FileFrameSource::open(QString* error)
+{
+    if (!QFileInfo::exists(m_path)) {
+        if (error) *error = tr("video file not found: %1").arg(m_path);
+        return false;
+    }
+    m_player = std::make_unique<QMediaPlayer>();
+    m_sink = std::make_unique<QVideoSink>();
+    m_player->setVideoSink(m_sink.get());
+
+    connect(m_sink.get(), &QVideoSink::videoFrameChanged, this,
+            &FileFrameSource::handleVideoFrame);
+    connect(m_player.get(), &QMediaPlayer::mediaStatusChanged, this,
+            [this](QMediaPlayer::MediaStatus status) {
+                if (status == QMediaPlayer::EndOfMedia && !m_finishedEmitted) {
+                    m_finishedEmitted = true;
+                    emit finished();
+                }
+                if (status == QMediaPlayer::LoadedMedia) {
+                    const auto rate = m_player->metaData()
+                                          .value(QMediaMetaData::VideoFrameRate);
+                    if (rate.isValid())
+                        m_nativeFps = rate.toDouble();
+                }
+            });
+    connect(m_player.get(), &QMediaPlayer::errorOccurred, this,
+            [this](QMediaPlayer::Error, const QString& message) {
+                emit errorOccurred(tr("video decode error: %1").arg(message));
+            });
+
+    m_player->setSource(QUrl::fromLocalFile(m_path));
+    return true;
+}
+
+void FileFrameSource::start()
+{
+    if (!m_player) {
+        emit errorOccurred(tr("start() before open()"));
+        return;
+    }
+    m_decimator.reset();
+    m_frameIndex = 0;
+    m_finishedEmitted = false;
+    m_player->play();
+}
+
+void FileFrameSource::stop()
+{
+    if (m_player)
+        m_player->stop();
+}
+
+void FileFrameSource::handleVideoFrame()
+{
+    const QVideoFrame vf = m_sink->videoFrame();
+    if (!vf.isValid())
+        return;
+    const qint64 index = m_frameIndex++;
+    // Prefer the frame's own timestamp; fall back to the player clock.
+    double t = vf.startTime() >= 0
+                   ? vf.startTime() / 1e6
+                   : (m_player ? m_player->position() / 1e3 : 0.0);
+    if (!m_decimator.shouldEmit(t))
+        return;
+    MocapFrame frame;
+    frame.image = mocapFrameToRgb888(vf.toImage());
+    frame.timeSec = t;
+    frame.frameIndex = index;
+    if (!frame.image.isNull())
+        emit frameReady(frame);
+}
+
+// ---------------------------------------------------------------------------
+// CameraFrameSource
+// ---------------------------------------------------------------------------
+
+QList<CameraFrameSource::DeviceInfo> CameraFrameSource::availableDevices()
+{
+    QList<DeviceInfo> out;
+    const auto devices = QMediaDevices::videoInputs();
+    for (const QCameraDevice& d : devices)
+        out.append({QString::fromUtf8(d.id()), d.description()});
+    return out;
+}
+
+CameraFrameSource::CameraFrameSource(const QString& deviceId, QObject* parent)
+    : VideoFrameSource(parent), m_deviceId(deviceId)
+{
+}
+
+CameraFrameSource::~CameraFrameSource()
+{
+    stop();
+}
+
+bool CameraFrameSource::open(QString* error)
+{
+    QCameraDevice device;
+    const auto devices = QMediaDevices::videoInputs();
+    if (m_deviceId.isEmpty()) {
+        device = QMediaDevices::defaultVideoInput();
+    } else {
+        for (const QCameraDevice& d : devices) {
+            if (QString::fromUtf8(d.id()) == m_deviceId) {
+                device = d;
+                break;
+            }
+        }
+    }
+    if (device.isNull()) {
+        if (error)
+            *error = devices.isEmpty()
+                         ? tr("no camera available")
+                         : tr("camera not found: %1").arg(m_deviceId);
+        return false;
+    }
+
+    m_camera = std::make_unique<QCamera>(device);
+    m_session = std::make_unique<QMediaCaptureSession>();
+    m_sink = std::make_unique<QVideoSink>();
+    m_session->setCamera(m_camera.get());
+    m_session->setVideoSink(m_sink.get());
+    m_clock = std::make_unique<QElapsedTimer>();
+
+    const auto formats = device.videoFormats();
+    if (!formats.isEmpty())
+        m_nativeFps = formats.first().maxFrameRate();
+
+    connect(m_sink.get(), &QVideoSink::videoFrameChanged, this,
+            &CameraFrameSource::handleVideoFrame);
+    connect(m_camera.get(), &QCamera::errorOccurred, this,
+            [this](QCamera::Error err, const QString& message) {
+                if (err == QCamera::CameraError && message.contains(
+                        QStringLiteral("permission"), Qt::CaseInsensitive)) {
+                    emit errorOccurred(tr("camera permission denied — allow "
+                                          "camera access for QtMeshEditor in "
+                                          "the system settings"));
+                } else {
+                    emit errorOccurred(tr("camera error: %1").arg(message));
+                }
+            });
+    return true;
+}
+
+void CameraFrameSource::start()
+{
+    if (!m_camera) {
+        emit errorOccurred(tr("start() before open()"));
+        return;
+    }
+    m_frameIndex = 0;
+    m_clock->start();
+    m_camera->start();
+}
+
+void CameraFrameSource::stop()
+{
+    if (m_camera)
+        m_camera->stop();
+}
+
+void CameraFrameSource::handleVideoFrame()
+{
+    const QVideoFrame vf = m_sink->videoFrame();
+    if (!vf.isValid())
+        return;
+    MocapFrame frame;
+    frame.image = mocapFrameToRgb888(vf.toImage());
+    if (frame.image.isNull())
+        return;
+    frame.timeSec = m_clock->isValid() ? m_clock->elapsed() / 1e3 : 0.0;
+    frame.frameIndex = m_frameIndex++;
+    // Latest-wins for the inference consumer; the signal serves lightweight
+    // observers (preview HUD) that keep up with the camera.
+    m_mailbox.put(frame);
+    emit frameReady(frame);
+}
+
+#endif  // ENABLE_MOCAP