Add metadata field to character (#920)

Summary:
Pull Request resolved: #920

Added a metadata field to the character.

Both name and metadata are now read as a user property from the skeleton root node and properly copied along with other values in the character.

The metadata field can be used to uniquely identify an asset or store other information about processing. It's limited to 65536 characters at the moment and on loading tested to be in json format.

Reviewed By: maxouellet

Differential Revision: D87269358

fbshipit-source-id: a47ceba5e86ff656a04cb6eb121de578def92c18

Author: cstollmeta

momentum/character/character.cpp

@@ -38,7 +38,8 @@ CharacterT<T>::CharacterT(
     BlendShapeBase_const_p faceExpressionBlendShapes,
     const std::string& nameIn,
     const momentum::TransformationList& inverseBindPose_in,
-    const SkinnedLocatorList& skinnedLocators)
+    const SkinnedLocatorList& skinnedLocators,
+    std::string_view metadataIn)
     : skeleton(s),
       parameterTransform(pt),
       parameterLimits(pl),
@@ -47,7 +48,8 @@ CharacterT<T>::CharacterT(
       blendShape(std::move(blendShapes)),
       faceExpressionBlendShape(std::move(faceExpressionBlendShapes)),
       inverseBindPose(inverseBindPose_in),
-      name(nameIn) {
+      name(nameIn),
+      metadata(metadataIn) {
   if (m) {
     mesh = std::make_unique<Mesh>(*m);
     // create skinweights copy only if both mesh and skinweights exist
@@ -87,7 +89,8 @@ CharacterT<T>::CharacterT(const CharacterT& c)
       faceExpressionBlendShape(c.faceExpressionBlendShape),
       inverseBindPose(c.inverseBindPose),
       jointMap(c.jointMap),
-      name(c.name) {
+      name(c.name),
+      metadata(c.metadata) {
   if (c.mesh) {
     mesh = std::make_unique<Mesh>(*c.mesh);
     // create skinweights copy only if both mesh and skinweights exist
@@ -136,6 +139,7 @@ CharacterT<T>& CharacterT<T>::operator=(const CharacterT& rhs) {
   std::swap(blendShape, tmp.blendShape);
   std::swap(faceExpressionBlendShape, tmp.faceExpressionBlendShape);
   std::swap(name, tmp.name);
+  std::swap(metadata, tmp.metadata);
 
   return *this;
 }
@@ -320,6 +324,8 @@ CharacterT<T> CharacterT<T>::simplifySkeleton(const std::vector<bool>& activeJoi
 
   // create character result
   CharacterT<T> result(simplifiedSkeleton, simplifiedTransform);
+  result.name = name;
+  result.metadata = metadata;
 
   result.jointMap = simplifiedJointMap;
 
@@ -387,7 +393,8 @@ CharacterT<T> CharacterT<T>::simplifyParameterTransform(const ParameterSet& para
       faceExpressionBlendShape,
       name,
       inverseBindPose,
-      skinnedLocators);
+      skinnedLocators,
+      metadata);
 }
 
 template <typename T>

momentum/character/character.h

@@ -70,6 +70,9 @@ struct CharacterT {
   /// Character identifier
   std::string name;
 
+  /// Metadata (as a JSON-serialized string)
+  std::string metadata;
+
   /// @}
 
   /// Default constructor
@@ -92,6 +95,8 @@ struct CharacterT {
   /// @param faceExpressionBlendShapes Optional facial expression blend shapes
   /// @param nameIn Optional character identifier
   /// @param inverseBindPose Optional inverse bind pose transformations
+  /// @param skinnedLocators Optional points of interest attached to joints, with skinning weights
+  /// @param metadataIn Optional metadata
   CharacterT(
       const Skeleton& s,
       const ParameterTransform& pt,
@@ -105,7 +110,8 @@ struct CharacterT {
       BlendShapeBase_const_p faceExpressionBlendShapes = {},
       const std::string& nameIn = "",
       const momentum::TransformationList& inverseBindPose = {},
-      const SkinnedLocatorList& skinnedLocators = {});
+      const SkinnedLocatorList& skinnedLocators = {},
+      std::string_view metadataIn = "");
 
   /// Copy constructor
   CharacterT(const CharacterT& c);

momentum/io/fbx/fbx_io.cpp

@@ -551,6 +551,16 @@ void saveBlendShapesToFbx(
   mesh->AddDeformer(blendShape);
 }
 
+void addMetaData(::fbxsdk::FbxNode* skeletonRootNode, const Character& character) {
+  // add metadata
+  if (skeletonRootNode != nullptr) {
+    ::fbxsdk::FbxProperty::Create(skeletonRootNode, ::fbxsdk::FbxStringDT, "metadata")
+        .Set(FbxString(character.metadata.c_str()));
+    ::fbxsdk::FbxProperty::Create(skeletonRootNode, ::fbxsdk::FbxStringDT, "name")
+        .Set(FbxString(character.name.c_str()));
+  }
+}
+
 void saveFbxCommon(
     const filesystem::path& filename,
     const Character& character,
@@ -658,6 +668,9 @@ void saveFbxCommon(
     }
   }
 
+  // add metadata
+  addMetaData(skeletonRootNode, character);
+
   // ---------------------------------------------
   // create the locator nodes
   // ---------------------------------------------

momentum/io/fbx/openfbx_loader.cpp

@@ -24,6 +24,7 @@
 #include "momentum/math/mesh.h"
 #include "momentum/math/utility.h"
 
+#include <nlohmann/json.hpp>
 #include <ofbx.h>
 #include <gsl/span_ext>
 
@@ -221,6 +222,20 @@ double resolveDoubleProperty(const ofbx::Object& object, const char* name) {
   }
 }
 
+std::string resolveStringProperty(const ofbx::Object& object, const char* name) {
+  const ofbx::IElement* element = resolveProperty(object, name);
+  MT_THROW_IF(element == nullptr, "Unable to find property element in {}", object.name);
+  const ofbx::IElementProperty* x = getElementProperty(element, 4);
+  MT_THROW_IF(x == nullptr, "Unable to find property {} in {}", name, object.name);
+  if (x->getType() == ofbx::IElementProperty::STRING) {
+    char result[65536];
+    x->getValue().toString(result);
+    return {result};
+  } else {
+    MT_THROW("For property {}, expected string but got {}.", name, propertyTypeStr(x->getType()));
+  }
+}
+
 template <typename VecArray, typename EltType>
 VecArray extractPropertyArrayImp(const ofbx::IElementProperty* prop, const char* what) {
   using VecType = typename VecArray::value_type;
@@ -1238,6 +1253,25 @@ std::tuple<Character, std::vector<MatrixXf>, float> loadOpenFbx(
   const auto [skeleton, jointFbxNodes, locators, collision] =
       parseSkeleton(scene->getRoot(), {}, permissive);
 
+  std::string name;
+  std::string metadata = "{}"; // empty json string
+  if (!jointFbxNodes.empty()) {
+    auto* resName = resolveProperty(*jointFbxNodes[0], "name");
+    if (resName != nullptr) {
+      name = resolveStringProperty(*jointFbxNodes[0], "name");
+    }
+    auto* resMeta = resolveProperty(*jointFbxNodes[0], "metadata");
+    if (resMeta != nullptr) {
+      metadata = resolveStringProperty(*jointFbxNodes[0], "metadata");
+    }
+  }
+  // ensure metadata is valid JSON
+  try {
+    auto json = nlohmann::json::parse(metadata);
+  } catch (const nlohmann::json::parse_error& e) {
+    MT_LOGW("Failed to parse metadata: {}", e.what());
+  }
+
   TransformationList inverseBindPoseTransforms;
   for (const auto& j : jointFbxNodes) {
     Eigen::Affine3d mat = Eigen::Affine3d::Identity();
@@ -1284,7 +1318,12 @@ std::tuple<Character, std::vector<MatrixXf>, float> loadOpenFbx(
       skinWeights.get(),
       collision.empty() ? nullptr : &collision,
       nullptr,
-      blendShape);
+      blendShape,
+      {}, // faceExpressionBlendShapes
+      name, // nameIn
+      {}, // inverseBindPose
+      {}, // skinnedLocators
+      metadata);
   result.resetJointMap();
   result.inverseBindPose = inverseBindPoseTransforms;
 

momentum/io/gltf/gltf_builder.cpp

@@ -926,6 +926,8 @@ void GltfBuilder::addCharacter(
     parameterLimitsToJson(character, def["parameterLimits"]);
     parameterSetsToJson(character, def["parameterSet"]);
     poseConstraintsToJson(character, def["poseConstraints"]);
+    // add metadata
+    def["metadata"] = character.metadata;
   }
 }
 

momentum/io/gltf/gltf_io.cpp

@@ -798,6 +798,15 @@ void loadGlobalExtensions(const fx::gltf::Document& model, Character& character)
     if (def.count("parameterLimits") > 0) {
       character.parameterLimits = parameterLimitsFromJson(character, def["parameterLimits"]);
     }
+    if (def.count("metadata") > 0) {
+      character.metadata = def.value<std::string>("metadata", {});
+      // ensure metadata is valid JSON
+      try {
+        auto json = nlohmann::json::parse(character.metadata);
+      } catch (const nlohmann::json::parse_error& e) {
+        MT_LOGW("Failed to parse metadata: {}", e.what());
+      }
+    }
   } catch (std::runtime_error& err) {
     MT_THROW("Unable to load gltf : {}", err.what());
   }

momentum/test/character/character_helpers.cpp

@@ -237,7 +237,8 @@ CharacterT<T> createTestCharacter(size_t numJoints) {
       BlendShapeBase_const_p{},
       std::string("test character"),
       momentum::TransformationList{},
-      createDefaultSkinnedLocatorList(skeleton));
+      createDefaultSkinnedLocatorList(skeleton),
+      std::string(R"({"name":"testCharacterV1","version":1})"));
 }
 
 template CharacterT<float> createTestCharacter(size_t numJoints);

momentum/test/character/character_test.cpp

@@ -182,6 +182,9 @@ TYPED_TEST(CharacterTest, CopyConstructor) {
     copiedCharacter.mesh->vertices[0] = Vector3f(99, 99, 99);
     EXPECT_NE(copiedCharacter.mesh->vertices[0], this->character.mesh->vertices[0]);
   }
+
+  // Check that we preserve metadata
+  EXPECT_EQ(copiedCharacter.metadata, this->character.metadata);
 }
 
 // Test assignment operator
@@ -227,6 +230,9 @@ TYPED_TEST(CharacterTest, AssignmentOperator) {
     assignedCharacter = temp;
   }
   EXPECT_EQ(assignedCharacter.name, "assigned");
+
+  // Check that we preserve metadata
+  EXPECT_EQ(assignedCharacter.metadata, this->character.metadata);
 }
 
 // Test move constructor
@@ -258,6 +264,9 @@ TYPED_TEST(CharacterTest, MoveConstructor) {
     const auto bindPose = movedCharacter.bindPose();
     EXPECT_EQ(bindPose.pose.size(), static_cast<Eigen::Index>(originalParamCount));
   }
+
+  // Check that we preserve metadata
+  EXPECT_EQ(movedCharacter.metadata, this->character.metadata);
 }
 
 // Test move assignment operator
@@ -543,6 +552,9 @@ TYPED_TEST(CharacterTest, SimplifySkeleton) {
 
   // Check that the locators were remapped
   EXPECT_EQ(simplifiedCharacter.locators.size(), this->character.locators.size());
+
+  // Check that we preserve metadata
+  EXPECT_EQ(simplifiedCharacter.metadata, this->character.metadata);
 }
 
 // Test simplifyParameterTransform method
@@ -573,6 +585,9 @@ TYPED_TEST(CharacterTest, SimplifyParameterTransform) {
   EXPECT_EQ(simplifiedCharacter.mesh->vertices.size(), this->character.mesh->vertices.size());
   EXPECT_EQ(
       simplifiedCharacter.skinWeights->index.rows(), this->character.skinWeights->index.rows());
+
+  // Check that we preserve metadata
+  EXPECT_EQ(simplifiedCharacter.metadata, this->character.metadata);
 }
 
 // Test simplify method
@@ -599,6 +614,9 @@ TYPED_TEST(CharacterTest, Simplify) {
   // EXPECT_EQ(simplifiedCharacter.parameterTransform.name[0], "root_tx");
   // EXPECT_EQ(simplifiedCharacter.parameterTransform.name[1], "root_rx");
   // EXPECT_EQ(simplifiedCharacter.parameterTransform.name[2], "joint1_rx");
+
+  // Check that we preserve metadata
+  EXPECT_EQ(simplifiedCharacter.metadata, this->character.metadata);
 }
 
 // Test remapSkinWeights method

pymomentum/geometry/character_pybind.cpp

@@ -69,6 +69,7 @@ void registerCharacterBindings(py::class_<mm::Character>& characterClass) {
   // =====================================================
   // momentum::Character
   // - name
+  // - metadata
   // - skeleton
   // - parameter_transform
   // - locators
@@ -176,7 +177,8 @@ void registerCharacterBindings(py::class_<mm::Character>& characterClass) {
                 character.faceExpressionBlendShape,
                 character.name,
                 character.inverseBindPose,
-                character.skinnedLocators);
+                character.skinnedLocators,
+                character.metadata);
           },
           "Adds mesh and skin weight to the character and return a new character instance",
           py::arg("mesh"),
@@ -197,7 +199,9 @@ void registerCharacterBindings(py::class_<mm::Character>& characterClass) {
                 character.blendShape,
                 character.faceExpressionBlendShape,
                 character.name,
-                character.inverseBindPose);
+                character.inverseBindPose,
+                character.skinnedLocators,
+                character.metadata);
           },
           "Returns a new character with the parameter limits set to the passed-in limits.",
           py::arg("parameter_limits"))
@@ -231,7 +235,8 @@ void registerCharacterBindings(py::class_<mm::Character>& characterClass) {
                 character.faceExpressionBlendShape,
                 character.name,
                 character.inverseBindPose,
-                character.skinnedLocators);
+                character.skinnedLocators,
+                character.metadata);
           },
           R"(Returns a new character with the passed-in locators.  If 'replace' is true, the existing locators are replaced, otherwise (the default) the new locators are appended to the existing ones.
 
@@ -282,7 +287,8 @@ void registerCharacterBindings(py::class_<mm::Character>& characterClass) {
                 character.faceExpressionBlendShape,
                 character.name,
                 character.inverseBindPose,
-                combinedSkinnedLocators);
+                combinedSkinnedLocators,
+                character.metadata);
           },
           R"(Returns a new character with the passed-in skinned locators.  If 'replace' is true, the existing skinned locators are replaced, otherwise (the default) the new skinned locators are appended to the existing ones.
 
@@ -292,6 +298,7 @@ void registerCharacterBindings(py::class_<mm::Character>& characterClass) {
           py::arg("skinned_locators"),
           py::arg("replace") = false)
       .def_readonly("name", &mm::Character::name, "The character's name.")
+      .def_readonly("metadata", &mm::Character::metadata, "The character's metadata.")
       .def_readonly(
           "skeleton", &mm::Character::skeleton, "The character's skeleton. See :class:`Skeleton`.")
       .def_readonly(
@@ -401,7 +408,9 @@ It can be used to solve for facial expressions.
                 c.blendShape,
                 c.faceExpressionBlendShape,
                 c.name,
-                c.inverseBindPose);
+                c.inverseBindPose,
+                c.skinnedLocators,
+                c.metadata);
           },
           "Returns a new :class:`Character` with the collision geometry replaced.")
       .def(