From 1cf6edc6f7ecebf8ec3c3eb77de337dd66181627 Mon Sep 17 00:00:00 2001 From: Cedrik Fuoco Date: Fri, 30 Sep 2022 12:40:53 -0400 Subject: [PATCH 1/8] OCIOZ archive feature (#1627) Signed-off-by: Cedrik Fuoco --- include/OpenColorIO/OpenColorIO.h | 200 +++++- include/OpenColorIO/OpenColorTypes.h | 11 + share/cmake/modules/FindExtPackages.cmake | 5 + share/cmake/modules/Findminizip-ng.cmake | 226 ++++++ share/cmake/modules/GetZLIB.cmake | 140 ++++ src/OpenColorIO/CMakeLists.txt | 2 + src/OpenColorIO/Config.cpp | 195 ++++- src/OpenColorIO/Context.cpp | 21 +- src/OpenColorIO/OCIOYaml.cpp | 15 +- src/OpenColorIO/OCIOZArchive.cpp | 679 ++++++++++++++++++ src/OpenColorIO/OCIOZArchive.h | 110 +++ src/OpenColorIO/PathUtils.cpp | 24 +- src/OpenColorIO/PathUtils.h | 4 +- src/OpenColorIO/Platform.cpp | 15 +- src/OpenColorIO/Platform.h | 8 + src/OpenColorIO/SystemMonitor_windows.cpp | 4 +- src/OpenColorIO/transforms/CDLTransform.cpp | 6 +- src/OpenColorIO/transforms/FileTransform.cpp | 110 ++- src/OpenColorIO/transforms/FileTransform.h | 3 +- src/apps/CMakeLists.txt | 1 + src/apps/ocioarchive/CMakeLists.txt | 27 + src/apps/ocioarchive/main.cpp | 288 ++++++++ src/apps/ociocheck/main.cpp | 5 + src/bindings/python/CMakeLists.txt | 1 + src/bindings/python/PyConfig.cpp | 20 +- src/bindings/python/PyConfigIOProxy.cpp | 64 ++ src/bindings/python/PyOpenColorIO.cpp | 1 + src/bindings/python/PyOpenColorIO.h | 1 + tests/cpu/CMakeLists.txt | 3 + tests/cpu/Config_tests.cpp | 265 +++++++ tests/cpu/OCIOZArchive_tests.cpp | 577 +++++++++++++++ tests/cpu/UnitTestUtils.cpp | 112 ++- tests/cpu/UnitTestUtils.h | 15 + .../files/configs/context_test1/config.ocio | 113 +++ .../context_test1/context_test1_linux.ocioz | Bin 0 -> 3290 bytes .../context_test1/context_test1_windows.ocioz | Bin 0 -> 3938 bytes .../files/configs/context_test1/looks.cdl | 42 ++ .../data/files/configs/context_test1/lut1.clf | 10 + .../configs/context_test1/shot1/lut1.clf | 10 + .../configs/context_test1/shot2/lut1.clf | 10 + .../configs/context_test1/shot2/lut2.clf | 10 + .../configs/context_test1/shot3/lut1.clf | 10 + .../context_test1/shot3/subdir/lut3.clf | 10 + .../configs/context_test1/shot4/lut1.clf | 10 + .../configs/context_test1/shot4/lut4.clf | 10 + .../config_missing_luts.ocioz | Bin 0 -> 924 bytes .../configs/ocioz_archive_configs/empty.ocioz | Bin 0 -> 22 bytes .../missing_config.ocioz | Bin 0 -> 3245 bytes tests/python/ConfigTest.py | 180 ++++- tests/python/OCIOZArchiveTest.py | 399 ++++++++++ tests/python/OpenColorIOTestSuite.py | 2 + 51 files changed, 3903 insertions(+), 71 deletions(-) create mode 100644 share/cmake/modules/Findminizip-ng.cmake create mode 100644 share/cmake/modules/GetZLIB.cmake create mode 100644 src/OpenColorIO/OCIOZArchive.cpp create mode 100644 src/OpenColorIO/OCIOZArchive.h create mode 100644 src/apps/ocioarchive/CMakeLists.txt create mode 100644 src/apps/ocioarchive/main.cpp create mode 100644 src/bindings/python/PyConfigIOProxy.cpp create mode 100644 tests/cpu/OCIOZArchive_tests.cpp create mode 100644 tests/data/files/configs/context_test1/config.ocio create mode 100644 tests/data/files/configs/context_test1/context_test1_linux.ocioz create mode 100644 tests/data/files/configs/context_test1/context_test1_windows.ocioz create mode 100644 tests/data/files/configs/context_test1/looks.cdl create mode 100644 tests/data/files/configs/context_test1/lut1.clf create mode 100644 tests/data/files/configs/context_test1/shot1/lut1.clf create mode 100644 tests/data/files/configs/context_test1/shot2/lut1.clf create mode 100644 tests/data/files/configs/context_test1/shot2/lut2.clf create mode 100644 tests/data/files/configs/context_test1/shot3/lut1.clf create mode 100644 tests/data/files/configs/context_test1/shot3/subdir/lut3.clf create mode 100644 tests/data/files/configs/context_test1/shot4/lut1.clf create mode 100644 tests/data/files/configs/context_test1/shot4/lut4.clf create mode 100644 tests/data/files/configs/ocioz_archive_configs/config_missing_luts.ocioz create mode 100644 tests/data/files/configs/ocioz_archive_configs/empty.ocioz create mode 100644 tests/data/files/configs/ocioz_archive_configs/missing_config.ocioz create mode 100644 tests/python/OCIOZArchiveTest.py diff --git a/include/OpenColorIO/OpenColorIO.h b/include/OpenColorIO/OpenColorIO.h index 89f58cc08e..8d98e89439 100644 --- a/include/OpenColorIO/OpenColorIO.h +++ b/include/OpenColorIO/OpenColorIO.h @@ -10,13 +10,14 @@ #include #include #include +#include +#include #include "OpenColorABI.h" #include "OpenColorTypes.h" #include "OpenColorTransforms.h" #include "OpenColorAppHelpers.h" - /* C++ API @@ -174,6 +175,8 @@ extern OCIOEXPORT void LogMessage(LoggingLevel level, const char * message); /** * \brief Set the Compute Hash Function to use; otherwise, use the default. * + * This is not used when using CreateFromFile with an OCIOZ archive or CreateFromConfigProxyIO. + * * \param ComputeHashFunction */ extern OCIOEXPORT void SetComputeHashFunction(ComputeHashFunction hashFunction); @@ -201,6 +204,26 @@ extern OCIOEXPORT ConstConfigRcPtr GetCurrentConfig(); /// Set the current configuration. This will then store a copy of the specified config. extern OCIOEXPORT void SetCurrentConfig(const ConstConfigRcPtr & config); +/** + * \brief Extract an OCIO Config archive. + * + * Converts an archived config file (.ocioz file) back to its original form as a config file + * and associated LUT files. This creates destinationDir and then creates a config.ocio file + * at the root of that working directory and then unpacks the LUT files into their relative + * locations relative to that working directory, creating any necessary sub-directories in the + * process. Note that configs which contain LUT files outside the working directory are not + * archivable, and so this function will not create directories outside the working directory. + * + * \param archivePath Absolute path to the .ocioz file. + * \param destinationDir Absolute path of the directory you want to be created to contain the + * contents of the unarchived config. + * \throw Exception If the archive is not found or there is a problem extracting it. + */ +extern OCIOEXPORT void ExtractOCIOZArchive( + const char * archivePath, + const char * destinationDir +); + /** * \brief * A config defines all the color spaces to be available at runtime. @@ -245,49 +268,89 @@ class OCIOEXPORT Config * \brief Create an empty config of the current version. * * Note that an empty config will not pass validation since required elements will be missing. + * \return The Config object. */ static ConfigRcPtr Create(); + /** * \brief Create a fall-back config. * * This may be useful to allow client apps to launch in cases when the * supplied config path is not loadable. + * \return The Config object. */ static ConstConfigRcPtr CreateRaw(); + /** * \brief Create a configuration using the OCIO environment variable. * - * Also support OCIO URI format. See CreateFromFile. + * Also supports the OCIO URI format for Built-in configs and supports archived configs. + * See \ref Config::CreateFromFile. * * If the variable is missing or empty, returns the same result as - * \ref Config::CreateRaw . + * \ref Config::CreateRaw. + * \return The Config object. */ static ConstConfigRcPtr CreateFromEnv(); + /** * \brief Create a configuration using a specific config file. * - * Also support the following OCIO URI format : + * Also supports the following OCIO URI format for Built-in configs: * "ocio://default" - Default Built-in config. - * "ocio://configName" - Built-in config named configName - * + * "ocio://" - A specific Built-in config. For the list of available + * strings, see \ref Config::CreateFromBuiltinConfig. + * + * Also supports archived configs (.ocioz files). + * + * \throw Exception If the file may not be read or does not parse. + * \return The Config object. */ static ConstConfigRcPtr CreateFromFile(const char * filename); - /// Create a configuration using a stream. + + /** + * \brief Create a configuration using a stream. + * + * Note that CreateFromStream does not set the working directory so the caller would need to + * set that separately in order to resolve FileTransforms. This function is typically only + * used for self-contained configs (no LUTs). + * + * Configs created from CreateFromStream can not be archived unless the working directory is + * set and contains any necessary LUT files. + * + * \param istream Stream to the config. + * \throw Exception If the stream does not parse. + * \return The Config object. + */ static ConstConfigRcPtr CreateFromStream(std::istream & istream); + /** + * \brief Create a config from the supplied ConfigIOProxy object. This allows the calling + * program to directly provide the config and associated LUTs rather than reading them from + * the standard file system. + * + * See the \ref ConfigIOProxy class documentation for more info. + * + * \param ciop ConfigIOProxy object providing access to the config's files. + * \throw Exception If the config may not be read from the proxy, or does not parse. + * \return The Config object. + */ + static ConstConfigRcPtr CreateFromConfigIOProxy(ConfigIOProxyRcPtr ciop); + /** * \brief Create a configuration using an OCIO built-in config. * - * \param configName Built-in config name + * \param configName Built-in config name. * * The available configNames are: * "cg-config-v0.1.0_aces-v1.3_ocio-v2.1.1" -- ACES CG config, basic color spaces for computer - * graphics apps. More information is available at: + * graphics apps. More information about these configs is available at: * %https://github.com/AcademySoftwareFoundation/OpenColorIO-Config-ACES * - * \throw Exception If the configName is not recognized. + * Information about the available configs is available from the \ref BuiltinConfigRegistry. * - * \return one of the configs built into OCIO library + * \throw Exception If the configName is not recognized. + * \return One of the configs built into the OCIO library. */ static ConstConfigRcPtr CreateFromBuiltinConfig(const char * configName); @@ -1149,6 +1212,62 @@ class OCIOEXPORT Config const char * dstColorSpaceName, const char * dstInterchangeName); + /// Set the ConfigIOProxy object used to provision the config and LUTs from somewhere other + /// than the file system. (This is set on the config's embedded Context object.) + void setConfigIOProxy(ConfigIOProxyRcPtr ciop); + ConfigIOProxyRcPtr getConfigIOProxy() const; + + /** + * \brief Verify if the config is archivable. + * + * A config is not archivable if any of the following are true: + * -- The working directory is not set + * -- It contains FileTransforms with a src outside the working directory + * -- The search path contains paths outside the working directory + * -- The search path contains paths that start with a context variable + * + * Context variables are allowed but the intent is that they may only resolve to paths that + * are within or below the working directory. This is because the archiving function will + * only archive files that are within the working directory in order to ensure that if it is + * later expanded, that it will not create any files outside this directory. + * + * For example, a context variable on the search path intended to contain the name of a + * sub-directory under the working directory must have the form "./$DIR_NAME" rather than just + * "$DIR_NAME" to be considered archivable. This is imperfect since there is no way to + * prevent the context variable from creating a path outside the working dir, but it should + * at least draw attention to the fact that the archive would fail if used with context vars + * that try to abuse the intended functionality. + * + * \return bool Archivable if true. + */ + bool isArchivable() const; + + /** + * \brief Archive the config and its LUTs into the specified output stream. + * + * The config is archived by serializing the Config object into a file named "config.ocio" and + * then walking through the current working directory and any sub-directories. Any files that + * have an extension matching a supported LUT file format are added to the archive. Any files + * that do not have an extension (or have some unsupported LUT extension, including .ocio), + * will not be added to the archive. To reiterate, it is the in-memory Config object that is + * archived, and not any .ocio file in the current working directory. The directory structure + * relative to the working directory is preserved. No files outside the working directory are + * archived so that if it is later expanded, no files will be created outside the working dir. + * + * The reason the archive is created using all supported LUT file extensions rather than by + * trying to resolve all the FileTransforms in the Config to specific files is because of the + * goal to allow context variables to continue to work. + * + * If a Config is created with CreateFromStream, CreateFromFile with an OCIOZ archive, or + * CreateFromConfigIOProxy, it cannot be archived unless the working directory is manually set + * to a directory that contains any necessary LUT files. + * + * The provided output stream must be closed by the caller, if necessary (e.g., an ofstream). + * + * \param ostream The output stream to write to. + */ + void archive(std::ostream & ostream) const; + Config(const Config &) = delete; Config& operator= (const Config &) = delete; @@ -3269,6 +3388,11 @@ class OCIOEXPORT Context /// used to resolve the filename (empty if no context variables were used). const char * resolveFileLocation(const char * filename, ContextRcPtr & usedContextVars) const; + /// Set the ConfigIOProxy object used to provision the config and LUTs from somewhere other + /// than the file system. + void setConfigIOProxy(ConfigIOProxyRcPtr ciop); + ConfigIOProxyRcPtr getConfigIOProxy() const; + Context(const Context &) = delete; Context& operator= (const Context &) = delete; /// Do not use (needed only for pybind11). @@ -3437,6 +3561,60 @@ class OCIOEXPORT SystemMonitors virtual ~SystemMonitors() = default; }; + +/////////////////////////////////////////////////////////////////////////// +// ConfigIOProxy + +/** + * ConfigIOProxy is a proxy class to allow the calling program to supply the config and any + * associated LUT files directly, without relying on the standard file system. + * + * The OCIOZ archive feature is implemented using this mechanism. + */ +class OCIOEXPORT ConfigIOProxy +{ +public: + ConfigIOProxy() = default; + virtual ~ConfigIOProxy() = default; + + /** + * \brief Provide the contents of a LUT file as a buffer of uint8_t data. + * + * \param buffer Contents of the LUT file. + * \param filepath Fully resolved path to the "file." + * + * The file path is based on the Config's current working directory and is the same absolute + * path that would have been provided to the file system. + */ + virtual void getLutData(std::vector & buffer, const char * filepath) const = 0; + + /** + * \brief Provide the config file Yaml to be parsed. + * + * \return String with the config Yaml. + */ + virtual const std::string getConfigData() const = 0; + + /** + * \brief Provide a fast unique ID for a LUT file. + * + * This is intended to supply the string that will be used in OCIO's FileCacheMap. + * This should be highly performant and typically should not require extensive + * computation such as calculating the md5 hash of the file, unless it is pre-computed. + * + * If the "file" does not exist, in other words, if the proxy is unable to supply the requested + * file contents, the function must return an empty string. + * + * \param filepath Fully resolve path to the "file." + * + * The file path is based on the Config's current working directory and is the same absolute + * path that would have been provided to the file system. + * + * \return The file hash string. + */ + virtual const std::string getFastLutFileHash(const char * filepath) const = 0; +}; + } // namespace OCIO_NAMESPACE #endif // INCLUDED_OCIO_OPENCOLORIO_H diff --git a/include/OpenColorIO/OpenColorTypes.h b/include/OpenColorIO/OpenColorTypes.h index 8b6852a97c..9976ac821d 100644 --- a/include/OpenColorIO/OpenColorTypes.h +++ b/include/OpenColorIO/OpenColorTypes.h @@ -114,6 +114,10 @@ class OCIOEXPORT GradingRGBCurve; typedef OCIO_SHARED_PTR ConstGradingRGBCurveRcPtr; typedef OCIO_SHARED_PTR GradingRGBCurveRcPtr; +class OCIOEXPORT ConfigIOProxy; +typedef OCIO_SHARED_PTR ConstConfigIOProxyRcPtr; +typedef OCIO_SHARED_PTR ConfigIOProxyRcPtr; + typedef std::array Float3; @@ -960,6 +964,13 @@ extern OCIOEXPORT const char * OCIO_DISABLE_CACHE_FALLBACK; /** @}*/ + +// Archive config feature +// Default filename (with extension) of an config. +extern OCIOEXPORT const char * OCIO_CONFIG_DEFAULT_NAME; +extern OCIOEXPORT const char * OCIO_CONFIG_DEFAULT_FILE_EXT; +extern OCIOEXPORT const char * OCIO_CONFIG_ARCHIVE_FILE_EXT; + } // namespace OCIO_NAMESPACE #endif diff --git a/share/cmake/modules/FindExtPackages.cmake b/share/cmake/modules/FindExtPackages.cmake index ea18ffb69c..9dd8c2a35d 100644 --- a/share/cmake/modules/FindExtPackages.cmake +++ b/share/cmake/modules/FindExtPackages.cmake @@ -41,6 +41,11 @@ find_package(pystring 1.1.3 REQUIRED) set(_Imath_ExternalProject_VERSION "3.1.5") find_package(Imath 3.0 REQUIRED) +# minizip-ng +# https://github.com/zlib-ng/minizip-ng +set(ZLIB_FIND_VERSION 1.2.12) +find_package(minizip-ng 3.0.6 REQUIRED) + if(OCIO_BUILD_APPS) # NOTE: Depending of the compiler version lcms2 2.2 does not compile with diff --git a/share/cmake/modules/Findminizip-ng.cmake b/share/cmake/modules/Findminizip-ng.cmake new file mode 100644 index 0000000000..e1e53093f3 --- /dev/null +++ b/share/cmake/modules/Findminizip-ng.cmake @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright Contributors to the OpenColorIO Project. +# +# Locate or install minizip-ng +# +# Variables defined by this module: +# minizip-ng_FOUND - If FALSE, do not try to link to minizip-ng +# minizip-ng_LIBRARY - minizip-ng library to link to +# minizip-ng_INCLUDE_DIR - Where to find mz.h and other headers +# minizip-ng_VERSION - The version of the library +# +# Targets defined by this module: +# minizip-ng::minizip-ng - IMPORTED target, if found +# +############################################################################### +### Try to find package ### + +# Search for ZLIB using FindZLIB from cmake. +# If ZLIB is not found, it will be downloaded and built. +include(GetZLIB) + +# Once ZLIB is sorted out, continue with minizip-ng. + +if(NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL ALL) + if(NOT DEFINED minizip-ng_ROOT) + # Search for minizip-ng-config.cmake + find_package(minizip-ng ${minizip-ng_FIND_VERSION} CONFIG) + endif() + + if (minizip-ng_FOUND) + get_target_property(minizip-ng_LIBRARY MINIZIP::minizip-ng LOCATION) + get_target_property(minizip-ng_INCLUDE_DIR MINIZIP::minizip-ng INTERFACE_INCLUDE_DIRECTORIES) + else () + list(APPEND _minizip-ng_REQUIRED_VARS minizip-ng_INCLUDE_DIR) + + # Search for minizip-ng.pc + find_package(PkgConfig QUIET) + pkg_check_modules(PC_minizip-ng QUIET "minizip-ng>=${minizip-ng_FIND_VERSION}") + + # Find include directory + find_path(minizip-ng_INCLUDE_DIR + NAMES + mz.h + HINTS + ${minizip-ng_ROOT} + ${PC_minizip-ng_INCLUDE_DIRS} + PATH_SUFFIXES + include + minizip-ng/include + ) + + # Lib names to search for + set(_minizip-ng_LIB_NAMES minizip-ng libminizip-ng) + if(BUILD_TYPE_DEBUG) + # Prefer Debug lib names (Windows only) + list(INSERT _minizip-ng_LIB_NAMES 0 minizip-ngd) + endif() + + if(minizip-ng_STATIC_LIBRARY) + # Prefer static lib names + set(_minizip-ng_STATIC_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}minizip-ng${CMAKE_STATIC_LIBRARY_SUFFIX}") + if(WIN32 AND BUILD_TYPE_DEBUG) + # Prefer static Debug lib names (Windows only) + list(INSERT _minizip-ng_STATIC_LIB_NAMES 0 + "${CMAKE_STATIC_LIBRARY_PREFIX}minizip-ngd${CMAKE_STATIC_LIBRARY_SUFFIX}") + endif() + endif() + + # Find library + find_library(minizip-ng_LIBRARY + NAMES + ${_minizip-ng_STATIC_LIB_NAMES} + ${_minizip-ng_LIB_NAMES} + HINTS + ${minizip-ng_ROOT} + ${PC_minizip-ng_LIBRARY_DIRS} + PATH_SUFFIXES + lib64 lib + ) + + # Get version from header or pkg-config + set(minizip-ng_VERSION "${minizip-ng_FIND_VERSION}") + endif() + + # Override REQUIRED if package can be installed + if(OCIO_INSTALL_EXT_PACKAGES STREQUAL MISSING) + set(minizip-ng_FIND_REQUIRED FALSE) + endif() + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(minizip-ng + REQUIRED_VARS + minizip-ng_LIBRARY + minizip-ng_INCLUDE_DIR + VERSION_VAR + minizip-ng_VERSION + ) +endif() + +############################################################################### +### Create target + +if(NOT TARGET minizip-ng::minizip-ng) + add_library(minizip-ng::minizip-ng UNKNOWN IMPORTED GLOBAL) + set(_minizip-ng_TARGET_CREATE TRUE) +endif() + +############################################################################### +### Install package from source ### +if(NOT minizip-ng_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) + include(ExternalProject) + include(GNUInstallDirs) + + set(_EXT_DIST_ROOT "${CMAKE_BINARY_DIR}/ext/dist") + set(_EXT_BUILD_ROOT "${CMAKE_BINARY_DIR}/ext/build") + + # Set find_package standard args + set(minizip-ng_FOUND TRUE) + set(minizip-ng_VERSION ${minizip-ng_FIND_VERSION}) + set(minizip-ng_INCLUDE_DIR "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_INCLUDEDIR}") + + # Minizip-ng use a hardcoded lib prefix instead of CMAKE_STATIC_LIBRARY_PREFIX + set(_minizip-ng_LIB_PREFIX "lib") + + set(minizip-ng_LIBRARY + "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_LIBDIR}/${_minizip-ng_LIB_PREFIX}minizip-ng${_minizip-ng_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + + if(_minizip-ng_TARGET_CREATE) + set(MINIZIP-NG_CMAKE_ARGS + ${MINIZIP-NG_CMAKE_ARGS} + -DCMAKE_CXX_VISIBILITY_PRESET=${CMAKE_CXX_VISIBILITY_PRESET} + -DCMAKE_VISIBILITY_INLINES_HIDDEN=${CMAKE_VISIBILITY_INLINES_HIDDEN} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} + -DCMAKE_INSTALL_MESSAGE=${CMAKE_INSTALL_MESSAGE} + -DCMAKE_INSTALL_PREFIX=${_EXT_DIST_ROOT} + -DCMAKE_INSTALL_BINDIR=${CMAKE_INSTALL_BINDIR} + -DCMAKE_INSTALL_DATADIR=${CMAKE_INSTALL_DATADIR} + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -DCMAKE_INSTALL_INCLUDEDIR=${CMAKE_INSTALL_INCLUDEDIR}/minizip-ng + -DCMAKE_OBJECT_PATH_MAX=${CMAKE_OBJECT_PATH_MAX} + -DBUILD_SHARED_LIBS=OFF + -DMZ_OPENSSL=OFF + -DMZ_LIBBSD=OFF + -DMZ_BUILD_TESTS=OFF + -DMZ_COMPAT=OFF + -DMZ_BZIP2=OFF + -DMZ_LZMA=OFF + -DMZ_LIBCOMP=OFF + -DMZ_ZSTD=OFF + -DMZ_PKCRYPT=OFF + -DMZ_WZAES=OFF + -DMZ_SIGNING=OFF + -DMZ_ZLIB=ON + -DMZ_ICONV=OFF + -DMZ_FETCH_LIBS=OFF + -DMZ_FORCE_FETCH_LIBS=OFF + -DZLIB_LIBRARY=${ZLIB_LIBRARIES} + -DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIRS} + ) + + if(CMAKE_TOOLCHAIN_FILE) + set(minizip-ng_CMAKE_ARGS + ${minizip-ng_CMAKE_ARGS} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}) + endif() + + if(APPLE) + string(REPLACE ";" "$" ESCAPED_CMAKE_OSX_ARCHITECTURES "${CMAKE_OSX_ARCHITECTURES}") + + set(minizip-ng_CMAKE_ARGS + ${minizip-ng_CMAKE_ARGS} + -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} + -DCMAKE_OSX_ARCHITECTURES=${ESCAPED_CMAKE_OSX_ARCHITECTURES} + ) + endif() + + if (ANDROID) + set(minizip-ng_CMAKE_ARGS + ${minizip-ng_CMAKE_ARGS} + -DANDROID_PLATFORM=${ANDROID_PLATFORM} + -DANDROID_ABI=${ANDROID_ABI} + -DANDROID_STL=${ANDROID_STL}) + endif() + endif() + + # Hack to let imported target be built from ExternalProject_Add + file(MAKE_DIRECTORY ${minizip-ng_INCLUDE_DIR}) + + ExternalProject_Add(minizip-ng_install + GIT_REPOSITORY "https://github.com/zlib-ng/minizip-ng.git" + GIT_TAG "${minizip-ng_VERSION}" + GIT_CONFIG advice.detachedHead=false + GIT_SHALLOW TRUE + PREFIX "${_EXT_BUILD_ROOT}/libminizip-ng" + BUILD_BYPRODUCTS ${minizip-ng_LIBRARY} + CMAKE_ARGS ${MINIZIP-NG_CMAKE_ARGS} + EXCLUDE_FROM_ALL TRUE + BUILD_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} --build . + --config ${CMAKE_BUILD_TYPE} + --target install + --parallel + ) + + add_dependencies(minizip-ng::minizip-ng minizip-ng_install) + message(STATUS "Installing minizip-ng: ${minizip-ng_LIBRARY} (version \"${minizip-ng_VERSION}\")") +endif() + +############################################################################### +### Configure target ### + +if(_minizip-ng_TARGET_CREATE) + set_target_properties(minizip-ng::minizip-ng PROPERTIES + IMPORTED_LOCATION "${minizip-ng_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${minizip-ng_INCLUDE_DIR}" + ) + + mark_as_advanced(minizip-ng_INCLUDE_DIR minizip-ng_LIBRARY minizip-ng_VERSION) + + if (NOT MSVC) + target_link_libraries(minizip-ng::minizip-ng INTERFACE ZLIB::ZLIB) + endif() +endif() \ No newline at end of file diff --git a/share/cmake/modules/GetZLIB.cmake b/share/cmake/modules/GetZLIB.cmake new file mode 100644 index 0000000000..c21729b662 --- /dev/null +++ b/share/cmake/modules/GetZLIB.cmake @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright Contributors to the OpenColorIO Project. +# +# Locate or install minizip-ng +# +# Variables defined by this module: +# ZLIB_FOUND - If FALSE, do not try to link to minizip-ng +# ZLIB_LIBRARIES - ZLIB library to link to +# ZLIB_INCLUDE_DIRS - Where to find zlib.h and other headers +# ZLIB_VERSION - The version of the library +# +# Targets defined by this module: +# ZLIB::ZLIB - IMPORTED target, if found +# +# This module is named GetZLIB because it is not used with find_package(). +# It must be included using include(). +# +# The reason is that CMake provide a FindZLIB already and the current file is +# using it. +# +############################################################################### +### Try to find package ### + +# Assign the rigtt name for ZLIB depending on the OS. +if(UNIX) + set(ZLIB_NAME libz) +else() + set(ZLIB_NAME zlib) +endif() + +if(NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL ALL) + set(_ZLIB_REQUIRED_VARS ZLIB_LIBRARIES) + + if(NOT DEFINED ZLIB_ROOT) + # findZLIB provided by CMAKE. + find_package(ZLIB ${ZLIB_FIND_VERSION}) + endif() +endif() + +############################################################################### +### Create target + +if(NOT TARGET ZLIB::ZLIB) + add_library(ZLIB::ZLIB UNKNOWN IMPORTED GLOBAL) + set(_ZLIB_TARGET_CREATE TRUE) +endif() + +############################################################################### +### Install package from source ### +if(NOT ZLIB_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) + include(ExternalProject) + include(GNUInstallDirs) + + set(_EXT_DIST_ROOT "${CMAKE_BINARY_DIR}/ext/dist") + set(_EXT_BUILD_ROOT "${CMAKE_BINARY_DIR}/ext/build") + + # Set find_package standard args + set(ZLIB_FOUND TRUE) + set(ZLIB_VERSION ${ZLIB_FIND_VERSION}) + set(ZLIB_INCLUDE_DIRS "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_INCLUDEDIR}") + + set(ZLIB_LIBRARIES + "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_LIBDIR}/${_ZLIB_LIB_PREFIX}${ZLIB_NAME}${_ZLIB_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + + if(_ZLIB_TARGET_CREATE) + set(ZLIB_CMAKE_ARGS + ${ZLIB_CMAKE_ARGS} + -DCMAKE_CXX_VISIBILITY_PRESET=${CMAKE_CXX_VISIBILITY_PRESET} + -DCMAKE_VISIBILITY_INLINES_HIDDEN=${CMAKE_VISIBILITY_INLINES_HIDDEN} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} + -DCMAKE_INSTALL_MESSAGE=${CMAKE_INSTALL_MESSAGE} + -DCMAKE_INSTALL_PREFIX=${_EXT_DIST_ROOT} + -DCMAKE_INSTALL_BINDIR=${CMAKE_INSTALL_BINDIR} + -DCMAKE_INSTALL_DATADIR=${CMAKE_INSTALL_DATADIR} + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -DCMAKE_INSTALL_INCLUDEDIR=${CMAKE_INSTALL_INCLUDEDIR} + -DCMAKE_OBJECT_PATH_MAX=${CMAKE_OBJECT_PATH_MAX} + ) + + if(CMAKE_TOOLCHAIN_FILE) + set(ZLIB_CMAKE_ARGS + ${ZLIB_CMAKE_ARGS} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}) + endif() + + if(APPLE) + string(REPLACE ";" "$" ESCAPED_CMAKE_OSX_ARCHITECTURES "${CMAKE_OSX_ARCHITECTURES}") + + set(ZLIB_CMAKE_ARGS + ${ZLIB_CMAKE_ARGS} + -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} + -DCMAKE_OSX_ARCHITECTURES=${ESCAPED_CMAKE_OSX_ARCHITECTURES} + ) + endif() + + if (ANDROID) + set(ZLIB_CMAKE_ARGS + ${ZLIB_CMAKE_ARGS} + -DANDROID_PLATFORM=${ANDROID_PLATFORM} + -DANDROID_ABI=${ANDROID_ABI} + -DANDROID_STL=${ANDROID_STL}) + endif() + endif() + + # Hack to let imported target be built from ExternalProject_Add + file(MAKE_DIRECTORY ${ZLIB_INCLUDE_DIRS}) + + ExternalProject_Add(ZLIB_install + GIT_REPOSITORY "https://github.com/madler/zlib.git" + GIT_TAG "v${ZLIB_VERSION}" + GIT_CONFIG advice.detachedHead=false + GIT_SHALLOW TRUE + PREFIX "${_EXT_BUILD_ROOT}/${ZLIB_NAME}" + BUILD_BYPRODUCTS ${ZLIB_LIBRARIES} + CMAKE_ARGS ${ZLIB_CMAKE_ARGS} + EXCLUDE_FROM_ALL TRUE + BUILD_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} --build . + --config ${CMAKE_BUILD_TYPE} + --target install + --parallel + ) + + add_dependencies(ZLIB::ZLIB ZLIB_install) + message(STATUS "Installing ZLIB: ${ZLIB_LIBRARIES} (version \"${ZLIB_VERSION}\")") +endif() + +############################################################################### +### Configure target ### + +if(_ZLIB_TARGET_CREATE) + set_target_properties(ZLIB::ZLIB PROPERTIES + IMPORTED_LOCATION ${ZLIB_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS} + ) + + mark_as_advanced(ZLIB_INCLUDE_DIRS ZLIB_LIBRARIES ZLIB_VERSION) +endif() \ No newline at end of file diff --git a/src/OpenColorIO/CMakeLists.txt b/src/OpenColorIO/CMakeLists.txt index 8d98c7ebbb..a54f9486b1 100755 --- a/src/OpenColorIO/CMakeLists.txt +++ b/src/OpenColorIO/CMakeLists.txt @@ -69,6 +69,7 @@ set(SOURCES md5/md5.cpp NamedTransform.cpp OCIOYaml.cpp + OCIOZArchive.cpp Op.cpp OpOptimizers.cpp ops/allocation/AllocationOp.cpp @@ -275,6 +276,7 @@ target_link_libraries(OpenColorIO "$" "$" yaml-cpp + minizip-ng::minizip-ng ) if(APPLE) diff --git a/src/OpenColorIO/Config.cpp b/src/OpenColorIO/Config.cpp index 784a400419..26d8b949e6 100644 --- a/src/OpenColorIO/Config.cpp +++ b/src/OpenColorIO/Config.cpp @@ -10,9 +10,11 @@ #include #include #include +#include #include +#include "builtinconfigs/BuiltinConfigRegistry.h" #include "ContextVariableUtils.h" #include "Display.h" #include "fileformats/FileFormatICC.h" @@ -24,19 +26,19 @@ #include "Mutex.h" #include "NamedTransform.h" #include "OCIOYaml.h" +#include "OCIOZArchive.h" #include "OpBuilders.h" #include "ParseUtils.h" #include "PathUtils.h" #include "Platform.h" #include "PrivateTypes.h" #include "Processor.h" +#include "pystring/pystring.h" +#include "transforms/FileTransform.h" #include "utils/StringUtils.h" #include "ViewingRules.h" #include "SystemMonitor.h" -#include "builtinconfigs/BuiltinConfigRegistry.h" - - namespace OCIO_NAMESPACE { @@ -47,6 +49,11 @@ const char * OCIO_INACTIVE_COLORSPACES_ENVVAR = "OCIO_INACTIVE_COLORSPACES"; const char * OCIO_OPTIMIZATION_FLAGS_ENVVAR = "OCIO_OPTIMIZATION_FLAGS"; const char * OCIO_USER_CATEGORIES_ENVVAR = "OCIO_USER_CATEGORIES"; +// Default filename (with extension) of a config and archived config. +const char * OCIO_CONFIG_DEFAULT_NAME = "config"; +const char * OCIO_CONFIG_DEFAULT_FILE_EXT = ".ocio"; +const char * OCIO_CONFIG_ARCHIVE_FILE_EXT = ".ocioz"; + // A shared view using this for the color space name will use a display color space that // has the same name as the display the shared view is used by. const char * OCIO_VIEW_USE_DISPLAY_NAME = ""; @@ -575,6 +582,7 @@ class Config::Impl void getAllInternalTransforms(ConstTransformVec & transformVec) const; static ConstConfigRcPtr Read(std::istream & istream, const char * filename); + static ConstConfigRcPtr Read(std::istream & istream, ConfigIOProxyRcPtr ciop); // Validate view object that can be a config defined shared view or a display-defined view. void validateView(const std::string & display, const View & view, bool checkUseDisplayName) const @@ -1070,7 +1078,6 @@ class Config::Impl // That should never happen. return -1; } - }; @@ -1104,7 +1111,10 @@ ConstConfigRcPtr Config::CreateFromEnv() std::string file; Platform::Getenv(OCIO_CONFIG_ENVVAR, file); - // File can be a filename/path or an URI to a default configuration (ocio://). + // File may be one of the following: + // 1) Path to a config file (e.g. /home/user/ocio/config.ocio) + // 2) Path to an archived config file (e.g. /home/user/ocio/archived_config.ocioz) + // 3) URI to a built-in config (e.g. ocio://cg-config-v0.1.0_aces-v1.3_ocio-v2.1.1) if(!file.empty()) return CreateFromFile(file.c_str()); static const char err[] = @@ -1137,8 +1147,12 @@ ConstConfigRcPtr Config::CreateFromFile(const char * filename) return CreateFromBuiltinConfig(match.str(1).c_str()); } - std::ifstream istream = Platform::CreateInputFileStream(filename, std::ios_base::in); - if (istream.fail()) + std::ifstream ifstream = Platform::CreateInputFileStream( + filename, + std::ios_base::in | std::ios_base::binary + ); + + if (ifstream.fail()) { std::ostringstream os; os << "Error could not read '" << filename; @@ -1146,7 +1160,34 @@ ConstConfigRcPtr Config::CreateFromFile(const char * filename) throw Exception (os.str().c_str()); } - return Config::Impl::Read(istream, filename); + char magicNumber[2] = { 0 }; + if (ifstream.read(magicNumber, 2)) + { + // Check if it is an OCIOZ archive. + if (magicNumber[0] == 'P' && magicNumber[1] == 'K') + { + // Closing ifstream even though it should be close by ifstream deconstructor (RAII). + ifstream.close(); + + // The file should be an OCIOZ archive file. + + // Using new because OCIO doesn't have access to make_unique since OCIO still supports + // C++11. + std::shared_ptr ciop = std::shared_ptr( + new CIOPOciozArchive() + ); + // Store archive absolute path. + ciop->setArchiveAbsPath(filename); + // Build the entries map. + ciop->buildEntries(); + return CreateFromConfigIOProxy(ciop); + } + } + + // Not an OCIOZ archive. Continue as usual. + ifstream.clear(); + ifstream.seekg(0); + return Config::Impl::Read(ifstream, filename); } ConstConfigRcPtr Config::CreateFromStream(std::istream & istream) @@ -1154,6 +1195,25 @@ ConstConfigRcPtr Config::CreateFromStream(std::istream & istream) return Config::Impl::Read(istream, nullptr); } +ConstConfigRcPtr Config::CreateFromConfigIOProxy(ConfigIOProxyRcPtr ciop) +{ + ConstConfigRcPtr config = nullptr; + + // Get a stream of the config. + std::string configStr = ciop->getConfigData(); + std::stringstream configStream(configStr); + config = Config::Impl::Read(configStream, ciop); + + if (config == nullptr) + { + std::ostringstream os; + os << "Could not create config using ConfigIOProxy."; + throw Exception (os.str().c_str()); + } + + return config; +} + ConstConfigRcPtr Config::CreateFromBuiltinConfig(const char * configName) { ConstConfigRcPtr builtinConfig; @@ -4005,6 +4065,7 @@ ConstProcessorRcPtr Config::getProcessor(const ConstContextRcPtr & context, ContextRcPtr usedContext = Context::Create(); usedContext->setSearchPath(context->getSearchPath()); usedContext->setWorkingDir(context->getWorkingDir()); + usedContext->setConfigIOProxy(context->getConfigIOProxy()); const bool needContextVariables = CollectContextVariables(*this, *context, transform, usedContext); @@ -4275,7 +4336,7 @@ const char * Config::getCacheID(const ConstContextRcPtr & context) const try { const std::string resolvedLocation = context->resolveFileLocation(iter.c_str()); - filehash << GetFastFileHash(resolvedLocation) << " "; + filehash << GetFastFileHash(resolvedLocation, *context) << " "; } catch(...) { @@ -4533,6 +4594,28 @@ ConstConfigRcPtr Config::Impl::Read(std::istream & istream, const char * filenam return config; } +ConstConfigRcPtr Config::Impl::Read(std::istream & istream, ConfigIOProxyRcPtr ciop) +{ + ConfigRcPtr config = Config::Create(); + // Passing special string for the file path to enable the parser to provide a more + // meaningful error message if a problem is encountered. (The working directory is not + // set to this string.) + OCIOYaml::Read(istream, config, "from Archive/ConfigIOProxy"); + + config->getImpl()->checkVersionConsistency(); + + // An API request always supersedes the env. variable. As the OCIOYaml helper methods + // use the Config public API, the variable reset highlights that only the + // env. variable and the config contents are valid after a config file read. + config->getImpl()->m_inactiveColorSpaceNamesAPI.clear(); + config->getImpl()->refreshActiveColorSpaces(); + + // Set the ConfigIOProxy object. + config->setConfigIOProxy(ciop); + + return config; +} + void Config::Impl::checkVersionConsistency(ConstTransformRcPtr & transform) const { if (transform) @@ -4782,5 +4865,99 @@ void Config::Impl::checkVersionConsistency() const } +void Config::setConfigIOProxy(ConfigIOProxyRcPtr ciop) +{ + getImpl()->m_context->setConfigIOProxy(ciop); + + AutoMutex lock(getImpl()->m_cacheidMutex); + getImpl()->resetCacheIDs(); +} + +ConfigIOProxyRcPtr Config::getConfigIOProxy() const +{ + return getImpl()->m_context->getConfigIOProxy(); +} + +bool Config::isArchivable() const +{ + ConstContextRcPtr context = getCurrentContext(); + + // Current archive implementation needs a working directory to look for LUT files and + // working directory must be an absolute path. + const char * workingDirectory = getWorkingDir(); + if ((workingDirectory && !workingDirectory[0]) || !pystring::os::path::isabs(workingDirectory)) + { + return false; + } + + // Utility lambda to check the following criteria. + auto validatePathForArchiving = [](const std::string & path) + { + // Using the normalized path. + const std::string normPath = pystring::os::path::normpath(path); + if ( + // 1) Path may not be absolute. + pystring::os::path::isabs(normPath) || + // 2) Path may not start with double dot ".." (going above working directory). + pystring::startswith(normPath, "..") || + // 3) A context variable may not be located at the start of the path. + (ContainsContextVariables(path) && + (StringUtils::Find(path, "$") == 0 || + StringUtils::Find(path, "%") == 0))) + { + return false; + } + + return true; + }; + + /////////////////////////////// + // Search path verification. // + /////////////////////////////// + // Check that search paths are not absolute nor have context variables outside of config + // working directory. + int numSearchPaths = getNumSearchPaths(); + for (int i = 0; i < numSearchPaths; i++) + { + std::string currentPath = getSearchPath(i); + if (!validatePathForArchiving(currentPath)) + { + // Exit and return false. + return false; + } + } + + ///////////////////////////////// + // FileTransform verification. // + ///////////////////////////////// + ConstTransformVec allTransforms; + getImpl()->getAllInternalTransforms(allTransforms); + + std::set files; + for(const auto & transform : allTransforms) + { + GetFileReferences(files, transform); + } + + // Check that FileTransform sources are not absolute nor have context variables outside of + // config working directory. + for (auto path : files) + { + if (!validatePathForArchiving(path)) + { + // Exit and return false. + return false; + } + } + + return true; +} + +void Config::archive(std::ostream & ostream) const +{ + // Using utility functions in OCIOZArchive.cpp. + archiveConfig(ostream, *this, getCurrentContext()->getWorkingDir()); +} + } // namespace OCIO_NAMESPACE diff --git a/src/OpenColorIO/Context.cpp b/src/OpenColorIO/Context.cpp index 26fdcbeed9..c37ce2ec05 100644 --- a/src/OpenColorIO/Context.cpp +++ b/src/OpenColorIO/Context.cpp @@ -12,12 +12,12 @@ #include "ContextVariableUtils.h" #include "HashUtils.h" #include "Mutex.h" +#include "OCIOZArchive.h" #include "PathUtils.h" #include "PrivateTypes.h" #include "pystring/pystring.h" #include "utils/StringUtils.h" - namespace OCIO_NAMESPACE { @@ -51,6 +51,8 @@ class Context::Impl mutable ResolvedStringCache m_resultsFilepathCache; mutable Mutex m_resultsCacheMutex; + ConfigIOProxyRcPtr m_configIOProxy; + Impl() = default; ~Impl() = default; @@ -70,6 +72,8 @@ class Context::Impl m_resultsFilepathCache = rhs.m_resultsFilepathCache; m_cacheID = rhs.m_cacheID; + + m_configIOProxy = rhs.m_configIOProxy; } return *this; } @@ -436,7 +440,7 @@ const char * Context::resolveFileLocation(const char * filename, ContextRcPtr & // If the file reference is absolute, check if the file exists (independent of the search paths). if(pystring::os::path::isabs(resolvedFilename)) { - if(FileExists(resolvedFilename)) + if(FileExists(resolvedFilename, *this)) { // That's already an absolute path so no extra context variables are present. UsedEnvs envs; @@ -484,8 +488,7 @@ const char * Context::resolveFileLocation(const char * filename, ContextRcPtr & { // Make an attempt to find the LUT in one of the search paths. const std::string resolvedfullpath = pystring::os::path::join(searchpaths[i], resolvedFilename); - - if (!ContainsContextVariables(resolvedfullpath) && FileExists(resolvedfullpath)) + if (!ContainsContextVariables(resolvedfullpath) && FileExists(resolvedfullpath, *this)) { // Collect all the used context variables. if (usedContextVars) @@ -511,6 +514,16 @@ const char * Context::resolveFileLocation(const char * filename, ContextRcPtr & throw ExceptionMissingFile(errortext.str().c_str()); } +void Context::setConfigIOProxy(ConfigIOProxyRcPtr ciop) +{ + getImpl()->m_configIOProxy = ciop; +} + +ConfigIOProxyRcPtr Context::getConfigIOProxy() const +{ + return getImpl()->m_configIOProxy; +} + std::ostream& operator<< (std::ostream& os, const Context& context) { os << " ") - << "does not appear to have a valid version " + << ((filename && *filename) ? filename : "") + << " does not appear to have a valid version " << (version.empty() ? "" : version) << "."; @@ -4647,7 +4647,10 @@ inline void load(const YAML::Node& node, ConfigRcPtr & config, const char* filen } } - if (filename) + // Do not set the working dir when the filename is empty or contains the special string + // "from Archive/ConfigIOProxy". + if (filename && filename[0] && + Platform::Strcasecmp(filename, "from Archive/ConfigIOProxy") != 0) { std::string realfilename = AbsPath(filename); std::string configrootdir = pystring::os::path::dirname(realfilename); @@ -5108,7 +5111,11 @@ void OCIOYaml::Read(std::istream & istream, ConfigRcPtr & config, const char * f { std::ostringstream os; os << "Error: Loading the OCIO profile "; - if(filename) os << "'" << filename << "' "; + if (filename && filename[0] && + Platform::Strcasecmp(filename, "from Archive/ConfigIOProxy") != 0) + { + os << "'" << filename << "' "; + } os << "failed. " << e.what(); throw Exception(os.str().c_str()); } diff --git a/src/OpenColorIO/OCIOZArchive.cpp b/src/OpenColorIO/OCIOZArchive.cpp new file mode 100644 index 0000000000..6df14a868a --- /dev/null +++ b/src/OpenColorIO/OCIOZArchive.cpp @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright Contributors to the OpenColorIO Project. + +#include +#include +#include +#include +#include +#include + +#include +#include "Mutex.h" +#include "Platform.h" +#include "pystring/pystring.h" +#include "utils/StringUtils.h" +#include "transforms/FileTransform.h" + +#include "OCIOZArchive.h" + +#include "minizip-ng/mz.h" +#include "minizip-ng/mz_os.h" +#include "minizip-ng/mz_strm.h" +#include "minizip-ng/mz_strm_buf.h" +#include "minizip-ng/mz_strm_mem.h" +#include "minizip-ng/mz_strm_os.h" +#include "minizip-ng/mz_strm_split.h" +#include "minizip-ng/mz_strm_zlib.h" +#include "minizip-ng/mz_zip.h" +#include "minizip-ng/mz_zip_rw.h" + +namespace OCIO_NAMESPACE +{ +// Minizip-np compression levels. +enum ArchiveCompressionLevels +{ + DEFAULT = -1, + FAST = 2, + NORMAL = 6, + BEST = 9 +}; + +// Minizip-np compression methods. +enum ArchiveCompressionMethods +{ + DEFLATE = 8 +}; + +// Minizip-ng options for archive. +struct ArchiveOptions { + uint8_t include_path = 0; + int16_t compress_level = ArchiveCompressionLevels::BEST; + uint8_t compress_method = ArchiveCompressionMethods::DEFLATE; + uint8_t overwrite = 0; + uint8_t append = 0; + int64_t disk_size = 0; + uint8_t follow_links = 0; + uint8_t store_links = 0; + uint8_t zip_cd = 0; + int32_t encoding = 0; + uint8_t verbose = 0; + uint8_t aes = 0; + const char* cert_path; + const char* cert_pwd; +}; + +/** + * @brief Guard against early throws with Minizip-ng objects. + * + */ +struct MinizipNgHandlerGuard +{ + MinizipNgHandlerGuard(void *& handle, bool isWriter, bool usingEntry) + : m_handle(handle), m_isWriter(isWriter), m_usingEntry(usingEntry) + { + // Nothing to do. + } + + ~MinizipNgHandlerGuard() + { + if (m_handle != nullptr) + { + if (m_isWriter) + { + // Clean up writer object. + if (m_usingEntry) + { + // Clean up writer entry object. + mz_zip_writer_entry_close(m_handle); + } + // Close and delete the handle. + mz_zip_writer_delete(&m_handle); + } + else + { + // Clean up reader object. + if (m_usingEntry) + { + // Clean up reader entry object. + mz_zip_reader_entry_close(m_handle); + } + + // Close and delete the handle. + mz_zip_reader_delete(&m_handle); + } + m_handle = nullptr; + } + } + + // Minizip-ng handler + void *& m_handle; + bool m_isWriter; + bool m_usingEntry; +}; + +struct MinizipNgMemStreamGuard +{ + MinizipNgMemStreamGuard(void *& memStream) + : m_memStream(memStream) + { + // Nothing to do. + } + + ~MinizipNgMemStreamGuard() + { + if (m_memStream != nullptr) + { + // Clean up memory stream. + mz_stream_mem_close(m_memStream); + mz_stream_mem_delete(&m_memStream); + m_memStream = nullptr; + } + } + + // Minizip-ng memory stream object. + void *& m_memStream; +}; + +////////////////////////////////////////////////////////////////////////////////////// +// Utility functions section +////////////////////////////////////////////////////////////////////////////////////// +/** + * Utility function for archived Configs. + * + * @param archiver Minizip-ng handle object. + * @param path Path of the file or the folder to add inside the OCIOZ archive. + * @param configWorkingDirectory Working directory of the current config. + */ +void addSupportedFiles(void * archiver, const char * path, const char * configWorkingDirectory) +{ + DIR *dir = mz_os_open_dir(path); + if (dir != NULL) + { + struct dirent *entry = NULL; + while ((entry = mz_os_read_dir(dir)) != NULL) + { + // Join the current path and the directory/file name to get the absolute path. + std::string absPath = pystring::os::path::join(path, entry->d_name); + // Check if the absolute path is a directory. + if (mz_os_is_dir(absPath.c_str()) == MZ_OK) + { + // Since mz_os_read_dir is listing the whole directory, "." and ".." must be + // ignored. + if (!StringUtils::Compare(".", entry->d_name) && + !StringUtils::Compare("..", entry->d_name)) + { + // Add the current directory. + addSupportedFiles(archiver, absPath.c_str(), configWorkingDirectory); + } + } + else + { + // Absolute path is a file. + std::string root, ext; + pystring::os::path::splitext(root, ext, std::string(entry->d_name)); + // Strip leading dot character in order to get the extension name only. + ext = pystring::lstrip(ext, "."); + + // Check if the extension is supported. Using logic from LoadFileUncached(). + FormatRegistry & formatRegistry = FormatRegistry::GetInstance(); + FileFormatVector possibleFormats; + formatRegistry.getFileFormatForExtension(ext, possibleFormats); + FileFormatVector::const_iterator endFormat = possibleFormats.end(); + FileFormatVector::const_iterator itFormat = possibleFormats.begin(); + while(itFormat != endFormat) + { + // Valid format. Archive current file. + if (mz_zip_writer_add_path( + archiver, absPath.c_str(), + configWorkingDirectory, 0, 1) != MZ_OK) + { + // Close DIR object before throwing. + mz_os_close_dir(dir); + + std::ostringstream os; + os << "Could not write LUT file " << absPath << " to in-memory archive."; + throw Exception(os.str().c_str()); + } + ++itFormat; + } + } + } + mz_os_close_dir(dir); + } +} + +void archiveConfig(std::ostream & ostream, const Config & config, const char * configWorkingDirectory) +{ + void * archiver = nullptr; + void *write_mem_stream = NULL; + const uint8_t *buffer_ptr = NULL; + int32_t buffer_size = 0; + mz_zip_file file_info; + + if (!config.isArchivable()) + { + std::ostringstream os; + os << "Config is not archivable."; + throw Exception(os.str().c_str()); + } + + // Initialize. + memset(&file_info, 0, sizeof(file_info)); + + // Retrieve and store the config as string. + std::stringstream ss; + config.serialize(ss); + std::string configStr = ss.str(); + + // Write zip to memory stream. + mz_stream_mem_create(&write_mem_stream); + mz_stream_mem_set_grow_size(write_mem_stream, 128 * 1024); + mz_stream_open(write_mem_stream, NULL, MZ_OPEN_MODE_CREATE); + + // Minizip archive options. + ArchiveOptions options; + // Make sure that the compression method is set to DEFLATE. + options.compress_method = ArchiveCompressionMethods::DEFLATE; + // Make sure that the compression level is set to BEST. + options.compress_level = ArchiveCompressionLevels::BEST; + + // Create the writer handle. + mz_zip_writer_create(&archiver); + + // Archive options. + // Compression method + mz_zip_writer_set_compress_method(archiver, options.compress_method); + // Compress level. + mz_zip_writer_set_compress_level(archiver, options.compress_level); + + MinizipNgMemStreamGuard memStreamGuard(write_mem_stream); + MinizipNgHandlerGuard archiverGuard(archiver, true, true); + + // Open the in-memory zip. + if (mz_zip_writer_open(archiver, write_mem_stream, 0) == MZ_OK) + { + // Use a hardcoded name for the config's filename inside the archive. + std::string configFullname = std::string(OCIO_CONFIG_DEFAULT_NAME) + + std::string(OCIO_CONFIG_DEFAULT_FILE_EXT); + + // Get config string size. + int32_t configSize = (int32_t) configStr.size(); + + // Initialize the config file information structure. + file_info.filename = configFullname.c_str(); + file_info.modified_date = time(NULL); + file_info.version_madeby = MZ_VERSION_MADEBY; + file_info.compression_method = MZ_COMPRESS_METHOD_DEFLATE; + file_info.flag = MZ_ZIP_FLAG_UTF8; + file_info.uncompressed_size = configSize; + + ////////////////////////////// + // Adding config to archive // + ////////////////////////////// + int32_t written = 0; + // Opens an entry in the in-memory zip file for writing. + if (mz_zip_writer_entry_open(archiver, &file_info) == MZ_OK) + { + // Add config to the in-memory zip using buffer. + written = mz_zip_writer_entry_write(archiver, configStr.c_str(), configSize); + if (written < MZ_OK) + { + std::ostringstream os; + os << "Could not write config to in-memory archive."; + throw Exception(os.str().c_str()); + } + // Close the entry. + mz_zip_writer_entry_close(archiver); + } + else + { + std::ostringstream os; + os << "Could not prepare an entry for writing."; + throw Exception(os.str().c_str()); + } + + /////////////////////// + // Adding LUT files // + /////////////////////// + // Add all supported files to in-memory zip from any directories under working directory. + // (recursive) + addSupportedFiles(archiver, configWorkingDirectory, configWorkingDirectory); + + // Close in-memory zip. + mz_zip_writer_close(archiver); + } + + // Clean up. + mz_zip_writer_delete(&archiver); + + // Get the buffer to write to the ostream. + mz_stream_mem_get_buffer(write_mem_stream, (const void **)&buffer_ptr); + mz_stream_mem_seek(write_mem_stream, 0, MZ_SEEK_END); + buffer_size = (int32_t)mz_stream_mem_tell(write_mem_stream); + + ostream.write((char*)&buffer_ptr[0], buffer_size); + + mz_stream_mem_close(write_mem_stream); + mz_stream_mem_delete(&write_mem_stream); +} + +/** + * @brief Extract the specified OCIOZ archive. + * + * This function can only be used with the OCIOZ archive format (not arbitrary zip files). + * + * Note: Signature is in OpenColorIO.h since the function is OCIOEXPORT-ed for client apps. + */ +void ExtractOCIOZArchive(const char * archivePath, const char * destination) +{ + void * extracter = NULL; + int32_t err = MZ_OK; + + // Normalize the path for the platform. + std::string outputDestination = pystring::os::path::normpath(destination); + + // Create zip reader. + mz_zip_reader_create(&extracter); + + MinizipNgHandlerGuard extracterGuard(extracter, false, false); + + // Open the OCIOZ archive. + if (mz_zip_reader_open_file(extracter, archivePath) != MZ_OK) + { + std::ostringstream os; + os << "Could not open " << archivePath << " for reading."; + throw Exception(os.str().c_str()); + } + else + { + // Extract all entries to outputDestination directory. + err = mz_zip_reader_save_all(extracter, outputDestination.c_str()); + if (err == MZ_END_OF_LIST) + { + // The archive has no files. + std::ostringstream os; + os << "No files in archive."; + throw Exception(os.str().c_str()); + } + else if (err != MZ_OK) + { + std::ostringstream os; + os << "Could not extract: " << archivePath; + throw Exception(os.str().c_str()); + } + } + + // Close the OCIOZ archive. + if (mz_zip_reader_close(extracter) != MZ_OK) + { + std::ostringstream os; + os << "Could not close " << archivePath << " after reading."; + throw Exception(os.str().c_str()); + } + + // Clean up. + mz_zip_reader_delete(&extracter); +} + +/** + * @brief Callback function for getFileStringFromArchiveStream in order to get the contents of a + * file inside an OCIOZ archive as a buffer. + * + * The file is retrieved by comparing the paths. + * + * @param buffer Buffer of uint8_t + * @param reader Minizip-ng handle object. + * @param info File information. + * @param filepath Path to find. + */ +void getFileBufferByPath(std::vector & buffer, + void * reader, + mz_zip_file & info, + std::string filepath) +{ + // Verify that the file information and the current file matches while ignoring the slashes + // differences in platforms. + if (mz_path_compare_wc(filepath.c_str(), info.filename, 1) == MZ_OK) + { + // Initialize the buffer for the file. + int32_t buf_size = (int32_t)mz_zip_reader_entry_save_buffer_length(reader); + buffer.resize(buf_size); + // Read the content of the file and return it as buffer. + mz_zip_reader_entry_save_buffer(reader, &buffer[0], buf_size); + } +} + +/** + * @brief Callback function for getFileStringFromArchiveStream in order to Get the content of a + * file inside an OCIOZ archive as a buffer. + * + * The file is retrieved by comparing only the extension. (Used for the .ocio file.) + * + * @param buffer Buffer of uint8_t + * @param reader Minizip-ng reader object. + * @param info File information + * @param extension Extension to find + */ +void getFileBufferByExtension(std::vector & buffer, + void * reader, + mz_zip_file & info, + std::string extension) +{ + std::string root, ext; + pystring::os::path::splitext(root, ext, info.filename); + if (Platform::Strcasecmp(extension.c_str(), ext.c_str()) == 0) + { + int32_t buf_size = (int32_t)mz_zip_reader_entry_save_buffer_length(reader); + buffer.resize(buf_size); + mz_zip_reader_entry_save_buffer(reader, &buffer[0], buf_size); + } +} + +/** + * @brief Get the content of a file inside an OCIOZ archive as a buffer. + * + * The two possbiles callbacks are defined above: + * getFileBufferByPath and getFileBufferByExtension. + * + * @param buffer Buffer of uint8_t + * @param filepath File to retrieve from the OCIOZ archive. + * @param archivePath Path to the archive. + * @param fn Callback function to get the (file) buffer by path or by extension. + */ +void getFileStringFromArchiveFile(std::vector & buffer, + const std::string & filepath, + const std::string & archivePath, + void (*fn)(std::vector & buffer, void*, mz_zip_file&, std::string)) +{ + mz_zip_file *file_info = NULL; + int32_t err = MZ_OK; + void *reader = NULL; + + // Create the reader object. + mz_zip_reader_create(&reader); + + MinizipNgHandlerGuard extracterGuard(reader, false, true); + + // Open the zip in memory. + err = mz_zip_reader_open_file(reader, archivePath.c_str()); + if (err != MZ_OK) + { + std::ostringstream os; + os << "Could not open " << archivePath.c_str() + << " in order to get the file: " << filepath; + throw Exception(os.str().c_str()); + } + else + { + // Seek to the first entry in the OCIOZ archive. + if (mz_zip_reader_goto_first_entry(reader) == MZ_OK) + { + do + { + // Get the current entry information. + if (mz_zip_reader_entry_get_info(reader, &file_info) == MZ_OK) + { + fn(buffer, reader, *file_info, filepath); + if (!buffer.empty()) + { + break; + } + } + } while (mz_zip_reader_goto_next_entry(reader) == MZ_OK); + } + } +} + +////////////////////////////////////////////////////////////////////////////////////// + +////////////////////////////////////////////////////////////////////////////////////// +// API section +////////////////////////////////////////////////////////////////////////////////////// + +// See header file for information. +void getFileBufferFromArchive(std::vector & buffer, + const std::string & filepath, + const std::string & archivePath) +{ + return getFileStringFromArchiveFile(buffer, filepath, archivePath, &getFileBufferByPath); +} + +// See header file for information. +void getFileBufferFromArchiveByExtension(std::vector & buffer, + const std::string & extension, + const std::string & archivePath) +{ + return getFileStringFromArchiveFile(buffer, extension, archivePath, &getFileBufferByExtension); +} + +// See header file for information. +void getEntriesMappingFromArchiveFile(const std::string & archivePath, + std::map & map) +{ + mz_zip_file *file_info = NULL; + int32_t err = MZ_OK; + void *reader = NULL; + + // Create the reader object. + mz_zip_reader_create(&reader); + + MinizipNgHandlerGuard extracterGuard(reader, false, false); + + // Open the zip from file. + err = mz_zip_reader_open_file(reader, archivePath.c_str()); + if (err != MZ_OK) + { + std::ostringstream os; + os << "Could not open " << archivePath.c_str() << " in order to get the entries."; + throw Exception(os.str().c_str()); + } + else + { + // Seek to the first entry in the OCIOZ archive. + if (mz_zip_reader_goto_first_entry(reader) == MZ_OK) + { + do + { + // Get the current entry information. + if (mz_zip_reader_entry_get_info(reader, &file_info) == MZ_OK) + { + // file_info->filename is the complete path of the file from the root of the + // archive. + map.insert( + std::pair( + file_info->filename, + std::string(file_info->filename) + std::to_string(file_info->crc) + ) + ); + } + } while (mz_zip_reader_goto_next_entry(reader) == MZ_OK); + } + } +} + +////////////////////////////////////////////////////////////////////////////////////// + + +////////////////////////////////////////////////////////////////////////////////////// +// Implementation of CIOPOciozArchive class. +////////////////////////////////////////////////////////////////////////////////////// + +void CIOPOciozArchive::getLutData(std::vector & buffer, const char * filepath) const +{ + // In order to ease the implementation and to facilitate a future Python binding, this method + // uses std::vector buffer instead of a std::istream. + // + // It is expected that in most cases, the compiler will be able to avoid copying the buffer + // (RVO/NRVO). + // + // During testings with ocioperf, the first iteration was a little slower using the buffer + // instead of a std::istream (max 5%). But the following iterations are just as fast due to + // the FileTransform cache. + + std::ifstream ociozStream = Platform::CreateInputFileStream( + m_archiveAbsPath.c_str(), + std::ios_base::in | std::ios_base::binary + ); + + if (ociozStream.fail()) + { + std::ostringstream os; + os << "Error could not open OCIOZ archive: " << m_archiveAbsPath; + throw Exception (os.str().c_str()); + } + + ociozStream.seekg(0, std::ios::end); + std::streamsize size = ociozStream.tellg(); + ociozStream.seekg(0, std::ios::beg); + + std::vector archiveBuffer(size); + if (ociozStream.read(archiveBuffer.data(), size)) + { + std::string fpath = pystring::os::path::normpath(filepath); + getFileBufferFromArchive(buffer, fpath, m_archiveAbsPath); + } +} + +const std::string CIOPOciozArchive::getConfigData() const +{ + // In order to ease the implementation and to facilitate a future Python binding, this method + // returns a std::string instead of a std::istream. + // + // It is expected that in most cases, the compiler will be able to avoid copying the buffer + // (RVO/NRVO). + + std::ifstream ociozStream = Platform::CreateInputFileStream( + m_archiveAbsPath.c_str(), + std::ios_base::in | std::ios_base::binary + ); + + if (ociozStream.fail()) + { + std::ostringstream os; + os << "Error could not read OCIOZ archive: " << m_archiveAbsPath; + throw Exception (os.str().c_str()); + } + + std::string configFilename = std::string(OCIO_CONFIG_DEFAULT_NAME) + + std::string(OCIO_CONFIG_DEFAULT_FILE_EXT); + std::vector configBuffer; + getFileBufferFromArchive( + configBuffer, + configFilename, + m_archiveAbsPath + ); + + if (configBuffer.size() > 0) + { + // Create an string stream from the config array of chars. + // Specifiy the size of the string since the char * returned by minizip-ng function + // (mz_zip_reader_entry_save_buffer) is not null-terminated. + return std::string(reinterpret_cast(configBuffer.data()), configBuffer.size()); + } + + return ""; +} + +const std::string CIOPOciozArchive::getFastLutFileHash(const char * filepath) const +{ + std::string hash = ""; + // Check into the map to check if the file exists in the archive. + // The key is the full path of the file inside the archive and the value is the hash. + // Normalize filepath and find it in the std::map. + std::string fpath = pystring::os::path::normpath(filepath); + for (auto it = m_entries.begin(); it != m_entries.end(); ++it) + { + // Verify that the key and the specfied filepath matches while ignoring the slashes + // differences in platforms. + if (mz_path_compare_wc(it->first.c_str(), fpath.c_str(), 1) == MZ_OK) + { + hash = std::string(it->second); + } + } + return hash; +} + +void CIOPOciozArchive::setArchiveAbsPath(std::string absPath) +{ + m_archiveAbsPath = absPath; +} + +void CIOPOciozArchive::buildEntries() +{ + std::ifstream ociozStream = Platform::CreateInputFileStream( + m_archiveAbsPath.c_str(), + std::ios_base::in | std::ios_base::binary + ); + + if (ociozStream.fail()) + { + std::ostringstream os; + os << "Error could not read OCIOZ archive: " << m_archiveAbsPath; + throw Exception (os.str().c_str()); + } + + getEntriesMappingFromArchiveFile(m_archiveAbsPath, m_entries); +} + +} // namespace OCIO_NAMESPACE \ No newline at end of file diff --git a/src/OpenColorIO/OCIOZArchive.h b/src/OpenColorIO/OCIOZArchive.h new file mode 100644 index 0000000000..274b31eb73 --- /dev/null +++ b/src/OpenColorIO/OCIOZArchive.h @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright Contributors to the OpenColorIO Project. + + +#ifndef INCLUDED_OCIO_ARCHIVEUTILS_H +#define INCLUDED_OCIO_ARCHIVEUTILS_H + +#include +#include +#include +#include +#include + +#include + +namespace OCIO_NAMESPACE +{ +/** + * @brief Archive a config into an OCIOZ file. + * + * Note: The config file inside the archive is hardcoded to "config.ocio". + * + * @param ostream Output stream to write the data into. + * @param config Config object. + * @param configWorkingDirectory Working directory of the current config. + */ +void archiveConfig( + std::ostream & ostream, + const Config & config, + const char * configWorkingDirectory); + +/** + * @brief Get the content of a file inside an OCIOZ archive as a buffer. + * + * The file is retrieve by comparing the paths. + * + * @param buffer Buffer + * @param filepath Path to find. + * @param archivePath Path to archive + */ +void getFileBufferFromArchive( + std::vector & buffer, + const std::string & filepath, + const std::string & archivePath); + +/** + * @brief Get the content of a file inside an OCIOZ archive as a buffer. + * + * The file is retrieve by comparing the extensions. + * + * @param buffer Buffer + * @param extension Extension to find + * @param archivePath Path to archive + */ +void getFileBufferFromArchiveByExtension( + std::vector & buffer, + const std::string & extension, + const std::string & archivePath); + +/** + * @brief Get the Entries from OCIOZ archive + * + * Populate a std::map object with the following information: + * key => value : full_path_of_the_file_inside_archive => calculated_hash_of_the_file + * + * The hash is calculated using the full path of the file inside the archive and its CRC32. + * + * @param buffer Path to archive. + * @param map std::map object to be populated + */ +void getEntriesMappingFromArchiveFile( + const std::string & archivePath, + std::map & map); + +////////////////////////////////////////////////////////////////////////////////////// + +class CIOPOciozArchive : public ConfigIOProxy +{ +public: + CIOPOciozArchive() = default; + + // See OpenColorIO.h for informations on these five methods. + void getLutData(std::vector & buffer, const char * filepath) const override; + const std::string getConfigData() const override; + // Currently using the filepath of the file + the CRC32. + const std::string getFastLutFileHash(const char * filepath) const override; + + // Following methods are specific to CIOPOciozArchive. + /** + * @brief Set the OCIOZ archive absolute path. + * + * @param absPath Absolute path to OCIOZ archive. + */ + void setArchiveAbsPath(std::string absPath); + + /** + * @brief Build a map of the zip file table of contents for the files in the archive. + * + * The structure is a std::map with the key as the full path of the file and the value as a + * calculated hash. + */ + void buildEntries(); +private: + std::string m_archiveAbsPath; + std::map m_entries; +}; + +} // namespace OCIO_NAMESPACE + +#endif diff --git a/src/OpenColorIO/PathUtils.cpp b/src/OpenColorIO/PathUtils.cpp index 72a2f65413..9dc8c6b743 100644 --- a/src/OpenColorIO/PathUtils.cpp +++ b/src/OpenColorIO/PathUtils.cpp @@ -12,6 +12,7 @@ #include "Platform.h" #include "pystring/pystring.h" #include "utils/StringUtils.h" +#include "OCIOZArchive.h" #if !defined(_WIN32) #include @@ -59,7 +60,7 @@ void ResetComputeHashFunction() g_hashFunction = Platform::CreateFileContentHash; } -std::string GetFastFileHash(const std::string & filename) +std::string GetFastFileHash(const std::string & filename, const Context & context) { FileHashResultPtr fileHashResultPtr; { @@ -84,18 +85,31 @@ std::string GetFastFileHash(const std::string & filename) // NB: OCIO does not attempt to detect if files have changed and caused the cache to // become stale. fileHashResultPtr->ready = true; - fileHashResultPtr->hash = g_hashFunction(filename); - } + std::string h = ""; + if (!context.getConfigIOProxy()) + { + // Default case. + h = g_hashFunction(filename); + } + else + { + // Case for when ConfigIOProxy is used (callbacks mechanism). + h = context.getConfigIOProxy()->getFastLutFileHash(filename.c_str()); + } + + fileHashResultPtr->hash = h; + } + hash = fileHashResultPtr->hash; } return hash; } -bool FileExists(const std::string & filename) +bool FileExists(const std::string & filename, const Context & context) { - std::string hash = GetFastFileHash(filename); + std::string hash = GetFastFileHash(filename, context); return (!hash.empty()); } diff --git a/src/OpenColorIO/PathUtils.h b/src/OpenColorIO/PathUtils.h index 499451ab83..0f1554d34f 100644 --- a/src/OpenColorIO/PathUtils.h +++ b/src/OpenColorIO/PathUtils.h @@ -18,11 +18,11 @@ namespace OCIO_NAMESPACE std::string AbsPath(const std::string & path); // Check if a file exists -bool FileExists(const std::string & filename); +bool FileExists(const std::string & filename, const Context & context); // Get a fast hash for a file, without reading all the contents. // Currently, this checks the mtime and the inode number. -std::string GetFastFileHash(const std::string & filename); +std::string GetFastFileHash(const std::string & filename, const Context & context); void ClearPathCaches(); diff --git a/src/OpenColorIO/Platform.cpp b/src/OpenColorIO/Platform.cpp index df1a13e176..1dbd846f35 100644 --- a/src/OpenColorIO/Platform.cpp +++ b/src/OpenColorIO/Platform.cpp @@ -262,6 +262,19 @@ void OpenInputFileStream(std::ifstream & stream, const char * filename, std::ios #endif } +#if defined(_WIN32) && defined(UNICODE) +const std::wstring filenameToUTF(const std::string & filename) +{ + // Can not return by reference since it could return std::wstring() or std::string(). + return Utf8ToUtf16(filename); +} +#else +const std::string filenameToUTF(const std::string & filename) +{ + return filename; +} +#endif + std::wstring Utf8ToUtf16(const std::string & str) { if (str.empty()) { @@ -329,8 +342,6 @@ std::string CreateFileContentHash(const std::string &filename) return ""; } - - } // Platform } // namespace OCIO_NAMESPACE diff --git a/src/OpenColorIO/Platform.h b/src/OpenColorIO/Platform.h index c7371eea85..1fcf019956 100644 --- a/src/OpenColorIO/Platform.h +++ b/src/OpenColorIO/Platform.h @@ -95,6 +95,14 @@ std::ifstream CreateInputFileStream(const char * filename, std::ios_base::openmo // Open an input file stream (std::ifstream) using a UTF-8 filename on any platform. void OpenInputFileStream(std::ifstream & stream, const char * filename, std::ios_base::openmode mode); +#if defined(_WIN32) && defined(UNICODE) + // Returns the specified filename string as a UTF16 wstring for Windows. + const std::wstring filenameToUTF(const std::string & str); +#else + // Return the specified filename string as is for Unix-like OS. + const std::string filenameToUTF(const std::string & str); +#endif + // Create a unique hash of a file provided as a UTF-8 filename on any platform. std::string CreateFileContentHash(const std::string &filename); diff --git a/src/OpenColorIO/SystemMonitor_windows.cpp b/src/OpenColorIO/SystemMonitor_windows.cpp index c44df2c111..d981fbe7b4 100644 --- a/src/OpenColorIO/SystemMonitor_windows.cpp +++ b/src/OpenColorIO/SystemMonitor_windows.cpp @@ -37,7 +37,9 @@ void SystemMonitorsImpl::getAllMonitors() { const std::tstring deviceName = dispDevice.DeviceName; - // Only select active monitors. + // Only select active monitors. + // NOTE: Currently the two DISPLAY enums are equivalent, but we check both in case one may + // change in the future. if ((dispDevice.StateFlags & DISPLAY_DEVICE_ACTIVE) && (dispDevice.StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP)) { diff --git a/src/OpenColorIO/transforms/CDLTransform.cpp b/src/OpenColorIO/transforms/CDLTransform.cpp index 27dd87a91c..cee7ed963d 100755 --- a/src/OpenColorIO/transforms/CDLTransform.cpp +++ b/src/OpenColorIO/transforms/CDLTransform.cpp @@ -93,7 +93,8 @@ CDLTransformRcPtr CDLTransform::CreateFromFile(const char * src, const char * cd FileFormat * format = nullptr; CachedFileRcPtr cachedFile; - GetCachedFileAndFormat(format, cachedFile, src, INTERP_DEFAULT); + // The config object won't be used in this use-case. Empty config. + GetCachedFileAndFormat(format, cachedFile, src, INTERP_DEFAULT, *(Config::Create())); GroupTransformRcPtr group = cachedFile->getCDLGroup(); const std::string cdlId{ cdlId_ ? cdlId_ : "" }; @@ -110,7 +111,8 @@ GroupTransformRcPtr CDLTransform::CreateGroupFromFile(const char * src) FileFormat * format = nullptr; CachedFileRcPtr cachedFile; - GetCachedFileAndFormat(format, cachedFile, src, INTERP_DEFAULT); + // The config object won't be used in this use-case. Empty config. + GetCachedFileAndFormat(format, cachedFile, src, INTERP_DEFAULT, *(Config::Create())); return cachedFile->getCDLGroup(); } diff --git a/src/OpenColorIO/transforms/FileTransform.cpp b/src/OpenColorIO/transforms/FileTransform.cpp index 2419ba04d9..5f695f1c64 100755 --- a/src/OpenColorIO/transforms/FileTransform.cpp +++ b/src/OpenColorIO/transforms/FileTransform.cpp @@ -1,12 +1,13 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright Contributors to the OpenColorIO Project. - #include #include #include #include #include +#include +#include #include @@ -14,13 +15,13 @@ #include "FileTransform.h" #include "Logging.h" #include "Mutex.h" +#include "OCIOZArchive.h" #include "ops/noop/NoOps.h" #include "PathUtils.h" #include "Platform.h" #include "pystring/pystring.h" #include "utils/StringUtils.h" - namespace OCIO_NAMESPACE { FileTransformRcPtr FileTransform::Create() @@ -177,6 +178,46 @@ std::ostream& operator<< (std::ostream& os, const FileTransform& t) return os; } +// Wrapper around ConfigIOProxy getLutData implementation. +std::unique_ptr getLutData( + const Config & config, + const std::string & filepath, + std::ios_base::openmode mode) +{ + if (config.getConfigIOProxy()) + { + std::vector buffer; + config.getConfigIOProxy()->getLutData(buffer, filepath.c_str()); + std::stringstream ss; + ss.write(reinterpret_cast(buffer.data()), buffer.size()); + + return std::unique_ptr( + new std::stringstream(std::move(ss)) + ); + } + + // Default behavior. Return file stream. + return std::unique_ptr( + new std::ifstream(Platform::filenameToUTF(filepath).c_str(), mode) + ); +} + +// Close stream returned by getLutData +void closeLutStream(const Config & config, const std::istream & istream) +{ + // No-op when it is using ConfigIOProxy since getLutData returns a vector. + if (config.getConfigIOProxy() == nullptr) + { + // The std::istream is coming from a std::ifstream. + // Pointer cast to ifstream and then close it. + std::ifstream * pIfStream = (std::ifstream *) &istream; + if (pIfStream->is_open()) + { + pIfStream->close(); + } + } +} + bool CollectContextVariables(const Config &, const Context & context, const FileTransform & tr, @@ -193,6 +234,7 @@ bool CollectContextVariables(const Config &, ContextRcPtr ctxFilename = Context::Create(); ctxFilename->setSearchPath(context.getSearchPath()); ctxFilename->setWorkingDir(context.getWorkingDir()); + ctxFilename->setConfigIOProxy(context.getConfigIOProxy()); const std::string resolvedString = context.resolveStringVar(src, ctxFilename); if (0 != strcmp(resolvedString.c_str(), src)) @@ -210,18 +252,19 @@ bool CollectContextVariables(const Config &, ContextRcPtr emptyContext = Context::Create(); emptyContext->setSearchPath(context.getSearchPath()); emptyContext->setWorkingDir(context.getWorkingDir()); + emptyContext->setConfigIOProxy(context.getConfigIOProxy()); // Used to collect the context variables needed to resolve the search_path. Note that this may // contain some variables that are not actually used. ContextRcPtr ctxFilepath = Context::Create(); ctxFilepath->setSearchPath(context.getSearchPath()); ctxFilepath->setWorkingDir(context.getWorkingDir()); + ctxFilepath->setConfigIOProxy(context.getConfigIOProxy()); try { // TODO: resolveFileLocation() tests the file existence (i.e. uses FileExists()) which is // useless here, and it could potentially add some performance penalty. - const std::string resolvedFilename = context.resolveFileLocation(resolvedString.c_str(), ctxFilepath); @@ -513,7 +556,8 @@ namespace void LoadFileUncached(FileFormat * & returnFormat, CachedFileRcPtr & returnCachedFile, const std::string & filepath, - Interpolation interp) + Interpolation interp, + const Config& config) { returnFormat = NULL; @@ -542,15 +586,16 @@ void LoadFileUncached(FileFormat * & returnFormat, { FileFormat * tryFormat = *itFormat; - std::ifstream filestream; + std::unique_ptr pStream = nullptr; try { - // Open the filePath - Platform::OpenInputFileStream( - filestream, - filepath.c_str(), - tryFormat->isBinary() - ? std::ios_base::binary : std::ios_base::in); + pStream = getLutData( + config, + filepath, + tryFormat->isBinary() ? std::ios_base::binary : std::ios_base::in + ); + auto & filestream = *pStream; + if (!filestream.good()) { std::ostringstream os; @@ -573,14 +618,16 @@ void LoadFileUncached(FileFormat * & returnFormat, returnFormat = tryFormat; returnCachedFile = cachedFile; - filestream.close(); + + closeLutStream(config, filestream); + return; } catch(std::exception & e) { - if (filestream.is_open()) + if (pStream) { - filestream.close(); + closeLutStream(config, *pStream); } primaryErrorText += " '"; @@ -616,12 +663,16 @@ void LoadFileUncached(FileFormat * & returnFormat, if(itAlt != endFormat) continue; - std::ifstream filestream; + std::unique_ptr pStream = nullptr; try { - Platform::OpenInputFileStream( - filestream, filepath.c_str(), altFormat->isBinary() - ? std::ios_base::binary : std::ios_base::in); + pStream = getLutData( + config, + filepath, + altFormat->isBinary() ? std::ios_base::binary : std::ios_base::in + ); + auto& filestream = *pStream; + if (!filestream.good()) { std::ostringstream os; @@ -645,16 +696,18 @@ void LoadFileUncached(FileFormat * & returnFormat, returnFormat = altFormat; returnCachedFile = cachedFile; - filestream.close(); + + closeLutStream(config, filestream); + return; } catch(std::exception & e) { - if (filestream.is_open()) + if (pStream) { - filestream.close(); + closeLutStream(config, *pStream); } - + if(IsDebugLoggingEnabled()) { std::ostringstream os; @@ -722,11 +775,11 @@ typedef OCIO_SHARED_PTR FileCacheResultPtr; template class GenericCache; GenericCache g_fileCache; - void GetCachedFileAndFormat(FileFormat * & format, CachedFileRcPtr & cachedFile, const std::string & filepath, - Interpolation interp) + Interpolation interp, + const Config& config) { // Have a two-mutex approach to decouple the cache entry creation from // the data creation. It was originally done to improve the multi-threaded @@ -765,7 +818,7 @@ void GetCachedFileAndFormat(FileFormat * & format, try { - LoadFileUncached(result->format, result->cachedFile, filepath, interp); + LoadFileUncached(result->format, result->cachedFile, filepath, interp, config); } catch (std::exception & e) { @@ -857,7 +910,11 @@ void BuildFileTransformOps(OpRcPtrVec & ops, FileFormat* format = NULL; CachedFileRcPtr cachedFile; - GetCachedFileAndFormat(format, cachedFile, filepath, fileTransform.getInterpolation()); + GetCachedFileAndFormat(format, + cachedFile, + filepath, + fileTransform.getInterpolation(), + config); try { @@ -888,4 +945,5 @@ void BuildFileTransformOps(OpRcPtrVec & ops, throw Exception(err.str().c_str()); } } + } // namespace OCIO_NAMESPACE diff --git a/src/OpenColorIO/transforms/FileTransform.h b/src/OpenColorIO/transforms/FileTransform.h index dbfbba58cf..8754e77978 100644 --- a/src/OpenColorIO/transforms/FileTransform.h +++ b/src/OpenColorIO/transforms/FileTransform.h @@ -112,7 +112,8 @@ class FileFormat void GetCachedFileAndFormat(FileFormat * & format, CachedFileRcPtr & cachedFile, const std::string & filepath, - Interpolation interp); + Interpolation interp, + const Config& config); typedef std::map FileFormatMap; typedef std::vector FileFormatVector; diff --git a/src/apps/CMakeLists.txt b/src/apps/CMakeLists.txt index 4c5563d471..bbedc3c219 100755 --- a/src/apps/CMakeLists.txt +++ b/src/apps/CMakeLists.txt @@ -2,6 +2,7 @@ # Copyright Contributors to the OpenColorIO Project. if(OCIO_BUILD_APPS) + add_subdirectory(ocioarchive) add_subdirectory(ociobakelut) add_subdirectory(ociocheck) add_subdirectory(ociochecklut) diff --git a/src/apps/ocioarchive/CMakeLists.txt b/src/apps/ocioarchive/CMakeLists.txt new file mode 100644 index 0000000000..d71d719aeb --- /dev/null +++ b/src/apps/ocioarchive/CMakeLists.txt @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright Contributors to the OpenColorIO Project. + +set(SOURCES + main.cpp +) + +add_executable(ocioarchive ${SOURCES}) + +set_target_properties(ocioarchive PROPERTIES + COMPILE_FLAGS "${PLATFORM_COMPILE_FLAGS}") + +target_include_directories(ocioarchive + PUBLIC + "$" +) + +target_link_libraries(ocioarchive + PRIVATE + apputils + OpenColorIO + minizip-ng::minizip-ng +) + +install(TARGETS ocioarchive + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} +) diff --git a/src/apps/ocioarchive/main.cpp b/src/apps/ocioarchive/main.cpp new file mode 100644 index 0000000000..dfa8c9c026 --- /dev/null +++ b/src/apps/ocioarchive/main.cpp @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright Contributors to the OpenColorIO Project. + +#include +#include +#include + +#include +namespace OCIO = OCIO_NAMESPACE; + +#include "apputils/argparse.h" + +// Config archive functionality. +#include "minizip-ng/mz.h" +#include "minizip-ng/mz_os.h" +#include "minizip-ng/mz_strm.h" +#include "minizip-ng/mz_zip.h" +#include "minizip-ng/mz_zip_rw.h" + +// Array of non OpenColorIO arguments. +static std::vector args; + +// Fill 'args' array with OpenColorIO arguments. +static int +parse_end_args(int argc, const char *argv[]) +{ + while (argc>0) + { + args.push_back(argv[0]); + argc--; + argv++; + } + + return 0; +} + +int main(int argc, const char **argv) +{ + ArgParse ap; + std::string inputconfig; + std::string archiveName; + std::string configFilename; + // Default value is current directory. + std::string extractDestination; + + bool extract = false; + bool list = false; + bool help = false; + + int32_t err = MZ_OK; + mz_zip_file* file_info = NULL; + void* reader = NULL; + + ap.options("ocioarchive -- Archive a config and its LUT files or extract a config archive. \n\n" + " Note that any existing OCIOZ archive with the same name will be overwritten.\n" + " The .ocioz extension will be added to the archive name, if not provided.\n\n" + "Usage:\n" + " # Archive from the OCIO environment variable into myarchive.ocioz\n" + " ocioarchive myarchive\n\n" + " # Archive myconfig/config.ocio into myarchive.ocioz\n" + " ocioarchive myarchive --iconfig myconfig/config.ocio\n\n" + " # Extract myarchive.ocioz into new directory named myarchive\n" + " ocioarchive --extract myarchive.ocioz\n\n" + " # Extract myarchive.ocioz into new directory named ocio_config\n" + " ocioarchive --extract myarchive.ocioz --newdir ocio_config\n\n" + " # List the files inside myarchive.ocioz\n" + " ocioarchive --list myarchive.ocioz\n", + "%*", parse_end_args, "", + "", "Options:", + "--iconfig %s", &configFilename, "Config to archive (takes precedence over $OCIO)", + "--extract", &extract, "Extract an OCIOZ config archive", + "--dir %s", &extractDestination, "Path where to extract the files (folder are created if missing)", + "--list", &list, "List the files inside an archive without extracting it", + "--help", &help, "Display the help and exit", + "-h", &help, "Display the help and exit", + NULL + ); + + if (ap.parse (argc, argv) < 0) + { + std::cerr << ap.geterror() << std::endl; + exit(1); + } + + if (help || args.size() == 0) + { + ap.usage(); + return 0; + } + + // Archiving. + + if (!extract && !list) + { + if (args.size() != 1) + { + std::cerr << "ERROR: Missing the name of the archive to create." << std::endl; + exit(1); + } + + archiveName = args[0].c_str(); + try + { + const char * ocioEnv = OCIO::GetEnvVariable("OCIO"); + OCIO::ConstConfigRcPtr config; + if (!configFilename.empty()) + { + // Archive a config from a config file (e.g. /home/user/ocio/config.ocio). + try + { + config = OCIO::Config::CreateFromFile(configFilename.c_str()); + } + catch (...) + { + // Capture any errors and display a custom message. + std::cerr << "ERROR: Could not load config: " << configFilename << std::endl; + exit(1); + } + + } + else if (ocioEnv && *ocioEnv) + { + // Archive a config from the environment variable. + std::cout << "Archiving $OCIO=" << ocioEnv << std::endl; + try + { + config = OCIO::Config::CreateFromEnv(); + } + catch (...) + { + // Capture any errors and display a custom message. + std::cerr << "ERROR: Could not load config from $OCIO variable: " + << ocioEnv << std::endl; + exit(1); + } + } + else + { + std::cerr << "ERROR: You must specify an input OCIO configuration." << std::endl; + exit(1); + } + + try + { + // The ocioz extension is added by the archive method. The assumption is that + // archiveName is the filename without extension. + + // Remove extension, if present. + char * archiveNameTmp = const_cast(archiveName.c_str()); + mz_path_remove_extension(archiveNameTmp); + archiveName = archiveNameTmp; + + std::ofstream ofstream(archiveName + std::string(OCIO::OCIO_CONFIG_ARCHIVE_FILE_EXT), + std::ofstream::out | std::ofstream::binary); + if (ofstream.good()) + { + config->archive(ofstream); + ofstream.close(); + } + else + { + std::cerr << "Could not open output stream for: " + << archiveName + std::string(OCIO::OCIO_CONFIG_ARCHIVE_FILE_EXT) + << std::endl; + exit(1); + } + } + catch (OCIO::Exception & e) + { + std::cerr << e.what() << std::endl; + exit(1); + } + } + catch (OCIO::Exception & exception) + { + std::cerr << "ERROR: " << exception.what() << std::endl; + exit(1); + } + catch (std::exception& exception) + { + std::cerr << "ERROR: " << exception.what() << std::endl; + exit(1); + } + catch (...) + { + std::cerr << "ERROR: Unknown problem encountered." << std::endl; + exit(1); + } + } + + // Extracting. + + else if (extract && !list) + { + if (args.size() != 1) + { + std::cerr << "ERROR: Missing the name of the archive to extract." << std::endl; + exit(1); + } + + archiveName = args[0].c_str(); + try + { + if (extractDestination.empty()) + { + // Set the default directory name to the name of the archive. + extractDestination = archiveName; + + // Remove extension, if present. + char * dst = const_cast(extractDestination.c_str()); + mz_path_remove_extension(dst); + extractDestination = dst; + } + + OCIO::ExtractOCIOZArchive(archiveName.c_str(), extractDestination.c_str()); + std::cout << archiveName << " has been extracted." << std::endl; + } + catch (OCIO::Exception & e) + { + std::cerr << e.what() << std::endl; + exit(1); + } + } + + // Listing. + + else if (list && !extract) + { + if (args.size() < 1) + { + std::cerr << "ERROR: Missing the name of the archive to list." << std::endl; + exit(1); + } + + std::string path = args[0]; + mz_zip_reader_create(&reader); + struct tm tmu_date; + + err = mz_zip_reader_open_file(reader, path.c_str()); + if (err != MZ_OK) + { + std::cerr << "ERROR: File not found: " << path << std::endl; + exit(1); + } + + err = mz_zip_reader_goto_first_entry(reader); + if (err != MZ_OK) + { + std::cerr << "ERROR: Could not find the first entry in the archive." << std::endl; + exit(1); + } + + std::cout << "\nThe archive contains the following files:\n" << std::endl; + std::cout << " Date Time CRC-32 Name" << std::endl; + std::cout << " ---- ---- ------ ----" << std::endl; + do + { + err = mz_zip_reader_entry_get_info(reader, &file_info); + if (err != MZ_OK) + { + std::cerr << "ERROR: Could not get information from entry: " << file_info->filename + << std::endl; + exit(1); + } + + mz_zip_time_t_to_tm(file_info->modified_date, &tmu_date); + + // Print entry information. + printf(" %2.2" PRIu32 "-%2.2" PRIu32 "-%2.2" PRIu32 " %2.2" PRIu32 \ + ":%2.2" PRIu32 " %8.8" PRIx32 " %s\n", + (uint32_t)tmu_date.tm_mon + 1, (uint32_t)tmu_date.tm_mday, + (uint32_t)tmu_date.tm_year % 100, + (uint32_t)tmu_date.tm_hour, (uint32_t)tmu_date.tm_min, + file_info->crc, file_info->filename); + + err = mz_zip_reader_goto_next_entry(reader); + } while (err == MZ_OK); + } + + // Generic error handling. + + else + { + std::cerr << "Archive, extract, and/or list functions " + "may not be used at the same time." << std::endl; + exit(1); + } +} diff --git a/src/apps/ociocheck/main.cpp b/src/apps/ociocheck/main.cpp index ae31f71062..26989f77bb 100644 --- a/src/apps/ociocheck/main.cpp +++ b/src/apps/ociocheck/main.cpp @@ -81,6 +81,11 @@ int main(int argc, const char **argv) return 1; } + std::cout << std::endl; + std::cout << "** Miscellaneous **" << std::endl; + std::cout << "CacheID: " << config->getCacheID() << std::endl; + std::cout << "Archivable: " << (config->isArchivable() ? "yes" : "no") << std::endl; + std::cout << std::endl; std::cout << "** General **" << std::endl; diff --git a/src/bindings/python/CMakeLists.txt b/src/bindings/python/CMakeLists.txt index 2b4e311aa0..f674139934 100644 --- a/src/bindings/python/CMakeLists.txt +++ b/src/bindings/python/CMakeLists.txt @@ -84,6 +84,7 @@ set(SOURCES PyColorSpace.cpp PyColorSpaceSet.cpp PyConfig.cpp + PyConfigIOProxy.cpp PyContext.cpp PyCPUProcessor.cpp PyDynamicProperty.cpp diff --git a/src/bindings/python/PyConfig.cpp b/src/bindings/python/PyConfig.cpp index 2eae7a8764..f9a38911fc 100644 --- a/src/bindings/python/PyConfig.cpp +++ b/src/bindings/python/PyConfig.cpp @@ -200,6 +200,8 @@ void bindPyConfig(py::module & m) .def_static("CreateFromBuiltinConfig", &Config::CreateFromBuiltinConfig, DOC(Config, CreateFromBuiltinConfig)) + .def_static("CreateFromConfigIOProxy", &Config::CreateFromConfigIOProxy, + DOC(Config, CreateFromConfigIOProxy)) .def("getMajorVersion", &Config::getMajorVersion, DOC(Config, getMajorVersion)) .def("setMajorVersion", &Config::setMajorVersion, "major"_a, @@ -555,7 +557,7 @@ void bindPyConfig(py::module & m) .def("clearViewTransforms", &Config::clearViewTransforms, DOC(Config, clearViewTransforms)) - // Named Transforms. + // Named Transforms .def("getNamedTransform", &Config::getNamedTransform, "name"_a) .def("getNamedTransformNames", [](ConfigRcPtr & self, @@ -729,7 +731,18 @@ void bindPyConfig(py::module & m) .def("setProcessorCacheFlags", &Config::setProcessorCacheFlags, "flags"_a, DOC(Config, setProcessorCacheFlags)) - + + // Archiving + .def("isArchivable", &Config::isArchivable, DOC(Config, isArchivable)) + .def("archive", [](ConfigRcPtr & self, const std::string filepath) + { + std::ofstream f(filepath.c_str(), std::ofstream::out | std::ofstream::binary); + self->archive(f); + f.close(); + }, + DOC(Config, isArchivable)) + + // Conversion to string .def("__str__", [](ConfigRcPtr & self) { std::ostringstream os; @@ -1207,6 +1220,9 @@ void bindPyConfig(py::module & m) DOC(PyOpenColorIO, GetCurrentConfig)); m.def("SetCurrentConfig", &SetCurrentConfig, "config"_a, DOC(PyOpenColorIO, SetCurrentConfig)); + + m.def("ExtractOCIOZArchive", &ExtractOCIOZArchive, + DOC(PyOpenColorIO, ExtractOCIOZArchive)); } } // namespace OCIO_NAMESPACE diff --git a/src/bindings/python/PyConfigIOProxy.cpp b/src/bindings/python/PyConfigIOProxy.cpp new file mode 100644 index 0000000000..2a18961720 --- /dev/null +++ b/src/bindings/python/PyConfigIOProxy.cpp @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright Contributors to the OpenColorIO Project. + +#include +#include + +#include "PyOpenColorIO.h" +#include "PyUtils.h" + +#include +#include + +PYBIND11_MAKE_OPAQUE(std::vector); + +namespace OCIO_NAMESPACE +{ +struct PyConfigIOProxy : ConfigIOProxy +{ + using ConfigIOProxy::ConfigIOProxy; + + void getLutData(std::vector & buffer, const char * filepath) const override + { + py::object dummy = py::cast(&buffer); + PYBIND11_OVERRIDE_PURE( + void, + ConfigIOProxy, // Parent class + getLutData, // Name of function in C++ (must match Python name) + buffer, + filepath // Argument(s) + ); + } + + const std::string getConfigData() const override + { + PYBIND11_OVERRIDE_PURE( + std::string, + ConfigIOProxy, // Parent class + getConfigData, // Name of function in C++ (must match Python name) + ); + } + + const std::string getFastLutFileHash(const char * filepath) const override + { + PYBIND11_OVERRIDE_PURE( + std::string, // Return type + ConfigIOProxy, // Parent class + getFastLutFileHash, // Name of function in C++ (must match Python name) + filepath // Argument(s) + ); + } +}; + +void bindPyConfigIOProxy(py::module & m) +{ + py::bind_vector>(m, "vector_of_uint8_t"); + + py::class_, PyConfigIOProxy>(m, "PyConfigIOProxy") + .def(py::init()) + .def("getLutData", &ConfigIOProxy::getLutData) + .def("getConfigData", &ConfigIOProxy::getConfigData) + .def("getFastLutFileHash", &ConfigIOProxy::getFastLutFileHash); +} + +} \ No newline at end of file diff --git a/src/bindings/python/PyOpenColorIO.cpp b/src/bindings/python/PyOpenColorIO.cpp index ca9a21d263..7adda36bb3 100644 --- a/src/bindings/python/PyOpenColorIO.cpp +++ b/src/bindings/python/PyOpenColorIO.cpp @@ -74,6 +74,7 @@ PYBIND11_MODULE(PyOpenColorIO, m) bindPyColorSpaceSet(m); bindPyConfig(m); bindPyContext(m); + bindPyConfigIOProxy(m); bindPyCPUProcessor(m); bindPyFileRules(m); bindPyGPUProcessor(m); diff --git a/src/bindings/python/PyOpenColorIO.h b/src/bindings/python/PyOpenColorIO.h index fa68ade3d8..6fc291ada4 100644 --- a/src/bindings/python/PyOpenColorIO.h +++ b/src/bindings/python/PyOpenColorIO.h @@ -33,6 +33,7 @@ void bindPyColorSpace(py::module & m); void bindPyColorSpaceSet(py::module & m); void bindPyConfig(py::module & m); void bindPyContext(py::module & m); +void bindPyConfigIOProxy(py::module & m); void bindPyCPUProcessor(py::module & m); void bindPyFileRules(py::module & m); void bindPyGPUProcessor(py::module & m); diff --git a/tests/cpu/CMakeLists.txt b/tests/cpu/CMakeLists.txt index 7350897982..657038ad6f 100755 --- a/tests/cpu/CMakeLists.txt +++ b/tests/cpu/CMakeLists.txt @@ -23,6 +23,7 @@ function(add_ocio_test NAME SOURCES PRIVATE_INCLUDES) utils::strings yaml-cpp testutils + minizip-ng::minizip-ng ) if(APPLE) @@ -93,6 +94,7 @@ set(SOURCES Look.cpp md5/md5.cpp OCIOYaml.cpp + OCIOZArchive.cpp ops/cdl/CDLOpCPU.cpp ops/cdl/CDLOpGPU.cpp ops/exposurecontrast/ExposureContrastOpGPU.cpp @@ -169,6 +171,7 @@ set(TESTS LookParse_tests.cpp MathUtils_tests.cpp NamedTransform_tests.cpp + OCIOZArchive_tests.cpp Op_tests.cpp OpOptimizers_tests.cpp ops/allocation/AllocationOp_tests.cpp diff --git a/tests/cpu/Config_tests.cpp b/tests/cpu/Config_tests.cpp index 3020cdce80..e987ccdefe 100644 --- a/tests/cpu/Config_tests.cpp +++ b/tests/cpu/Config_tests.cpp @@ -8813,3 +8813,268 @@ OCIO_ADD_TEST(Config, create_builtin_config) ); } } + +OCIO_ADD_TEST(Config, create_from_archive) +{ + { + // CreateFromFile using an archive built on Windows. + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string("context_test1_windows.ocioz") + }; + + const std::string archivePath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + OCIO::ConstConfigRcPtr config; + + OCIO_CHECK_NO_THROW(config = OCIO::Config::CreateFromFile(archivePath.c_str())); + OCIO_REQUIRE_ASSERT(config); + OCIO_CHECK_NO_THROW(config->validate()); + + // Simple check on the number of color spaces in the test config. + OCIO_CHECK_EQUAL(config->getNumColorSpaces(), 13); + + // Simple test to exercise ConfigIOProxy. + OCIO::ConstProcessorRcPtr proc; + OCIO_CHECK_NO_THROW(proc = config->getProcessor("plain_lut1_cs", "shot1_lut1_cs")); + OCIO_REQUIRE_ASSERT(proc); + OCIO_CHECK_NO_THROW(proc->getDefaultCPUProcessor()); + } + + { + // CreateFromFile using an archive built with Unix-style path. + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string("context_test1_linux.ocioz") + }; + + const std::string archivePath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + OCIO::ConstConfigRcPtr config; + + OCIO_CHECK_NO_THROW(config = OCIO::Config::CreateFromFile(archivePath.c_str())); + OCIO_REQUIRE_ASSERT(config); + OCIO_CHECK_NO_THROW(config->validate()); + + // Simple check on the number of color spaces in the test config. + OCIO_CHECK_EQUAL(config->getNumColorSpaces(), 13); + + // Simple test to exercise ConfigIOProxy. + OCIO::ConstProcessorRcPtr proc; + OCIO_CHECK_NO_THROW(proc = config->getProcessor("plain_lut1_cs", "shot1_lut1_cs")); + OCIO_REQUIRE_ASSERT(proc); + OCIO_CHECK_NO_THROW(proc->getDefaultCPUProcessor()); + } + + // Scenario with incomplete OCIOZ archive. + { + // Empty OCIOZ archive. + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("ocioz_archive_configs"), + std::string("empty.ocioz") + }; + + const std::string archivePath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + OCIO_CHECK_THROW_WHAT( + OCIO::Config::CreateFromFile(archivePath.c_str()), + OCIO::Exception, + "Loading the OCIO profile failed. At line 0, '' parsing failed: The specified OCIO "\ + "configuration file from Archive/ConfigIOProxy does not appear to have a valid version"\ + " " + ); + } + + { + // Missing config file but contains LUT files. + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("ocioz_archive_configs"), + std::string("missing_config.ocioz") + }; + + const std::string archivePath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + OCIO_CHECK_THROW_WHAT( + OCIO::Config::CreateFromFile(archivePath.c_str()), + OCIO::Exception, + "Loading the OCIO profile failed. At line 0, '' parsing failed: The specified OCIO "\ + "configuration file from Archive/ConfigIOProxy does not appear to have a valid version"\ + " " + ); + } + + { + // Missing LUT files but contains config file. + // FileTransform will requests a file that is not in the archive. + + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("ocioz_archive_configs"), + std::string("config_missing_luts.ocioz") + }; + + const std::string archivePath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + OCIO::ConstConfigRcPtr config; + OCIO_CHECK_NO_THROW(config = OCIO::Config::CreateFromFile(archivePath.c_str())); + OCIO_REQUIRE_ASSERT(config); + // The following validation will succeed because validate() does not try to fetch the LUT + // files. It resolves the context variables in the paths string. + OCIO_CHECK_NO_THROW(config->validate()); + + + // Trying to getProcessor from ocioz archive without any LUT files but the config needs them. + // Will throw in resolveFileLocation. +#ifdef _WIN32 + OCIO_CHECK_THROW_WHAT( + config->getProcessor("plain_lut11_cs", "shot1_lut11_cs"), + OCIO::Exception, + "The specified file reference 'lut11.clf' could not be located. The following "\ + "attempts were made: 'shot4\\lut11.clf' : 'shot1\\lut11.clf' : 'shot2\\lut11.clf' : "\ + "'shot3\\lut11.clf' : 'shot3\\subdir\\lut11.clf' : '.\\lut11.clf'." + ); +#else + OCIO_CHECK_THROW_WHAT( + config->getProcessor("plain_lut11_cs", "shot1_lut11_cs"), + OCIO::Exception, + "The specified file reference 'lut11.clf' could not be located. The following "\ + "attempts were made: 'shot4/lut11.clf' : 'shot1/lut11.clf' : 'shot2/lut11.clf' : "\ + "'shot3/lut11.clf' : 'shot3/subdir/lut11.clf' : './lut11.clf'." + ); +#endif + } +} + +OCIO_ADD_TEST(Config, create_from_config_io_proxy) +{ + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string("config.ocio"), + }; + static const std::string configPath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + { + // Provide a very minimal implementation of the ConfigIOProxy. + // (It's primitive but meets the needs of this test.) + class CIOPTest : public OCIO::ConfigIOProxy + { + public: + inline const std::string getConfigData() const override + { + // Get config data from filesystem, database, memory, etc. + // In this example, the config is simply coming from the filesystem. + // Return a stream to the config. + std::ifstream fstream( + OCIO::Platform::filenameToUTF(configPath).c_str(), std::ios_base::in); + if (fstream.fail()) + { + std::ostringstream os; + os << "Error could not read config file : " << configPath; + throw OCIO::Exception (os.str().c_str()); + } + + std::stringstream buffer; + buffer << fstream.rdbuf(); + return buffer.str(); + } + + inline void getLutData( + std::vector & buffer, + const char * filepath) const override + { + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string(filepath), + }; + const std::string lutPath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + std::ifstream fstream( + OCIO::Platform::filenameToUTF(lutPath).c_str(), + std::ios_base::in | std::ios_base::binary | std::ios::ate + ); + if (fstream.fail()) + { + std::ostringstream os; + os << "Error could not read LUT file : " << lutPath; + throw OCIO::Exception (os.str().c_str()); + } + + const auto eofPosition = static_cast(fstream.tellg()); + buffer.resize(eofPosition); + fstream.seekg(0, std::ios::beg); + fstream.read(reinterpret_cast(buffer.data()), eofPosition); + } + + const std::string getFastLutFileHash(const char * filename) const override + { + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string(filename), + }; + const std::string lutPath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + // Check if the file is present. + std::ifstream f(OCIO::Platform::filenameToUTF(lutPath).c_str(), std::ios_base::in); + if (f.good()) + { + // This is a bad hash, simply using the filename as the hash for simplicity and + // demonstration purposes. + return lutPath; + } + + // The ifstream above is closed at the end of the scope by its destructor. + // Return empty since the file couldn't be found. + return ""; + } + }; + + std::shared_ptr ciop = std::shared_ptr( + new CIOPTest() + ); + + OCIO::ConstConfigRcPtr config; + OCIO_CHECK_NO_THROW(config = OCIO::Config::CreateFromConfigIOProxy(ciop)); + OCIO_REQUIRE_ASSERT(config); + OCIO_CHECK_NO_THROW(config->validate()); + + // Simple check on the number of color spaces in the test config. + OCIO_CHECK_EQUAL(config->getNumColorSpaces(), 13); + + // Simple test to exercise ConfigIOProxy. + OCIO::ConstProcessorRcPtr proc; + OCIO_CHECK_NO_THROW(proc = config->getProcessor("plain_lut1_cs", "shot1_lut1_cs")); + OCIO_REQUIRE_ASSERT(proc); + OCIO_CHECK_NO_THROW(proc->getDefaultCPUProcessor()); + } +} diff --git a/tests/cpu/OCIOZArchive_tests.cpp b/tests/cpu/OCIOZArchive_tests.cpp new file mode 100644 index 0000000000..5a9b171ab6 --- /dev/null +++ b/tests/cpu/OCIOZArchive_tests.cpp @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright Contributors to the OpenColorIO Project. + +#include "OpenColorIO/OpenColorIO.h" +#include "testutils/UnitTest.h" +#include "UnitTestUtils.h" + +namespace OCIO = OCIO_NAMESPACE; + +namespace +{ +struct FileCreationGuard +{ + explicit FileCreationGuard(unsigned lineNo) + { + OCIO_CHECK_NO_THROW_FROM(m_filename = OCIO::Platform::CreateTempFilename(""), lineNo); + } + ~FileCreationGuard() + { + // Even if not strictly required on most OSes, perform the cleanup. + std::remove(m_filename.c_str()); + } + + std::string m_filename; +}; + +struct DirectoryCreationGuard +{ + explicit DirectoryCreationGuard(const std::string name, unsigned lineNo) + { + OCIO_CHECK_NO_THROW_FROM( + m_directoryPath = OCIO::CreateTemporaryDirectory(name), lineNo + ); + } + ~DirectoryCreationGuard() + { + // Even if not strictly required on most OSes, perform the cleanup. + OCIO::RemoveTemporaryDirectory(m_directoryPath); + } + + std::string m_directoryPath; +}; +} //anon. + + +OCIO_ADD_TEST(OCIOZArchive, is_config_archivable) +{ + // This test primarly tests the isArchivable method from the Config object + + const std::string CONFIG = + "ocio_profile_version: 2\n" + "\n" + "search_path:\n" + " - abc\n" + " - def\n" + "environment:\n" + " MYLUT: exposure_contrast_linear.ctf\n" + "\n" + "roles:\n" + " default: cs1\n" + "\n" + "displays:\n" + " disp1:\n" + " - ! {name: view1, colorspace: cs2}\n" + "\n" + "colorspaces:\n" + " - !\n" + " name: cs1\n" + "\n" + " - !\n" + " name: cs2\n" + " from_scene_reference: ! {src: ./$MYLUT}\n"; + + std::istringstream iss; + iss.str(CONFIG); + + OCIO::ConfigRcPtr cfg; + OCIO_CHECK_NO_THROW(cfg = OCIO::Config::CreateFromStream(iss)->createEditableCopy()); + // Since a working directory is needed to archive a config, settting a fake working directory + // in order to test the search paths and FileTransform source logic. +#ifdef _WIN32 + cfg->setWorkingDir(R"(C:\fake_working_dir)"); +#else + cfg->setWorkingDir("/fake_working_dir"); +#endif + OCIO_CHECK_NO_THROW(cfg->validate()); + + // Testing a few scenario by modifying the search_paths. + + // Testing search paths + { + /* + * Legal scenarios + */ + + // No search path. + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + // Valid search path. + cfg->setSearchPath("luts"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + cfg->setSearchPath(R"(luts/myluts1)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + cfg->setSearchPath(R"(luts\myluts1)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + // Valid Search path starting with "./" or ".\". + cfg->setSearchPath(R"(./myLuts)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + cfg->setSearchPath(R"(.\myLuts)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + // Valid search path starting with "./" or ".\" and a context variable. + cfg->setSearchPath(R"(./$SHOT/myluts)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + cfg->setSearchPath(R"(.\$SHOT\myluts)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + cfg->setSearchPath(R"(luts/$SHOT)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + cfg->setSearchPath(R"(luts/$SHOT/luts1)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + cfg->setSearchPath(R"(luts\$SHOT)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + cfg->setSearchPath(R"(luts\$SHOT\luts1)"); + OCIO_CHECK_EQUAL(true, cfg->isArchivable()); + + /* + * Illegal scenarios + */ + + // Illegal search path starting with "..". + cfg->setSearchPath(R"(luts:../luts)"); + OCIO_CHECK_EQUAL(false, cfg->isArchivable()); + + cfg->setSearchPath(R"(luts:..\myLuts)"); + OCIO_CHECK_EQUAL(false, cfg->isArchivable()); + + // Illegal search path starting with a context variable. + cfg->setSearchPath(R"(luts:$SHOT)"); + OCIO_CHECK_EQUAL(false, cfg->isArchivable()); + + // Illegal search path with absolute path. + cfg->setSearchPath(R"(luts:/luts)"); + OCIO_CHECK_EQUAL(false, cfg->isArchivable()); + + cfg->setSearchPath(R"(luts:/$SHOT)"); + OCIO_CHECK_EQUAL(false, cfg->isArchivable()); + +#ifdef _WIN32 + cfg->clearSearchPaths(); + cfg->addSearchPath(R"(C:\luts)"); + OCIO_CHECK_EQUAL(false, cfg->isArchivable()); + + cfg->clearSearchPaths(); + cfg->addSearchPath(R"(C:\)"); + OCIO_CHECK_EQUAL(false, cfg->isArchivable()); + + cfg->clearSearchPaths(); + cfg->addSearchPath(R"(C:\$SHOT)"); + OCIO_CHECK_EQUAL(false, cfg->isArchivable()); +#endif + } + + // Clear search paths so it doesn't affect the tests below. + cfg->clearSearchPaths(); + + // Lambda function to facilitate adding a new FileTransform to a config. + auto addFTAndTestIsArchivable = [&cfg](const std::string & path, bool isArchivable) + { + std::string fullPath = pystring::os::path::join(path, "fake_lut.clf"); + auto ft = OCIO::FileTransform::Create(); + ft->setSrc(fullPath.c_str()); + + auto cs = OCIO::ColorSpace::Create(); + cs->setName("csTest"); + cs->setTransform(ft, OCIO::COLORSPACE_DIR_TO_REFERENCE); + cfg->addColorSpace(cs); + + OCIO_CHECK_EQUAL(isArchivable, cfg->isArchivable()); + + cfg->removeColorSpace("csTest"); + }; + + // Testing FileTransfrom paths + { + /* + * Legal scenarios + */ + + // Valid FileTransfrom path. + addFTAndTestIsArchivable("luts", true); + addFTAndTestIsArchivable(R"(luts/myluts1)", true); + addFTAndTestIsArchivable(R"(luts\myluts1)", true); + + // Valid Search path starting with "./" or ".\". + addFTAndTestIsArchivable(R"(./myLuts)", true); + addFTAndTestIsArchivable(R"(.\myLuts)", true); + + // Valid search path starting with "./" or ".\" and a context variable. + addFTAndTestIsArchivable(R"(./$SHOT/myluts)", true); + addFTAndTestIsArchivable(R"(.\$SHOT\myluts)", true); + addFTAndTestIsArchivable(R"(luts/$SHOT)", true); + addFTAndTestIsArchivable(R"(luts/$SHOT/luts1)", true); + addFTAndTestIsArchivable(R"(luts\$SHOT)", true); + addFTAndTestIsArchivable(R"(luts\$SHOT\luts1)", true); + + /* + * Illegal scenarios + */ + + // Illegal search path starting with "..". + addFTAndTestIsArchivable(R"(../luts)", false); + addFTAndTestIsArchivable(R"(..\myLuts)", false); + + // Illegal search path starting with a context variable. + addFTAndTestIsArchivable(R"($SHOT)", false); + + // Illegal search path with absolute path. + addFTAndTestIsArchivable(R"(/luts)", false); + addFTAndTestIsArchivable(R"(/$SHOT)", false); + +#ifdef _WIN32 + addFTAndTestIsArchivable(R"(C:\luts)", false); + addFTAndTestIsArchivable(R"(C:\)", false); + addFTAndTestIsArchivable(R"(\$SHOT)", false); +#endif + } +} + +OCIO_ADD_TEST(OCIOZArchive, context_test_for_search_paths_and_filetransform_source_path) +{ + std::vector pathsWindows = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string("context_test1_windows.ocioz") + }; + static const std::string archivePathWindows = pystring::os::path::normpath( + pystring::os::path::join(pathsWindows) + ); + + OCIO::ConfigRcPtr cfgWindowsArchive; + OCIO_CHECK_NO_THROW( + cfgWindowsArchive = OCIO::Config::CreateFromFile( + archivePathWindows.c_str())->createEditableCopy()); + OCIO_CHECK_NO_THROW(cfgWindowsArchive->validate()); + + std::vector pathsLinux = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string("context_test1_linux.ocioz") + }; + static const std::string archivePathLinux = pystring::os::path::normpath( + pystring::os::path::join(pathsLinux) + ); + + OCIO::ConfigRcPtr cfgLinuxArchive; + OCIO_CHECK_NO_THROW( + cfgLinuxArchive = OCIO::Config::CreateFromFile( + archivePathLinux.c_str())->createEditableCopy()); + OCIO_CHECK_NO_THROW(cfgLinuxArchive->validate()); + + // OCIO will pick up context vars from the environment that runs the test, + // so set these explicitly, even though the config has default values. + + OCIO::ContextRcPtr ctxWindowsArchive = cfgWindowsArchive->getCurrentContext()->createEditableCopy(); + ctxWindowsArchive->setStringVar("SHOT", "none"); + ctxWindowsArchive->setStringVar("LUT_PATH", "none"); + ctxWindowsArchive->setStringVar("CAMERA", "none"); + ctxWindowsArchive->setStringVar("CCCID", "none"); + + OCIO::ContextRcPtr ctxLinuxArchive = cfgLinuxArchive->getCurrentContext()->createEditableCopy(); + ctxLinuxArchive->setStringVar("SHOT", "none"); + ctxLinuxArchive->setStringVar("LUT_PATH", "none"); + ctxLinuxArchive->setStringVar("CAMERA", "none"); + ctxLinuxArchive->setStringVar("CCCID", "none"); + + double mat[16] = { 0., 0., 0., 0., + 0., 0., 0., 0., + 0., 0., 0., 0., + 0., 0., 0., 0. }; + + auto testPaths = [&mat](const OCIO::ConfigRcPtr & cfg, const OCIO::ContextRcPtr ctx) + { + { + // This is independent of the context. + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "shot1_lut1_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 10.); + } + + { + // This is independent of the context. + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "shot2_lut1_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 20.); + } + + { + // This is independent of the context. + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "shot2_lut2_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 2.); + } + + { + // This is independent of the context but the file is in a second level sub-directory. + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "lut3_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 3.); + } + + { + // This uses a context for the FileTransform src but is independent of the search_path. + ctx->setStringVar("LUT_PATH", "shot3/lut1.clf"); + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "lut_path_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 30.); + } + + { + // The FileTransform src is ambiguous and the context configures the search_path. + ctx->setStringVar("SHOT", "."); // (use the working directory) + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "plain_lut1_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 5.); + } + + { + // The FileTransform src is ambiguous and the context configures the search_path. + ctx->setStringVar("SHOT", "shot2"); + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "plain_lut1_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 20.); + } + + { + // The FileTransform src is ambiguous and the context configures the search_path. + ctx->setStringVar("SHOT", "no_shot"); // (path doesn't exist) + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "plain_lut1_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 10.); + } + + { + // This should be in the archive but is not on the search path at all without the context. + ctx->setStringVar("SHOT", "no_shot"); // (path doesn't exist) + OCIO::ConstProcessorRcPtr processor; + OCIO_CHECK_THROW( + processor = cfg->getProcessor(ctx, "lut4_cs", "reference"), + OCIO::ExceptionMissingFile + ) + } + + { + // This should be in the archive but is not on the search path at all without the context. + ctx->setStringVar("SHOT", "shot4"); + OCIO::ConstProcessorRcPtr processor = cfg->getProcessor(ctx, "lut4_cs", "reference"); + OCIO::ConstTransformRcPtr tr = processor->createGroupTransform()->getTransform(0); + auto mtx = OCIO::DynamicPtrCast(tr); + OCIO_REQUIRE_ASSERT(mtx); + mtx->getMatrix(mat); + OCIO_CHECK_EQUAL(mat[0], 4.); + } + }; + + testPaths(cfgWindowsArchive, ctxWindowsArchive); + testPaths(cfgLinuxArchive, ctxLinuxArchive); +} + +OCIO_ADD_TEST(OCIOZArchive, archive_config_and_compare_to_original) +{ + /** + * This test is doing the following : + * + * 1 - Create a config object from tests/data/files/configs/context_test1/config.ocio. + * 2 - Archive the config in step 1 and save it in a temporary folder. + * 3 - Create a config object using the archived config from step 2. + * 4 - Compare different elements between the two configs. + * + * Testing CreateFromFile and archive method on a successful path. + */ + + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string("config.ocio"), + }; + const std::string configPath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + { + OCIO::EnvironmentVariableGuard guard("OCIO", configPath); + + // 1 - Create a config from a OCIO file. + OCIO::ConstConfigRcPtr configFromOcioFile; + OCIO_CHECK_NO_THROW(configFromOcioFile = OCIO::Config::CreateFromEnv()); + OCIO_REQUIRE_ASSERT(configFromOcioFile); + OCIO_CHECK_NO_THROW(configFromOcioFile->validate()); + + // 2 - Archive the config of step 1. + std::ostringstream ostringStream; + OCIO_CHECK_NO_THROW(configFromOcioFile->archive(ostringStream)); + + // 3 - Verify that the binary data starts with "PK". + OCIO_CHECK_EQUAL('P', ostringStream.str()[0]); + OCIO_CHECK_EQUAL('K', ostringStream.str()[1]); + + // 4 - Save the archive into a temporary file. + FileCreationGuard fGuard(__LINE__); + std::ofstream stream(fGuard.m_filename, std::ios_base::out | std::ios_base::binary); + OCIO_REQUIRE_ASSERT(stream.is_open()); + stream << ostringStream.str(); + stream.close(); + + // 5 - Create a config from the archived config of step 4. + OCIO::ConstConfigRcPtr configFromArchive; + OCIO_CHECK_NO_THROW(configFromArchive = OCIO::Config::CreateFromFile( + fGuard.m_filename.c_str() + )); + OCIO_REQUIRE_ASSERT(configFromArchive); + OCIO_CHECK_NO_THROW(configFromArchive->validate()); + + // 6 - Compare config cacheID - configFromOcioFile vs configFromArchive. + OCIO::ConstContextRcPtr context; + OCIO_CHECK_EQUAL( + std::string(configFromOcioFile->getCacheID(context)), + std::string(configFromArchive->getCacheID(context)) + ); + + // 7 - Compare a processor cacheID - configFromOcioFile to configFromArchive. + OCIO::ConstProcessorRcPtr procConfigFromOcioFile, procConfigFromArchive; + OCIO_CHECK_NO_THROW(procConfigFromOcioFile = configFromOcioFile->getProcessor( + "plain_lut1_cs", + "shot1_lut1_cs" + )); + OCIO_REQUIRE_ASSERT(procConfigFromOcioFile); + + OCIO_CHECK_NO_THROW(procConfigFromArchive = configFromArchive->getProcessor( + "plain_lut1_cs", + "shot1_lut1_cs" + )); + OCIO_REQUIRE_ASSERT(procConfigFromArchive); + + OCIO_CHECK_EQUAL( + std::string(procConfigFromOcioFile->getCacheID()), + std::string(procConfigFromArchive->getCacheID()) + ); + + // 8 - Compare serialization - configFromOcioFile vs configFromArchive. + std::ostringstream streamToConfigFromOcioFile, streamToConfigFromArchive; + configFromOcioFile->serialize(streamToConfigFromOcioFile); + configFromArchive->serialize(streamToConfigFromArchive); + OCIO_CHECK_EQUAL( + streamToConfigFromOcioFile.str(), + streamToConfigFromArchive.str() + ); + } +} + +OCIO_ADD_TEST(OCIOZArchive, extract_config_and_compare_to_original) +{ + /** + * This test is doing the following : + * + * 1 - Create a config object from context_test1_windows.ocioz. + * 1 - Extract the config context_test1_windows.ocioz. + * 2 - Create a config object using the extracted config in step 1. + * 4 - Compare different elements between the two configs. + * + * Testing CreateFromFile and ExtractOCIOZArchive on a successful path. + */ + + std::vector paths = { + std::string(OCIO::GetTestFilesDir()), + std::string("configs"), + std::string("context_test1"), + std::string("context_test1_windows.ocioz"), + }; + const std::string archivePath = pystring::os::path::normpath( + pystring::os::path::join(paths) + ); + + { + // 1 - Create config from OCIOZ archive. + OCIO::ConstConfigRcPtr configFromArchive; + OCIO_CHECK_NO_THROW(configFromArchive = OCIO::Config::CreateFromFile( + archivePath.c_str() + )); + OCIO_REQUIRE_ASSERT(configFromArchive); + OCIO_CHECK_NO_THROW(configFromArchive->validate()); + + // 2 - Extract the OCIOZ archive to temporary directory. + DirectoryCreationGuard dGuard("context_test1", __LINE__); + OCIO_CHECK_NO_THROW(OCIO::ExtractOCIOZArchive( + archivePath.c_str(), dGuard.m_directoryPath.c_str() + )); + + // 3 - Create config from extracted OCIOZ archive. + OCIO::ConstConfigRcPtr configFromExtractedArchive; + OCIO_CHECK_NO_THROW(configFromExtractedArchive = OCIO::Config::CreateFromFile( + pystring::os::path::join(dGuard.m_directoryPath, "config.ocio").c_str() + )); + OCIO_REQUIRE_ASSERT(configFromExtractedArchive); + OCIO_CHECK_NO_THROW(configFromArchive->validate()); + + // 4 - Compare config cacheID - configFromArchive vs configFromExtractedArchive. + OCIO::ConstContextRcPtr context; + OCIO_CHECK_EQUAL( + std::string(configFromArchive->getCacheID(context)), + std::string(configFromExtractedArchive->getCacheID(context)) + ); + + // 5 - Compare a processor cacheID - configFromArchive to configFromExtractedArchive. + OCIO::ConstProcessorRcPtr procConfigFromArchive, procConfigFromExtractedArchive; + OCIO_CHECK_NO_THROW(procConfigFromArchive = configFromArchive->getProcessor( + "plain_lut1_cs", + "shot1_lut1_cs" + )); + OCIO_REQUIRE_ASSERT(procConfigFromArchive); + + OCIO_CHECK_NO_THROW(procConfigFromExtractedArchive = configFromExtractedArchive->getProcessor( + "plain_lut1_cs", + "shot1_lut1_cs" + )); + OCIO_REQUIRE_ASSERT(procConfigFromExtractedArchive); + + OCIO_CHECK_EQUAL( + std::string(procConfigFromArchive->getCacheID()), + std::string(procConfigFromExtractedArchive->getCacheID()) + ); + + // 6 - Compare serialization - configFromArchive vs configFromExtractedArchive. + std::ostringstream streamToConfigFromArchive, streamToConfigFromExtractedArchive; + configFromArchive->serialize(streamToConfigFromArchive); + configFromExtractedArchive->serialize(streamToConfigFromExtractedArchive); + OCIO_CHECK_EQUAL( + streamToConfigFromArchive.str(), + streamToConfigFromExtractedArchive.str() + ); + } +} \ No newline at end of file diff --git a/tests/cpu/UnitTestUtils.cpp b/tests/cpu/UnitTestUtils.cpp index b2291b806f..87a7b72cb7 100644 --- a/tests/cpu/UnitTestUtils.cpp +++ b/tests/cpu/UnitTestUtils.cpp @@ -7,7 +7,15 @@ #include "Logging.h" #include "OpBuilders.h" #include "UnitTestUtils.h" - +#include "utils/StringUtils.h" + +#ifndef _WIN32 +#include +#include +#include +#else +#include +#endif namespace OCIO_NAMESPACE { @@ -63,6 +71,108 @@ ConstProcessorRcPtr GetFileTransformProcessor(const std::string & fileName) return config->getProcessor(fileTransform); } +std::string CreateTemporaryDirectory(const std::string & name) +{ + int nError = 0; + + #if defined(_WIN32) + std::string sPath = GetEnvVariable("TEMP"); + static const std::string directory = pystring::os::path::join(sPath, name); + nError = _mkdir(directory.c_str()); + #else + std::string sPath = "/tmp"; + const std::string directory = pystring::os::path::join(sPath, name); + nError = mkdir(directory.c_str(), 0777); + #endif + + if (nError != 0) + { + std::ostringstream error; + error << "Could not create a temporary directory." << " Make sure that the directory do " + << "not already exist or sufficient permissions are set"; + throw Exception(error.str().c_str()); + } + + return directory; +} + +#if defined(_WIN32) +void removeDirectory(const wchar_t* directoryPath) +{ + std::wstring search_path = std::wstring(directoryPath) + Platform::filenameToUTF("/*.*"); + std::wstring s_p = std::wstring(directoryPath) + Platform::filenameToUTF("/"); + WIN32_FIND_DATA fd; + HANDLE hFind = ::FindFirstFile(search_path.c_str(), &fd); + if (hFind != INVALID_HANDLE_VALUE) + { + do + { + if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + // Ignore "." and ".." directory. + if (wcscmp(fd.cFileName, Platform::filenameToUTF(".").c_str()) != 0 && + wcscmp(fd.cFileName, Platform::filenameToUTF("..").c_str()) != 0) + { + removeDirectory((wchar_t*)(s_p + fd.cFileName).c_str()); + } + } + else + { + DeleteFile((s_p + fd.cFileName).c_str()); + } + } while (::FindNextFile(hFind, &fd)); + + ::FindClose(hFind); + _wrmdir(directoryPath); + } +} +#else +void removeDirectory(const char * directoryPath) +{ + struct dirent *entry = NULL; + DIR *dir = NULL; + dir = opendir(directoryPath); + while((entry = readdir(dir))) + { + DIR *sub_dir = NULL; + FILE *file = NULL; + + std::string absPath; + // Ignore "." and ".." directory. + if (!StringUtils::Compare(".", entry->d_name) && + !StringUtils::Compare("..", entry->d_name)) + { + absPath = pystring::os::path::join(directoryPath, entry->d_name); + sub_dir = opendir(absPath.c_str()); + if(sub_dir) + { + closedir(sub_dir); + removeDirectory(absPath.c_str()); + } + else + { + file = fopen(absPath.c_str(), "r"); + if(file) + { + fclose(file); + remove(absPath.c_str()); + } + } + } + } + remove(directoryPath); +} +#endif + +void RemoveTemporaryDirectory(const std::string & directoryPath) +{ +#if defined(_WIN32) + removeDirectory(Platform::filenameToUTF(directoryPath).c_str()); +#else + removeDirectory(directoryPath.c_str()); +#endif +} + } // namespace OCIO_NAMESPACE diff --git a/tests/cpu/UnitTestUtils.h b/tests/cpu/UnitTestUtils.h index 9e0be02ec6..07cd7a1372 100644 --- a/tests/cpu/UnitTestUtils.h +++ b/tests/cpu/UnitTestUtils.h @@ -127,6 +127,21 @@ struct EnvironmentVariableGuard const std::string m_name; }; +/** + * @brief Create a Temporary Directory + * + * @param name Name of the directory + * @return std::string Full path to the directory + */ +std::string CreateTemporaryDirectory(const std::string & name); + +/** + * @brief Remove the directory specified in the path. + * + * @param directoryPath Path to the directory + */ +void RemoveTemporaryDirectory(const std::string & directoryPath); + } // namespace OCIO_NAMESPACE diff --git a/tests/data/files/configs/context_test1/config.ocio b/tests/data/files/configs/context_test1/config.ocio new file mode 100644 index 0000000000..0477a531d7 --- /dev/null +++ b/tests/data/files/configs/context_test1/config.ocio @@ -0,0 +1,113 @@ +# Context variable test. +# +# Directory layout for this test config is as follows. Each CLF file is a matrix that +# multiplies by the indicated factor. +# +# config_test1/ +# config.ocio +# lut1.clf x5 +# shot1/ +# lut1.clf x10 +# shot2/ +# lut1.clf x20 +# lut2.clf x2 +# shot3/ +# lut1.clf x30 +# subdir/ +# lut3.clf x3 +# shot4/ +# lut1.clf x40 +# lut4.clf x4 + +ocio_profile_version: 2 + +# Test the four places a context variable may be used are: +# 1) Search path +# 2) FileTransform src +# 3) FileTransform cccid +# 4) ColorSpaceTransform src or dst + +environment: + SHOT: shot4 + LUT_PATH: shot3/lut1.clf + CCCID: look-02 + CAMERA: arri + +search_path: + - ./$SHOT # Use a leading "./" to suggest that it _should_ be a local path. + - shot1 + - shot2 + - shot3 + - shot3/subdir + - . # Include the working directory itself. + +roles: + default: raw + scene_linear: reference + +displays: + sRGB: + - ! {name: Raw, colorspace: raw} + +looks: + + - ! + name: shot_look + process_space: reference + transform: ! {src: "looks.cdl", cccid: $CCCID} + +colorspaces: + + - ! + name: reference + isdata: false + + - ! + name: raw + isdata: true + + - ! + name: plain_lut1_cs + to_reference: ! {src: lut1.clf} + + - ! + name: shot1_lut1_cs + to_reference: ! {src: shot1/lut1.clf} + + - ! + name: shot2_lut1_cs + to_reference: ! {src: shot2/lut1.clf} + + - ! + name: shot2_lut2_cs + to_reference: ! {src: shot2/lut2.clf} + + - ! + name: lut3_cs + to_reference: ! {src: lut3.clf} + + - ! + name: lut4_cs + to_reference: ! {src: lut4.clf} + + - ! + name: SHOT_lut1_cs + to_reference: ! {src: ./$SHOT/lut1.clf} + + - ! + name: lut_path_cs + to_reference: ! {src: ./$LUT_PATH} + + + - ! + name: context_camera + to_reference: ! {src: $CAMERA, dst: reference} + + - ! + name: arri + to_reference: ! {src: lut3.clf} + + - ! + name: sony + to_reference: ! {src: lut4.clf} + diff --git a/tests/data/files/configs/context_test1/context_test1_linux.ocioz b/tests/data/files/configs/context_test1/context_test1_linux.ocioz new file mode 100644 index 0000000000000000000000000000000000000000..0e71b9b775153ad1f76fa3d870a0c6f5211750ca GIT binary patch literal 3290 zcma)83p|s38(uS~kx5(5Z*wL&PD&=iYUR)*E2nZktg;xdAvu;P=S&KG*GGvR>1K~NqZP}}~UR-hkC zf*%CpWBqm}dbxSJs}h|(i7j|DqPUbGZ9>_Dk)>bIfz63w>^y0yWTpJ_vs{dz;|CGW z7n_dg;!~C~!ydOr-<3DFdfd^h33DUIr@f6s)IY5JVCPv!X$o*%Uk#D+q|{p%CdlI= z$6OX>7ky1N{LL{W%~@a5f}-i$%?hCO%)i=N$C!~$`o)fI&8{N8XB!;{nT%Jt;d^UM zhO5#M=mzI^0>jfb-un0HiMwgiQ-l)RsnY=|N{36OH>V3xd!M)4SxVR`C8`Ri_gy%X z%3pg`d)Obw{b1k1G>o^@xYk-F_^EEPj+)=~wlT!zqga_VJeLSqp7iYjMK}DRQ+5R``i$`KiIjp4)?eSK&0^=~c)7w(_MK(%LCcLGakjim^W@8n*HT!3h zjw9Z;Hc~LLNAD!mfeBox12t_a`*OxbVF}Fi+{<*Ous&xP*m!oSYeJSFj@n8d?&h^4G(f`x4}BK1%*j8k#!i=K3IBxf*#{} zfn&tHYss}piwL=k(+Pqh-LYPYuR{xn$y(3b@=6N^QSmpZy<-sg&Uf1SaZaPx@aA1N4tK_Eb3xPP?wQ+L33FY6Lu zT_g{pucq1wKVJ=1=M!#L=4Aqv>inJa2#UXlr|>y#1k7)X5XWhQ6iB{t@)q8l1-;w5 z(aZ{ko{Zu8K<0x}t&Pds5UK;B;o41GG(SP(quh$BY-Ibpt?$r?b=*iRYQY{EYzORW z(QF%E(FF*(-dqi@*5c4Ghsyc~QpL)8~gX z&FB1&igA<@=9;b0>i|~o{nVAN{D}S6th&~+(p+X0R|l{%9$AFG^2d6pNjy1^(L+Q7 zw73Y-vQ|R4*`VA_1z#OZG6SyE-Fw^FOASW4^VzA$g%{QPxjf*^JTT=HYn8^>J90^Y zN)-%y0={1;wp$0qyfPQzeleAIlo6#JC>F+~^c{s2xNoB&OM(`b8oot;AouK|%DjNO zZNvn9?t9a(mNq3&fPtHhdsCD`~Z&`w( zsskF~+V$El?feH``V>`43y=5aHgz9naNv>w2`Y9>b=+NO+gS7oOlFh|#Bm`Dsrk*p zCF*5p89;)jb8id(%sXRXB#BB;Pp)~c?1asp03Qp``qC5d7G7(WbuF)y3&lZC96sRne8p+Se#nT5!8fsP2cCnHsR9l%))VDVyrV0eZJzlpp@#cQ;7Givh z^ci$p(#Wo^&p!Fv$(IG|$|QtI(GGTlx<08A2VN42Dj(KPN_~A_b;Q!tz&@@~H{wlT z2XXLAgmTRVe)44bzA!FiG4&BD_{qa3Xqm4;b4#Y*60M`&ps6yipo(}nHX#kN&8z&_ z1CiGw>KeDa4~$tpA}<#hZ9)h0?^5h#5&PLTP`_*&SSdcE&3#mJj8iBqr^!HNnN&!; z=LT5NXq|s{S*}C_Vz`~#NBdqY%aKRKYwcyvp1tj>K$AfcQwloFQe!_ zchWwXnKXm}qd$~KXL2FSs3pz8%**zf8#-0Kg~%`dZ5`BuzApEIAM54y5VT8W@1qC2u-UF4GG*;L^=)cZb1_ubl%X)f(NIXtbb#^&HOYR|5s>5p@ zbY%0;h=D>MT;#a+(BOzU2u228=62J&gB~;<)l#ZtT4lI#ygyw{LEqf?mhQ;`LbJgA zVc{W%x>nm(SRvD4Zv1gxJ)vAtUxbrVr)5?ZPfnA!xbsMZ;A|l?^e4s7W`uMt$-$Ur zqc(+~<;>f;m?xWc@w)Q5Iq4YW*`XG%?TX4I=*A>c3JHAUywmw3g9lvSgz9gY#Fa?G zL-5{#mvh8}0)3oex4kh=)9}uyu`H4GH4+x-_5#&8u{_^+&mId4SI?Js(VU`_6i2s9 zTyNk$9UmfOxLs(;R?ra(7t$pL;Ptaaj&73>VlCY0Wq#4?H)>G56cB=HuR6lb^FEr zDrDuN5B7B5LWE4`^XX1#B&SA{Rv672G;S)nYyvrH+nPv&lXv0^5tRK5?hzYFLo+F` zAE%A`Hp4;FXZP>HGBxucql|g^K(SEfxjyrmvHCCL7kh`tmQb^M-G+iEBIEd#6CNgo zL6l@&=mYap@>#Qz6jRSm)U49oqkt$(Bw{81f2C{uQt2K>YzEo#-v${&_+gN8F#+eU zo4F&>NYO8i^lY8vlAT6QWYdVXEQ7qk>Q25zo3+gr3qGA-ISIT>dxuz9!<9-r>|-Uf zFVcbdacSdJvU|VXJP4zQN9j+LR4eIx*e&ee-; z1jU)G0D(WTyDm`r-&ZJTDC{(A!dN{&ti`f>+iL7VmXQ9zF1z$vEW3fM#wtR-V^_Rz zEtb7Yuf{I1W0wtQEtb6r0)2di?|aVQboI5bU~f6Aw=d-S`HG))vKGr;X+Tq2+5S8? xTZ>u?V;3|~hAXhwtlQsh#x9uEfo8Dw2J*8aeVhp=D|iqH%KBIUnTG;*{{bUs)>Qxi literal 0 HcmV?d00001 diff --git a/tests/data/files/configs/context_test1/context_test1_windows.ocioz b/tests/data/files/configs/context_test1/context_test1_windows.ocioz new file mode 100644 index 0000000000000000000000000000000000000000..091b68e52dff5ca6a84804261552391182cb2e95 GIT binary patch literal 3938 zcmbW3c{r4d7sp==#t>pgh#JPeZ|TZdvXrt!MOm|NS0;u`ma-;Vh$yl&sjKXg$$qnB zPeqn2k@Z%iv1Utt^QwEjUGDR{-FN<&d7gRZJJ08Q&-tEXtWO6+0th%9;ItAq1-?Jr zEC9ev{p^7Abar!*!8y3$I?YUQ9dhjF7xLvgcJ)2>Wsa9pdg${s_M)csJEvr+7hYz+ z=J9)ET1R{yR8c)$##HDF-%p696j8neRjH2U7$+vQaUIqt-Ds2iRZIkasOWfGa)eTd z;1$u47+j`9bSYjUIld{9GrG8V?|?eZ5V_?_(5@Fo;-B$5FREEb>=*H&v7E?Eciw~5 zvhfxj&l$Dbm2uXqOZCc%8-EN-GRn=Yb|7maJKL(X7qv8W11`+iHJ&^?BoXCPJcK+o zr>KC=-&c~(zs)9YadP)|@2J=!i)DhJ471I9&B75$xyslB3b`zDFaxgkLl_~g_fA&D z(;tR+1ic4%0d{=ngIa7o;aj3M0;E`WI`; z=p%liXnyu!298j4QgP`~R&?e9$Bm}#vS4?j&ysh_#KTh${-K@`d6O%UIF~#e7UtND zLv>5!6{MoAak43HAQlENQV%c(m5~susF(EZ@#)0LJ=^@Ol~nUTsKh4|iR>&DH>lI%oEb}z71c1(xvzS9 z+`WS}_tTC^+oo>IZnh$_?ea`f#N^>)d3OgB%;ecbUkeI8 zJ7T^-)f#v#+5sgG|Pg~=!nTq&%I)jcm!u0J^@dA_mcgklOK;by$sga zK4)CQkGx>!<$vqJ&H#UJ2e#k5bnQPQ`y;0_xVP2wnWR{=K6`K#{*@`>R7j{?frz8R z{0LrbYA@fNR>r`X3mjT|Io2%M?NBk9ilw~@{jcmk)p&-s5NLI|GcN~QvE7|Mtp@Y^ zNK|~LrjxXb1}wCn*rX#X4FG`U+$i*T_(q`zs8xl7h4%H4mvQiLhSnMA!Krj^QD@U@ z&k;3p?BSGi(N2_JS8;;tu`k3-#I8A-!#jPeg@mJy4!*sQy70MxS#?%E{$^Ndwf2f; z8&}CKeb{Ns?(3CE!alPiRN~=LMv4W$mZ1QxFTI7!9<7fCzvpJNr77gWrbbsp{CR`P z7l(}&rkg*^Tp9W_y{5Qy(0MXwHvAfkWbC835SX}-BhT0=YY6mMF0SdUj+(138o7APnzNv~zV?dV^OFDwcx2)5)_e+1hAhP<0dB7m{ zj&LoDp0gvO;t;}npX^k^Ws|iw&SU5kC=Kh&q9C}}c#Uq-re^2-o8_qQW3Fh0_qBI) z^9K19|L%OSF7oh}{MRZ$=ffA&BgXhS8f#)n!v|BN>i{<6Vj8LS0^njg11@)5=+FiAjHg&da+5cJ)Tvk^0}W=$u1d5kczW6PXtVGWgIuUL>&4me@_3qg#1_q*^Y z##P{`gr}giTXyyDe)J;_3kC71Nv)0cf8?#_9FNSne5i7>vSTmHkp^#4jo^-%p`7-& zCr0UvznV(`qINmnxG|&Rw?a3J7D&|x9y&~aZ!SK4Gc;l`;g z(f>3ZOEsO>m64k98Xt}eF%W{Yw%M+a`#I{kPmfe51}?mpk!8e(MSc!Rkv=1T6W&Y8 z!vu&=`Q$iDN+mS1y)E7so~#A?fOy>I9^RF7-txq5tdkYfP}73poD%S6+q2Y9BMF^j zJkD!~Mqxa>g6$Z~PFvx}awQcy)by(A2j;>Xr6WHyBhJyfdKN7`!t! zq{pZ?$yU4fpRNw5GPup5+YU_j4dpj5;6Lnks6Mzep*!VA{SNwX`aicpq59xvf+qKm z`p+34`u}=QLG{7>hX&-2`VEZV^nY#}K=r|xh2{?!vm5n)3jb68KZGvyZ@@5v&iaoS s)lA?1=I5dfst+axbfA9JuRuWb|8;&C>(f*10{{f|l|XIrC+m~=KPH;OX#fBK literal 0 HcmV?d00001 diff --git a/tests/data/files/configs/context_test1/looks.cdl b/tests/data/files/configs/context_test1/looks.cdl new file mode 100644 index 0000000000..e14d42bb91 --- /dev/null +++ b/tests/data/files/configs/context_test1/looks.cdl @@ -0,0 +1,42 @@ + + + + + + 1.1 1.0 0.9 + -.03 -2e-2 0 + 1.25 1 1e0 + + + 1.700000 + + + + + + + + 0.9000 0.700 0.6000 + 0.100 0.100 0.100 + 0.9 0.9 0.9 + + + 0.7 + + + + + + + + 1.2000 1.1000 1.0000 + 0.000 0.0000 0.0000 + 0.9 1.0 1.2 + + + 1.000000 + + + + + diff --git a/tests/data/files/configs/context_test1/lut1.clf b/tests/data/files/configs/context_test1/lut1.clf new file mode 100644 index 0000000000..d948a1e5dd --- /dev/null +++ b/tests/data/files/configs/context_test1/lut1.clf @@ -0,0 +1,10 @@ + + + + +5 0 0 +0 5 0 +0 0 5 + + + \ No newline at end of file diff --git a/tests/data/files/configs/context_test1/shot1/lut1.clf b/tests/data/files/configs/context_test1/shot1/lut1.clf new file mode 100644 index 0000000000..4750b14b38 --- /dev/null +++ b/tests/data/files/configs/context_test1/shot1/lut1.clf @@ -0,0 +1,10 @@ + + + + +10 0 0 +0 10 0 +0 0 10 + + + \ No newline at end of file diff --git a/tests/data/files/configs/context_test1/shot2/lut1.clf b/tests/data/files/configs/context_test1/shot2/lut1.clf new file mode 100644 index 0000000000..14826c0f96 --- /dev/null +++ b/tests/data/files/configs/context_test1/shot2/lut1.clf @@ -0,0 +1,10 @@ + + + + +20 0 0 +0 20 0 +0 0 20 + + + \ No newline at end of file diff --git a/tests/data/files/configs/context_test1/shot2/lut2.clf b/tests/data/files/configs/context_test1/shot2/lut2.clf new file mode 100644 index 0000000000..e9195e7738 --- /dev/null +++ b/tests/data/files/configs/context_test1/shot2/lut2.clf @@ -0,0 +1,10 @@ + + + + +2 0 0 +0 2 0 +0 0 2 + + + \ No newline at end of file diff --git a/tests/data/files/configs/context_test1/shot3/lut1.clf b/tests/data/files/configs/context_test1/shot3/lut1.clf new file mode 100644 index 0000000000..7cb3ad2785 --- /dev/null +++ b/tests/data/files/configs/context_test1/shot3/lut1.clf @@ -0,0 +1,10 @@ + + + + +30 0 0 +0 30 0 +0 0 30 + + + \ No newline at end of file diff --git a/tests/data/files/configs/context_test1/shot3/subdir/lut3.clf b/tests/data/files/configs/context_test1/shot3/subdir/lut3.clf new file mode 100644 index 0000000000..3996477926 --- /dev/null +++ b/tests/data/files/configs/context_test1/shot3/subdir/lut3.clf @@ -0,0 +1,10 @@ + + + + +3 0 0 +0 3 0 +0 0 3 + + + \ No newline at end of file diff --git a/tests/data/files/configs/context_test1/shot4/lut1.clf b/tests/data/files/configs/context_test1/shot4/lut1.clf new file mode 100644 index 0000000000..a89315abc2 --- /dev/null +++ b/tests/data/files/configs/context_test1/shot4/lut1.clf @@ -0,0 +1,10 @@ + + + + +40 0 0 +0 40 0 +0 0 40 + + + diff --git a/tests/data/files/configs/context_test1/shot4/lut4.clf b/tests/data/files/configs/context_test1/shot4/lut4.clf new file mode 100644 index 0000000000..a5ebce68f2 --- /dev/null +++ b/tests/data/files/configs/context_test1/shot4/lut4.clf @@ -0,0 +1,10 @@ + + + + +4 0 0 +0 4 0 +0 0 4 + + + \ No newline at end of file diff --git a/tests/data/files/configs/ocioz_archive_configs/config_missing_luts.ocioz b/tests/data/files/configs/ocioz_archive_configs/config_missing_luts.ocioz new file mode 100644 index 0000000000000000000000000000000000000000..68088913aa40bc182ad049e2340f4e9f9f4e1e57 GIT binary patch literal 924 zcmWIWW@Zs#U|`^22#YrlRc5HlU|?ooh~Q*k;0B5&=jWwmrt9S=XXeihjm^JhAX0mN z{f_2+Az92b-XvZ$S-JRa)XK}troA&Y>SGS;?af)#PbXqOqF@3`6chgVnaelvksNmU_$fhOUu~r&e z7|jLE5*jl@I}9I8j56_D`P__c$HE+@=Jsd}mQ@wnJ}Zrr8UtQUD2wt^PYWvGUaT6o zX!0r6#eL5M0`w&gMSV<66+&Hre5=XPhDhCd-L8p zm;IcOm#mO-IUxD>?hSuUuT5JVGvmRwvNzxNCO-``R^2^gQm@9Ol*Ls3j;%GL`@8)q)*`K`90pV?rkw?v_a?zOp6^Vl}tv$QG9+R$qJO6_6SoWr`S zcz8p_)DyU_ZAf@gZ(?xFMb77>CEuJY&obLyy`RuvJSAG(HD)i@;;U1%r#rEHDEXc} zBaHRn<-?y1e%G4r@eV(_F1vZ#!GwbPxlLDovASQ4kNLZ&a*vlGxALMBZ9k0V>lOC3 z>?nSAGx2Zs;muDYgidDaU6F_?T+Zs3eEitsRTWnn3nnZq)BMcib}-sl%Q-UXe3WnG zUzbpovunYMr{N%bB};rbam#d1}q*Pk72pS1qI<(2I*VudMNo|w6I%@N-+B|^vf zdBv6M#-~YRx|H0+!_D$#aAB=ZhZqw>+wV&bH z?3CKcOmCw_ey&n7StY<<@k{IjqsXRHd%SlX{(1Q3rbyQdM+AEE?tT@iymywft)Y2> zlXk{Yo#|`4v&w=jxX&K7m~~=S#;l2}l5b5CHCx`dY=`HPr_~Lihh_`(?D)ga^ja%K zEX^=^n^$mpDZ_4~+Z_kLuI)PepCQ1Tk;$F`S5{F0W)=`o0Fx*ghYKpm$RNS6{aAQ( h>iHXtFq-MaH7^i9z?+o~q>~W{V}Z0LFb^^?007cip8fy; literal 0 HcmV?d00001 diff --git a/tests/data/files/configs/ocioz_archive_configs/empty.ocioz b/tests/data/files/configs/ocioz_archive_configs/empty.ocioz new file mode 100644 index 0000000000000000000000000000000000000000..15cb0ecb3e219d1701294bfdf0fe3f5cb5d208e7 GIT binary patch literal 22 NcmWIWW@Tf*000g10H*)| literal 0 HcmV?d00001 diff --git a/tests/data/files/configs/ocioz_archive_configs/missing_config.ocioz b/tests/data/files/configs/ocioz_archive_configs/missing_config.ocioz new file mode 100644 index 0000000000000000000000000000000000000000..386b4b27c42759bb70e0079de33ed1780b45a9fa GIT binary patch literal 3245 zcmbW4dos?(xFMI6R zPZ_~36O>D-9u_PY?K!6ffx_*}g-Us6FKh5ED#l|@jkO)!rm8_fNTyQKDbOotJkRVK zwLAQjZnANfSSpW-a0(2&aQ$0aSdbU;mq3E&8&r4fRG##PS~>e{cd?r5i9+v%PaTYm z(kW6Q>CFvLR42E|U2cM(NRE&&+a|H%Dt-u^oR2H()a!nBXuRrZ)MF}?Af2B;cSBy8 zI${h98D-#Vj4d+1<0D$Zg+f%vNMd{t2vAPq--RBI;THNOr>a6gp@Wa>YI*s4gX;|G z2rz8pQD?`bCx|K?@tBFw_`?&OK5A4Shj)y8#CNlNd%g{>R8o%H+w4FM;S(g zlY>htVX}1g>3-Y!sm77%v%TX}E4YQ--eci2F-f8tDR)vMVQNaGf#Ib&mAnP{3~S$R z+=AMT0|2v?T%vpyhYBF$GC#kl_;+*la()rxFjIU2j_bPdBBz@w2rBU+w=EZ|(h==m zLT7~dm`j~DKqG_ICHPO6w!w-l($s~ozcc=2C!SroxhH2l-j99zSbKBYR^|B6)0b_? z=uApnow>2Ar*fL-gnvFOpdOBKyi&Xidz`t+z-GDlwf}kHNmRMP_p*^qSyfKRzAAQq zj7LDQ6F z@N;tfD*5wZ9kuo3^4VO^fo}NZ`6{{Ix^537JMtt?Y5+X9m{>+U4zcvnmb-t3V2qCA zn4T`^7pLSXL0q5}d{g|^!zwp4tKsIe?p{VXhAiTysT{>@3?|(c$hDjEOmxf~+&%HL z7=s}mc^`VKM0Tekj-9m_O+L#peK6J_Og568(SHD0Oy0`El+xa&Mb%3kWR+N879)=FI^cD0lFQfFnM5ow=aFca~x04>Euprp5&lzw^R@@ z-_*~rC3w=2*IP_A&t2UERgXya-H4=3HiYDt7sxfC$2;Lc26Z)f!c(WR47PGzYjM`C zqAjWH-kcVN-%*Ohi+RCLSXJ`k5%+L*>KFlve^C*aD~Ks)l(vSmFSzG!?AClAp|bqP zep(OyvhpKQOK%e5!4HU_?b?&+3HB?^k`5SWw26DdW&k{JxQdXuuGu9&vmE_#%yDj% zU{8{75WtW7+<7_ywTCBv5({uXW`4)1A$f`VsuX7b?wt5)2$Fb@Z_8={0+QPlPvSdL zCw9F2GpJ}A^@4aqxttUw&ckigDCoSLU7a7Z>Q4Qv!rSN7`y8xI-IJP)qMwF!1&oeF zYt$x)QfDiEh!n*9mvI*te*eyWJoYVZamD)mx&5(E@eJ%^gsKzDG9?qX^=QSx+tN=a zbabx;+D$p!rLOioYsQeU-uV#F672)^ygqN|vC52Ao;mkeEJ{cG!Muh6&6)T;RaJab zUrZc)8M0UXsG_Ft&B})%0s~(2MQJ&iP4%9C6t3o+jMlJjltH5bsZ-RhE{IvBC^g+% z(DKT8ke`?aOuK5=T-S|v9g@t~Q0icXSdt*7gi+KQUaC`$#{$!Cz4)x;r^1t_=JL1{ zowVBDG(0WyXP^f|^+$R_pP}j<^DY&ZQK5kM87~*~FLK}jKyJ|CLFnLdUq#nnXFB!g zgFMkcnU3d}F6#s5WW349k4l16`i%`9tK%NZ8TYAy%8V29e`;yNDbcZSBC|D*>7Ezr zWENs+YLmwcyfwC@*CSuu+a8l;1{*=#ZT5?Ko_W&Mc@zGyn{aQ#yv?jWEx!)rD0)t87ItPWq#qkmi4ty=0|OI%@}RTY$+vAJN(Gvpu&avn=J$Z z^bgyO52}ga9#7zH4;)V)9?ZwjuR5iOR$OV|TmUas;9THJ`*O|;UMuuV#{*aySjym~ z1q3u#*&Pb`7b_a953D}$YFMl9%sGmH6X5F>Ay^++FyH`Qt8Whj>Hp((1J(y(9Gr7& z^{oUx=zm?PgY|*51jh)FmfZSV2!7K47ugAZ4UkFTkzI>HfPZ++*CiBK9~gY_T&&eM c6$0u1;}{@X32^X%KOw>pH%_A%2?O8$0#9p {name: Raw, colorspace: raw} + +active_displays: [] +active_views: [] + +colorspaces: + + - ! + name: raw + isdata: true + + - ! + name: c1 + to_reference: ! {src: my_unique_lut1.clf} + + - ! + name: c2 + to_reference: ! {src: my_unique_lut2.clf} +""" + + C1_LUT = """ + + + +1 0 0 +0 1 0 +0 0 1 + + + +""" + + C2_LUT = """ + + + +2 0 0 +0 2 0 +0 0 2 + + + +""" + + class CIOPTest(OCIO.PyConfigIOProxy): + def getConfigData(self): + # This implementation is only to demonstrate the functionality. + + # Simulate that the config is coming from some kind of in-memory location. + return SIMPLE_CONFIG + + def getLutData(self, buffer, filepath): + # This implementation is only to demonstrate the functionality. + + # Simulate that the LUT are coming from some kind of in-memory location. + + if filepath == os.path.join('my_unique_luts', 'my_unique_lut1.clf'): + encoded_c1_lut = C1_LUT.encode('utf-8') + lst = list(encoded_c1_lut) + for c in lst: + buffer.append(c) + elif filepath == os.path.join('my_unique_luts', 'my_unique_lut2.clf'): + encoded_c2_lut = C2_LUT.encode('utf-8') + lst = list(encoded_c2_lut) + for c in lst: + buffer.append(c) + + def getFastLutFileHash(self, filepath): + # This implementation is only to demonstrate the functionality. + + def lutExists(filepath): + if filepath == os.path.join('my_unique_luts', 'my_unique_lut1.clf'): + return True + elif filepath == os.path.join('my_unique_luts', 'my_unique_lut2.clf'): + return True + return False + + # Check if the file exists. + if lutExists(filepath): + # This is a bad hash, but using the filename as hash for simplicity and + # demonstration purpose. + hash = filepath + return hash + + ciop = CIOPTest() + config = OCIO.Config.CreateFromConfigIOProxy(ciop) + config.validate() + + # Simple check on the number of color spaces in the test config. + self.assertEqual(len(config.getColorSpaceNames()), 3) + + # Simple test to exercice ConfigIOProxy. + processor = config.getProcessor("c1", "c2") + processor.getDefaultCPUProcessor() + class ConfigVirtualWithActiveDisplayTest(unittest.TestCase): def setUp(self): self.cfg_active_display = OCIO.Config.CreateFromStream( diff --git a/tests/python/OCIOZArchiveTest.py b/tests/python/OCIOZArchiveTest.py new file mode 100644 index 0000000000..8ee8a4edad --- /dev/null +++ b/tests/python/OCIOZArchiveTest.py @@ -0,0 +1,399 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright Contributors to the OpenColorIO Project. + +import unittest +import os +import platform +import tempfile + +import PyOpenColorIO as OCIO +from UnitTestUtils import (SIMPLE_CONFIG, TEST_DATAFILES_DIR) + +class ArchiveIsConfigArchivableTest(unittest.TestCase): + + def setUp(self): + """ + Initialize the CONFIG + """ + self.CONFIG = OCIO.Config().CreateFromStream(SIMPLE_CONFIG) + # Since a working directory is needed to archive a config, settting a fake working directory + # in order to test the search paths and FileTransform source logic. + if platform.system() == 'Windows': + self.CONFIG.setWorkingDir('C:\\fake_working_dir') + else: + self.CONFIG.setWorkingDir('/fake_working_dir') + self.CONFIG.validate() + + def tearDown(self): + self.CONFIG = None + + def test_is_archivable(self): + """ + Test different scenario for isArchivable + """ + ############################### + # Testing search paths + ############################### + + # No search path. + self.assertEqual(True, self.CONFIG.isArchivable()) + + # Valid search path. + self.CONFIG.setSearchPath('luts') + self.assertEqual(True, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('luts/myluts1') + self.assertEqual(True, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('luts\myluts1') + self.assertEqual(True, self.CONFIG.isArchivable()) + + # Valid Search path starting with './' or '.\'. + self.CONFIG.setSearchPath('./myLuts') + self.assertEqual(True, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('.\myLuts') + self.assertEqual(True, self.CONFIG.isArchivable()) + + # Valid search path starting with './' or '.\' and a context variable. + self.CONFIG.setSearchPath('./$SHOT/myluts') + self.assertEqual(True, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('.\$SHOT\myluts') + self.assertEqual(True, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('luts/$SHOT') + self.assertEqual(True, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('luts/$SHOT/luts1') + self.assertEqual(True, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('luts\$SHOT:luts\$SHOT\luts2') + self.assertEqual(True, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('luts\$SHOT\luts2') + self.assertEqual(True, self.CONFIG.isArchivable()) + + # + # Illegal scenarios + # + + # Illegal search path starting with '..'. + self.CONFIG.setSearchPath('luts:../luts') + self.assertEqual(False, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('luts:..\myLuts') + self.assertEqual(False, self.CONFIG.isArchivable()) + + # Illegal search path starting with a context variable. + self.CONFIG.setSearchPath('luts:$SHOT') + self.assertEqual(False, self.CONFIG.isArchivable()) + + # Illegal search path with absolute path. + self.CONFIG.setSearchPath('luts:/luts') + self.assertEqual(False, self.CONFIG.isArchivable()) + + self.CONFIG.setSearchPath('luts:/$SHOT') + self.assertEqual(False, self.CONFIG.isArchivable()) + + if platform.system() == 'Windows': + self.CONFIG.clearSearchPaths() + self.CONFIG.addSearchPath('C:\luts') + self.assertEqual(False, self.CONFIG.isArchivable()) + + self.CONFIG.clearSearchPaths() + self.CONFIG.addSearchPath('C:\\') + self.assertEqual(False, self.CONFIG.isArchivable()) + + self.CONFIG.clearSearchPaths() + self.CONFIG.addSearchPath('C:\$SHOT') + self.assertEqual(False, self.CONFIG.isArchivable()) + + + ############################### + # Testing FileTransfrom paths + ############################### + + # Clear search paths so it doesn't affect the tests below. + self.CONFIG.clearSearchPaths() + + # Function to facilitate adding a new FileTransform to a config. + def addFTAndTestIsArchivable(cfg, path, isArchivable): + fullPath = os.path.join(path, "fake_lut.cls") + ft = OCIO.FileTransform() + ft.setSrc(fullPath) + + cs = OCIO.ColorSpace() + cs.setName("csTest") + cs.setTransform(ft, OCIO.COLORSPACE_DIR_TO_REFERENCE) + cfg.addColorSpace(cs) + self.assertEqual(isArchivable, cfg.isArchivable()) + + cfg.removeColorSpace("csTest") + + # Valid FileTransfrom path. + addFTAndTestIsArchivable(self.CONFIG, "luts", True) + addFTAndTestIsArchivable(self.CONFIG, 'luts/myluts1', True) + addFTAndTestIsArchivable(self.CONFIG, 'luts\myluts1', True) + + # Valid Search path starting with './' or '.\'. + addFTAndTestIsArchivable(self.CONFIG, './myLuts', True) + addFTAndTestIsArchivable(self.CONFIG, '.\myLuts', True) + + # Valid search path starting with './' or '.\' and a context variable. + addFTAndTestIsArchivable(self.CONFIG, './$SHOT/myluts', True) + addFTAndTestIsArchivable(self.CONFIG, '.\$SHOT\myluts', True) + addFTAndTestIsArchivable(self.CONFIG, 'luts/$SHOT', True) + addFTAndTestIsArchivable(self.CONFIG, 'luts/$SHOT/luts1', True) + addFTAndTestIsArchivable(self.CONFIG, 'luts\$SHOT:luts\$SHOT\luts2', True) + addFTAndTestIsArchivable(self.CONFIG, 'luts\$SHOT\luts2', True) + + # + # Illegal scenarios + # + + # Illegal search path starting with '..'. + addFTAndTestIsArchivable(self.CONFIG, '../luts', False) + addFTAndTestIsArchivable(self.CONFIG, '..\myLuts', False) + + # Illegal search path starting with a context variable. + addFTAndTestIsArchivable(self.CONFIG, '$SHOT', False) + + # Illegal search path with absolute path. + addFTAndTestIsArchivable(self.CONFIG, '/luts', False) + addFTAndTestIsArchivable(self.CONFIG, '/$SHOT', False) + + if platform.system() == 'Windows': + addFTAndTestIsArchivable(self.CONFIG, 'C:\luts', False) + addFTAndTestIsArchivable(self.CONFIG, 'C:\\', False) + addFTAndTestIsArchivable(self.CONFIG, 'C:\$SHOT', False) + +class ArchiveContextTest(unittest.TestCase): + + def setUp(self): + """ + Initialize the CONFIG file and CONTEXT for the tests. + """ + ocioz_file = os.path.normpath( + os.path.join( + TEST_DATAFILES_DIR, 'configs', 'context_test1', 'context_test1_windows.ocioz' + ) + ) + self.CONFIG = OCIO.Config().CreateFromFile(ocioz_file) + + # OCIO will pick up context vars from the environment that runs the test, + # so set these explicitly, even though the config has default values. + ctx = self.CONFIG.getCurrentContext() + ctx['SHOT'] = 'none' + ctx['LUT_PATH'] = 'none' + ctx['CAMERA'] = 'none' + ctx['CCCID'] = 'none' + self.CONTEXT = ctx + + def tearDown(self): + self.CONFIG = None + self.CONTEXT = None + + def test_search_paths(self): + """ + Test search path processing involving context variables. + """ + # This is independent of the context. + processor = self.CONFIG.getProcessor(self.CONTEXT, 'shot1_lut1_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 10.) + + # This is independent of the context. + processor = self.CONFIG.getProcessor(self.CONTEXT, 'shot2_lut1_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 20.) + + # This is independent of the context. + processor = self.CONFIG.getProcessor(self.CONTEXT, 'shot2_lut2_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 2.) + + # This is independent of the context but the file is in a second level sub-directory. + processor = self.CONFIG.getProcessor(self.CONTEXT, 'lut3_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 3.) + + # This uses a context for the FileTransform src but is independent of the search_path. + self.CONTEXT['LUT_PATH'] = 'shot3/lut1.clf' + processor = self.CONFIG.getProcessor(self.CONTEXT, 'lut_path_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 30.) + + # The FileTransform src is ambiguous and the context configures the search_path. + self.CONTEXT['SHOT'] = '.' # (use the working directory) + processor = self.CONFIG.getProcessor(self.CONTEXT, 'plain_lut1_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 5.) + + # The FileTransform src is ambiguous and the context configures the search_path. + self.CONTEXT['SHOT'] = 'shot2' + processor = self.CONFIG.getProcessor(self.CONTEXT, 'plain_lut1_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 20.) + + # The FileTransform src is ambiguous and the context configures the search_path. + self.CONTEXT['SHOT'] = 'no_shot' # (path doesn't exist) + processor = self.CONFIG.getProcessor(self.CONTEXT, 'plain_lut1_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 10.) + + # This should be in the archive but is not on the search path at all without the context. + self.CONTEXT['SHOT'] = 'no_shot' # (path doesn't exist) + with self.assertRaises(OCIO.ExceptionMissingFile): + processor = self.CONFIG.getProcessor(self.CONTEXT, 'lut4_cs', 'reference') + + # This should be in the archive but is not on the search path at all without the context. + self.CONTEXT['SHOT'] = 'shot4' + processor = self.CONFIG.getProcessor(self.CONTEXT, 'lut4_cs', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 4.) + + def test_colorspace(self): + """ + Test a color space where a ColorSpaceTransform uses a context variable as the src. + """ + self.CONTEXT['CAMERA'] = 'sony' + self.CONTEXT['SHOT'] = 'shot4' + processor = self.CONFIG.getProcessor(self.CONTEXT, 'context_camera', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 4.) + + self.CONTEXT['CAMERA'] = 'arri' + processor = self.CONFIG.getProcessor(self.CONTEXT, 'context_camera', 'reference') + mtx = processor.createGroupTransform()[0] + self.assertEqual(mtx.getMatrix()[0], 3.) + + def test_cccid(self): + """ + Test a Look that uses a context variable for the FileTransform cccid. + """ + look_transform = OCIO.LookTransform('reference', 'reference', 'shot_look') + + self.CONTEXT['CCCID'] = 'look-02' + processor = self.CONFIG.getProcessor(self.CONTEXT, + look_transform, + OCIO.TRANSFORM_DIR_FORWARD) + cdl = processor.createGroupTransform()[0] + self.assertEqual(cdl.getSlope()[0], 0.9) + + # FIXME: There is a bug with the Processor cache, this clears it. + self.CONFIG.setStrictParsingEnabled(False) + + self.CONTEXT['CCCID'] = 'look-03' + processor = self.CONFIG.getProcessor(self.CONTEXT, + look_transform, + OCIO.TRANSFORM_DIR_FORWARD) + cdl = processor.createGroupTransform()[0] + self.assertEqual(cdl.getSlope()[0], 1.2) + + # FIXME: There is a bug with the Processor cache, this clears it. + self.CONFIG.setStrictParsingEnabled(False) + + self.CONTEXT['CCCID'] = 'look-01' + processor = self.CONFIG.getProcessor(self.CONTEXT, + look_transform, + OCIO.TRANSFORM_DIR_FORWARD) + cdl = processor.createGroupTransform()[0] + self.assertEqual(cdl.getSlope()[0], 1.1) + +class ArchiveAndExtractComparison(unittest.TestCase): + def setUp(self): + """ + Initialize the CONFIGs + """ + original_config = os.path.normpath( + os.path.join( + TEST_DATAFILES_DIR, 'configs', 'context_test1', 'config.ocio' + ) + ) + self.ORIGINAL_CONFIG = OCIO.Config().CreateFromFile(original_config) + self.ORIGINAL_CONFIG.validate() + + self.ORIGNAL_ARCHIVED_CONFIG_PATH = os.path.normpath( + os.path.join( + TEST_DATAFILES_DIR, 'configs', 'context_test1', 'context_test1_windows.ocioz' + ) + ) + self.ORIGNAL_ARCHIVED_CONFIG = OCIO.Config().CreateFromFile( + self.ORIGNAL_ARCHIVED_CONFIG_PATH + ) + self.ORIGINAL_CONFIG.validate() + + def tearDown(self): + self.CONFIG = None + + def test_archive_config_and_compare_to_original(self): + """ + Archive config X and create a new config object Y with it, and then compare X and Y. + """ + # Testing CreateFromFile and archive method on a successful path. + + # 1 - Archive the original config + temp_ocioz_file = tempfile.NamedTemporaryFile(delete = False) + # Close temporary file to make sure that archive can use it. + temp_ocioz_file.close() + try: + self.ORIGINAL_CONFIG.archive(str(temp_ocioz_file.name)) + + # 2 - Create a config from the archived config in step 1. + new_ocioz_config = OCIO.Config().CreateFromFile(str(temp_ocioz_file.name)) + new_ocioz_config.validate() + + # 3 - Compare config CacheIDs + ctx = self.ORIGINAL_CONFIG.getCurrentContext() + self.assertEqual(self.ORIGINAL_CONFIG.getCacheID(ctx), new_ocioz_config.getCacheID(ctx)) + + # 4 - Compare a processor cacheID + original_proc = self.ORIGINAL_CONFIG.getProcessor("plain_lut1_cs", "shot1_lut1_cs") + new_proc = new_ocioz_config.getProcessor("plain_lut1_cs", "shot1_lut1_cs") + self.assertEqual(original_proc.getCacheID(), new_proc.getCacheID()) + + # 5 - Compare serialization + self.assertEqual(self.ORIGINAL_CONFIG.serialize(), new_ocioz_config.serialize()) + finally: + # Delete the temporary file. + os.unlink(temp_ocioz_file.name) + + def test_extract_config_and_compare_to_original(self): + """ + Extract an OCIOZ archive that came from config X and create a new Config object Y with it, + and then compare X and Y. + """ + # Testing CreateFromFile and ExtractOCIOZArchive on a successful path. + + try: + temp_ocioz_directory = tempfile.TemporaryDirectory() + + # 1 - Extract the OCIOZ archive to temporary directory. + OCIO.ExtractOCIOZArchive(self.ORIGNAL_ARCHIVED_CONFIG_PATH, temp_ocioz_directory.name) + + # 2 - Create config from extracted OCIOZ archive. + new_config = OCIO.Config().CreateFromFile( + os.path.join(temp_ocioz_directory.name, "config.ocio") + ) + new_config.validate() + + # 3 - Compare config cacheID. + ctx = self.ORIGNAL_ARCHIVED_CONFIG.getCurrentContext() + self.assertEqual( + self.ORIGNAL_ARCHIVED_CONFIG.getCacheID(ctx), + new_config.getCacheID(ctx) + ) + + # 4 - Compare a processor cacheID + original_proc = self.ORIGNAL_ARCHIVED_CONFIG.getProcessor( + "plain_lut1_cs", + "shot1_lut1_cs" + ) + new_proc = new_config.getProcessor("plain_lut1_cs", "shot1_lut1_cs") + self.assertEqual(original_proc.getCacheID(), new_proc.getCacheID()) + + # 5 - Compare serialization + self.assertEqual(self.ORIGNAL_ARCHIVED_CONFIG.serialize(), new_config.serialize()) + finally: + temp_ocioz_directory.cleanup() \ No newline at end of file diff --git a/tests/python/OpenColorIOTestSuite.py b/tests/python/OpenColorIOTestSuite.py index 80b326e042..a9aaa5c930 100755 --- a/tests/python/OpenColorIOTestSuite.py +++ b/tests/python/OpenColorIOTestSuite.py @@ -84,6 +84,7 @@ import MatrixTransformTest import MixingHelpersTest import NamedTransformTest +import OCIOZArchiveTest import OpenColorIOTest import ProcessorTest import RangeTransformTest @@ -140,6 +141,7 @@ def suite(): suite.addTest(loader.loadTestsFromModule(MatrixTransformTest)) suite.addTest(loader.loadTestsFromModule(MixingHelpersTest)) suite.addTest(loader.loadTestsFromModule(NamedTransformTest)) + suite.addTest(loader.loadTestsFromModule(OCIOZArchiveTest)) suite.addTest(loader.loadTestsFromModule(OpenColorIOTest)) suite.addTest(loader.loadTestsFromModule(ProcessorTest)) suite.addTest(loader.loadTestsFromModule(RangeTransformTest)) From 198854c871b914fc4d1c240ce009a27b6952d0c7 Mon Sep 17 00:00:00 2001 From: Cedrik Fuoco Date: Tue, 4 Oct 2022 15:14:17 -0400 Subject: [PATCH 2/8] Add OCIOZ extension if not present. Signed-off-by: Cedrik Fuoco --- src/apps/ocioarchive/main.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/apps/ocioarchive/main.cpp b/src/apps/ocioarchive/main.cpp index dfa8c9c026..02bf8dcb9c 100644 --- a/src/apps/ocioarchive/main.cpp +++ b/src/apps/ocioarchive/main.cpp @@ -6,6 +6,8 @@ #include #include +#include "utils/StringUtils.h" + namespace OCIO = OCIO_NAMESPACE; #include "apputils/argparse.h" @@ -144,14 +146,14 @@ int main(int argc, const char **argv) { // The ocioz extension is added by the archive method. The assumption is that // archiveName is the filename without extension. + + // Do not add ocioz extension if already present. + if (!StringUtils::EndsWith(archiveName, ".ocioz")) + { + archiveName += std::string(OCIO::OCIO_CONFIG_ARCHIVE_FILE_EXT); + } - // Remove extension, if present. - char * archiveNameTmp = const_cast(archiveName.c_str()); - mz_path_remove_extension(archiveNameTmp); - archiveName = archiveNameTmp; - - std::ofstream ofstream(archiveName + std::string(OCIO::OCIO_CONFIG_ARCHIVE_FILE_EXT), - std::ofstream::out | std::ofstream::binary); + std::ofstream ofstream(archiveName, std::ofstream::out | std::ofstream::binary); if (ofstream.good()) { config->archive(ofstream); From 9978cfde0aa93d83a3af5947a481f73f513ddd79 Mon Sep 17 00:00:00 2001 From: Cedrik Fuoco Date: Thu, 6 Oct 2022 13:55:15 -0400 Subject: [PATCH 3/8] - Changing getLutData (return vector instead of passing by reference) in order to respect C++ idiom and letting compiler optimize with NRVO and Copy elision - Improving how cmake find zlib (renamed getZLIB to Findzlib) - Make sure that ZLIB is statically linked on all platform (it wasn't on windows) - Fix issue with Python Unit test when using Python 2 (problem discovered on OSX) - Change how OpenEXR finds ZLIB by using our custom Findzlib - Patching Minizip-ng makefile to change cmake minimum version Signed-off-by: Cedrik Fuoco --- include/OpenColorIO/OpenColorIO.h | 2 +- share/cmake/modules/FindExtPackages.cmake | 6 +- share/cmake/modules/FindOpenEXR.cmake | 3 +- share/cmake/modules/Findminizip-ng.cmake | 12 +--- .../modules/{GetZLIB.cmake => Findzlib.cmake} | 55 +++++++++++++++---- .../patches/PatchMinizip-ngCMakelists.cmake | 12 ++++ src/OpenColorIO/OCIOZArchive.cpp | 51 ++++++++--------- src/OpenColorIO/OCIOZArchive.h | 8 +-- src/OpenColorIO/transforms/FileTransform.cpp | 3 +- src/bindings/python/PyConfigIOProxy.cpp | 7 +-- tests/cpu/Config_tests.cpp | 6 +- tests/python/ConfigTest.py | 13 +++-- tests/python/OCIOZArchiveTest.py | 25 +++++++-- 13 files changed, 129 insertions(+), 74 deletions(-) rename share/cmake/modules/{GetZLIB.cmake => Findzlib.cmake} (72%) create mode 100644 share/cmake/patches/PatchMinizip-ngCMakelists.cmake diff --git a/include/OpenColorIO/OpenColorIO.h b/include/OpenColorIO/OpenColorIO.h index 8d98e89439..8793867bf0 100644 --- a/include/OpenColorIO/OpenColorIO.h +++ b/include/OpenColorIO/OpenColorIO.h @@ -3586,7 +3586,7 @@ class OCIOEXPORT ConfigIOProxy * The file path is based on the Config's current working directory and is the same absolute * path that would have been provided to the file system. */ - virtual void getLutData(std::vector & buffer, const char * filepath) const = 0; + virtual std::vector getLutData(const char * filepath) const = 0; /** * \brief Provide the config file Yaml to be parsed. diff --git a/share/cmake/modules/FindExtPackages.cmake b/share/cmake/modules/FindExtPackages.cmake index 9dd8c2a35d..b777e5bb8f 100644 --- a/share/cmake/modules/FindExtPackages.cmake +++ b/share/cmake/modules/FindExtPackages.cmake @@ -41,9 +41,13 @@ find_package(pystring 1.1.3 REQUIRED) set(_Imath_ExternalProject_VERSION "3.1.5") find_package(Imath 3.0 REQUIRED) +# ZLIB +# https://github.com/madler/zlib +set(_zlib_ExternalProject_VERSION "1.2.12") +find_package(zlib REQUIRED) + # minizip-ng # https://github.com/zlib-ng/minizip-ng -set(ZLIB_FIND_VERSION 1.2.12) find_package(minizip-ng 3.0.6 REQUIRED) if(OCIO_BUILD_APPS) diff --git a/share/cmake/modules/FindOpenEXR.cmake b/share/cmake/modules/FindOpenEXR.cmake index 8e742da364..dd438b792e 100644 --- a/share/cmake/modules/FindOpenEXR.cmake +++ b/share/cmake/modules/FindOpenEXR.cmake @@ -85,7 +85,8 @@ if(NOT OpenEXR_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) set(_EXT_BUILD_ROOT "${CMAKE_BINARY_DIR}/ext/build") # Required dependency - find_package(ZLIB) + # OCIO custom module to find ZLIB. (Findzlib) + find_package(zlib) if(NOT ZLIB_FOUND) message(STATUS "ZLib is required to build OpenEXR.") return() diff --git a/share/cmake/modules/Findminizip-ng.cmake b/share/cmake/modules/Findminizip-ng.cmake index e1e53093f3..ed507681db 100644 --- a/share/cmake/modules/Findminizip-ng.cmake +++ b/share/cmake/modules/Findminizip-ng.cmake @@ -15,12 +15,6 @@ ############################################################################### ### Try to find package ### -# Search for ZLIB using FindZLIB from cmake. -# If ZLIB is not found, it will be downloaded and built. -include(GetZLIB) - -# Once ZLIB is sorted out, continue with minizip-ng. - if(NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL ALL) if(NOT DEFINED minizip-ng_ROOT) # Search for minizip-ng-config.cmake @@ -193,6 +187,8 @@ if(NOT minizip-ng_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) GIT_TAG "${minizip-ng_VERSION}" GIT_CONFIG advice.detachedHead=false GIT_SHALLOW TRUE + PATCH_COMMAND + ${CMAKE_COMMAND} -P "${CMAKE_SOURCE_DIR}/share/cmake/patches/PatchMinizip-ngCMakelists.cmake" PREFIX "${_EXT_BUILD_ROOT}/libminizip-ng" BUILD_BYPRODUCTS ${minizip-ng_LIBRARY} CMAKE_ARGS ${MINIZIP-NG_CMAKE_ARGS} @@ -220,7 +216,5 @@ if(_minizip-ng_TARGET_CREATE) mark_as_advanced(minizip-ng_INCLUDE_DIR minizip-ng_LIBRARY minizip-ng_VERSION) - if (NOT MSVC) - target_link_libraries(minizip-ng::minizip-ng INTERFACE ZLIB::ZLIB) - endif() + target_link_libraries(minizip-ng::minizip-ng INTERFACE ZLIB::ZLIB) endif() \ No newline at end of file diff --git a/share/cmake/modules/GetZLIB.cmake b/share/cmake/modules/Findzlib.cmake similarity index 72% rename from share/cmake/modules/GetZLIB.cmake rename to share/cmake/modules/Findzlib.cmake index c21729b662..a305939da4 100644 --- a/share/cmake/modules/GetZLIB.cmake +++ b/share/cmake/modules/Findzlib.cmake @@ -22,18 +22,28 @@ ### Try to find package ### # Assign the rigtt name for ZLIB depending on the OS. -if(UNIX) - set(ZLIB_NAME libz) +if(WIN32) + set(_ZLIB_LIB_NAME "zlib") + set(_ZLIB_STATIC_LIB_NAME "zlibstatic") else() - set(ZLIB_NAME zlib) + set(_ZLIB_LIB_NAME "z") + set(_ZLIB_STATIC_LIB_NAME "z") endif() if(NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL ALL) set(_ZLIB_REQUIRED_VARS ZLIB_LIBRARIES) if(NOT DEFINED ZLIB_ROOT) - # findZLIB provided by CMAKE. - find_package(ZLIB ${ZLIB_FIND_VERSION}) + # Save old value of CMAKE_MODULE_PATH + set(_ZLIB__CMAKE_MODULE_PATH_OLD_ ${CMAKE_MODULE_PATH}) + # Force find_package to use CMAKE module and not custom modules. + set(CMAKE_MODULE_PATH "${CMAKE_ROOT}/Modules") + + # Use CMake FindZLIB module + find_package(ZLIB ${zlib_FIND_VERSION}) + + # Restore CMAKE_MODULE_PATH + set(CMAKE_MODULE_PATH ${_ZLIB__CMAKE_MODULE_PATH_OLD_}) endif() endif() @@ -41,7 +51,6 @@ endif() ### Create target if(NOT TARGET ZLIB::ZLIB) - add_library(ZLIB::ZLIB UNKNOWN IMPORTED GLOBAL) set(_ZLIB_TARGET_CREATE TRUE) endif() @@ -56,11 +65,20 @@ if(NOT ZLIB_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) # Set find_package standard args set(ZLIB_FOUND TRUE) - set(ZLIB_VERSION ${ZLIB_FIND_VERSION}) + if(_zlib_ExternalProject_VERSION) + set(ZLIB_VERSION ${_zlib_ExternalProject_VERSION}) + else() + set(ZLIB_VERSION ${zlib_FIND_VERSION}) + endif() set(ZLIB_INCLUDE_DIRS "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_INCLUDEDIR}") + # Windows need the "d" suffix at the end. + if(WIN32 AND BUILD_TYPE_DEBUG) + set(_ZLIB_LIB_SUFFIX "d") + endif() + set(ZLIB_LIBRARIES - "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_LIBDIR}/${_ZLIB_LIB_PREFIX}${ZLIB_NAME}${_ZLIB_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_ZLIB_STATIC_LIB_NAME}${_ZLIB_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") if(_ZLIB_TARGET_CREATE) set(ZLIB_CMAKE_ARGS @@ -111,8 +129,9 @@ if(NOT ZLIB_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) GIT_TAG "v${ZLIB_VERSION}" GIT_CONFIG advice.detachedHead=false GIT_SHALLOW TRUE - PREFIX "${_EXT_BUILD_ROOT}/${ZLIB_NAME}" - BUILD_BYPRODUCTS ${ZLIB_LIBRARIES} + PREFIX "${_EXT_BUILD_ROOT}/zlib" + BUILD_BYPRODUCTS + ${ZLIB_LIBRARIES} CMAKE_ARGS ${ZLIB_CMAKE_ARGS} EXCLUDE_FROM_ALL TRUE BUILD_COMMAND "" @@ -123,7 +142,23 @@ if(NOT ZLIB_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) --parallel ) + ExternalProject_Add_Step( + ZLIB_install zlib_remove_dll + COMMENT "Remove zlib.lib and zlib.dll, leaves only zlibstatic.lib" + DEPENDEES install + COMMAND ${CMAKE_COMMAND} -E remove -f ${_EXT_DIST_ROOT}/lib/zlib.lib ${_EXT_DIST_ROOT}/bin/zlib.dll + ) + + add_library(ZLIB::ZLIB STATIC IMPORTED GLOBAL) add_dependencies(ZLIB::ZLIB ZLIB_install) + set_property(TARGET ZLIB::ZLIB PROPERTY + IMPORTED_LOCATION "${ZLIB_LIBRARIES}" + ) + target_include_directories(ZLIB::ZLIB INTERFACE "${CMAKE_INSTALL_BINDIR}/include") + + set(ZLIB_LIBRARY ${ZLIB_LIBRARIES}) + set(ZLIB_INCLUDE_DIR ${ZLIB_INCLUDE_DIRS}) + message(STATUS "Installing ZLIB: ${ZLIB_LIBRARIES} (version \"${ZLIB_VERSION}\")") endif() diff --git a/share/cmake/patches/PatchMinizip-ngCMakelists.cmake b/share/cmake/patches/PatchMinizip-ngCMakelists.cmake new file mode 100644 index 0000000000..c2243d6360 --- /dev/null +++ b/share/cmake/patches/PatchMinizip-ngCMakelists.cmake @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright Contributors to the OpenColorIO Project. + +# Minizip-ng cmake minimum version has to be decrease to 3.12 since the container image +# Linux CentOS 7 VFX CY2019 image has cmake 3.12.4. +# CMake is easy to update. The image should probably be updated to a more recent version. + +# cmake_minimum_required(VERSION 3.13) -> cmake_minimum_required(VERSION 3.12) +file(READ CMakeLists.txt _minizip-ng_CMAKE_DATA) +string(REPLACE "cmake_minimum_required(VERSION 3.13)" "cmake_minimum_required(VERSION 3.12)" + _minizip-ng_CMAKE_DATA "${_minizip-ng_CMAKE_DATA}") +file(WRITE CMakeLists.txt "${_minizip-ng_CMAKE_DATA}") \ No newline at end of file diff --git a/src/OpenColorIO/OCIOZArchive.cpp b/src/OpenColorIO/OCIOZArchive.cpp index 6df14a868a..6d172b7373 100644 --- a/src/OpenColorIO/OCIOZArchive.cpp +++ b/src/OpenColorIO/OCIOZArchive.cpp @@ -387,13 +387,11 @@ void ExtractOCIOZArchive(const char * archivePath, const char * destination) * @param info File information. * @param filepath Path to find. */ -void getFileBufferByPath(std::vector & buffer, - void * reader, - mz_zip_file & info, - std::string filepath) +std::vector getFileBufferByPath(void * reader, mz_zip_file & info, std::string filepath) { // Verify that the file information and the current file matches while ignoring the slashes // differences in platforms. + std::vector buffer; if (mz_path_compare_wc(filepath.c_str(), info.filename, 1) == MZ_OK) { // Initialize the buffer for the file. @@ -402,6 +400,7 @@ void getFileBufferByPath(std::vector & buffer, // Read the content of the file and return it as buffer. mz_zip_reader_entry_save_buffer(reader, &buffer[0], buf_size); } + return buffer; } /** @@ -415,12 +414,10 @@ void getFileBufferByPath(std::vector & buffer, * @param info File information * @param extension Extension to find */ -void getFileBufferByExtension(std::vector & buffer, - void * reader, - mz_zip_file & info, - std::string extension) +std::vector getFileBufferByExtension(void * reader, mz_zip_file & info, std::string extension) { std::string root, ext; + std::vector buffer; pystring::os::path::splitext(root, ext, info.filename); if (Platform::Strcasecmp(extension.c_str(), ext.c_str()) == 0) { @@ -428,6 +425,7 @@ void getFileBufferByExtension(std::vector & buffer, buffer.resize(buf_size); mz_zip_reader_entry_save_buffer(reader, &buffer[0], buf_size); } + return buffer; } /** @@ -441,14 +439,14 @@ void getFileBufferByExtension(std::vector & buffer, * @param archivePath Path to the archive. * @param fn Callback function to get the (file) buffer by path or by extension. */ -void getFileStringFromArchiveFile(std::vector & buffer, - const std::string & filepath, +std::vector getFileStringFromArchiveFile(const std::string & filepath, const std::string & archivePath, - void (*fn)(std::vector & buffer, void*, mz_zip_file&, std::string)) + std::vector (*fn)(void*, mz_zip_file&, std::string)) { mz_zip_file *file_info = NULL; int32_t err = MZ_OK; void *reader = NULL; + std::vector buffer; // Create the reader object. mz_zip_reader_create(&reader); @@ -474,7 +472,7 @@ void getFileStringFromArchiveFile(std::vector & buffer, // Get the current entry information. if (mz_zip_reader_entry_get_info(reader, &file_info) == MZ_OK) { - fn(buffer, reader, *file_info, filepath); + buffer = fn(reader, *file_info, filepath); if (!buffer.empty()) { break; @@ -483,6 +481,8 @@ void getFileStringFromArchiveFile(std::vector & buffer, } while (mz_zip_reader_goto_next_entry(reader) == MZ_OK); } } + + return buffer; } ////////////////////////////////////////////////////////////////////////////////////// @@ -492,19 +492,15 @@ void getFileStringFromArchiveFile(std::vector & buffer, ////////////////////////////////////////////////////////////////////////////////////// // See header file for information. -void getFileBufferFromArchive(std::vector & buffer, - const std::string & filepath, - const std::string & archivePath) +std::vector getFileBufferFromArchive(const std::string & filepath, const std::string & archivePath) { - return getFileStringFromArchiveFile(buffer, filepath, archivePath, &getFileBufferByPath); + return getFileStringFromArchiveFile(filepath, archivePath, &getFileBufferByPath); } // See header file for information. -void getFileBufferFromArchiveByExtension(std::vector & buffer, - const std::string & extension, - const std::string & archivePath) +std::vector getFileBufferFromArchiveByExtension(const std::string & extension, const std::string & archivePath) { - return getFileStringFromArchiveFile(buffer, extension, archivePath, &getFileBufferByExtension); + return getFileStringFromArchiveFile(extension, archivePath, &getFileBufferByExtension); } // See header file for information. @@ -559,7 +555,7 @@ void getEntriesMappingFromArchiveFile(const std::string & archivePath, // Implementation of CIOPOciozArchive class. ////////////////////////////////////////////////////////////////////////////////////// -void CIOPOciozArchive::getLutData(std::vector & buffer, const char * filepath) const +std::vector CIOPOciozArchive::getLutData(const char * filepath) const { // In order to ease the implementation and to facilitate a future Python binding, this method // uses std::vector buffer instead of a std::istream. @@ -588,11 +584,14 @@ void CIOPOciozArchive::getLutData(std::vector & buffer, const char * fi ociozStream.seekg(0, std::ios::beg); std::vector archiveBuffer(size); + std::vector buffer; if (ociozStream.read(archiveBuffer.data(), size)) { std::string fpath = pystring::os::path::normpath(filepath); - getFileBufferFromArchive(buffer, fpath, m_archiveAbsPath); + buffer = getFileBufferFromArchive(fpath, m_archiveAbsPath); } + + return buffer; } const std::string CIOPOciozArchive::getConfigData() const @@ -617,13 +616,7 @@ const std::string CIOPOciozArchive::getConfigData() const std::string configFilename = std::string(OCIO_CONFIG_DEFAULT_NAME) + std::string(OCIO_CONFIG_DEFAULT_FILE_EXT); - std::vector configBuffer; - getFileBufferFromArchive( - configBuffer, - configFilename, - m_archiveAbsPath - ); - + std::vector configBuffer = getFileBufferFromArchive(configFilename, m_archiveAbsPath); if (configBuffer.size() > 0) { // Create an string stream from the config array of chars. diff --git a/src/OpenColorIO/OCIOZArchive.h b/src/OpenColorIO/OCIOZArchive.h index 274b31eb73..1055bfda12 100644 --- a/src/OpenColorIO/OCIOZArchive.h +++ b/src/OpenColorIO/OCIOZArchive.h @@ -38,8 +38,7 @@ void archiveConfig( * @param filepath Path to find. * @param archivePath Path to archive */ -void getFileBufferFromArchive( - std::vector & buffer, +std::vector getFileBufferFromArchive( const std::string & filepath, const std::string & archivePath); @@ -52,8 +51,7 @@ void getFileBufferFromArchive( * @param extension Extension to find * @param archivePath Path to archive */ -void getFileBufferFromArchiveByExtension( - std::vector & buffer, +std::vector getFileBufferFromArchiveByExtension( const std::string & extension, const std::string & archivePath); @@ -80,7 +78,7 @@ class CIOPOciozArchive : public ConfigIOProxy CIOPOciozArchive() = default; // See OpenColorIO.h for informations on these five methods. - void getLutData(std::vector & buffer, const char * filepath) const override; + std::vector getLutData(const char * filepath) const override; const std::string getConfigData() const override; // Currently using the filepath of the file + the CRC32. const std::string getFastLutFileHash(const char * filepath) const override; diff --git a/src/OpenColorIO/transforms/FileTransform.cpp b/src/OpenColorIO/transforms/FileTransform.cpp index 5f695f1c64..172c242470 100755 --- a/src/OpenColorIO/transforms/FileTransform.cpp +++ b/src/OpenColorIO/transforms/FileTransform.cpp @@ -186,8 +186,7 @@ std::unique_ptr getLutData( { if (config.getConfigIOProxy()) { - std::vector buffer; - config.getConfigIOProxy()->getLutData(buffer, filepath.c_str()); + std::vector buffer = config.getConfigIOProxy()->getLutData(filepath.c_str()); std::stringstream ss; ss.write(reinterpret_cast(buffer.data()), buffer.size()); diff --git a/src/bindings/python/PyConfigIOProxy.cpp b/src/bindings/python/PyConfigIOProxy.cpp index 2a18961720..5d04161f05 100644 --- a/src/bindings/python/PyConfigIOProxy.cpp +++ b/src/bindings/python/PyConfigIOProxy.cpp @@ -9,6 +9,7 @@ #include #include +#include PYBIND11_MAKE_OPAQUE(std::vector); @@ -18,14 +19,12 @@ struct PyConfigIOProxy : ConfigIOProxy { using ConfigIOProxy::ConfigIOProxy; - void getLutData(std::vector & buffer, const char * filepath) const override + std::vector getLutData(const char * filepath) const override { - py::object dummy = py::cast(&buffer); PYBIND11_OVERRIDE_PURE( - void, + std::vector, ConfigIOProxy, // Parent class getLutData, // Name of function in C++ (must match Python name) - buffer, filepath // Argument(s) ); } diff --git a/tests/cpu/Config_tests.cpp b/tests/cpu/Config_tests.cpp index e987ccdefe..31c60d1dc6 100644 --- a/tests/cpu/Config_tests.cpp +++ b/tests/cpu/Config_tests.cpp @@ -9001,8 +9001,7 @@ OCIO_ADD_TEST(Config, create_from_config_io_proxy) return buffer.str(); } - inline void getLutData( - std::vector & buffer, + inline std::vector getLutData( const char * filepath) const override { std::vector paths = { @@ -9027,9 +9026,12 @@ OCIO_ADD_TEST(Config, create_from_config_io_proxy) } const auto eofPosition = static_cast(fstream.tellg()); + std::vector buffer; buffer.resize(eofPosition); fstream.seekg(0, std::ios::beg); fstream.read(reinterpret_cast(buffer.data()), eofPosition); + + return buffer; } const std::string getFastLutFileHash(const char * filename) const override diff --git a/tests/python/ConfigTest.py b/tests/python/ConfigTest.py index c90f2bf966..57c09bdc04 100644 --- a/tests/python/ConfigTest.py +++ b/tests/python/ConfigTest.py @@ -1110,21 +1110,22 @@ def getConfigData(self): # Simulate that the config is coming from some kind of in-memory location. return SIMPLE_CONFIG - def getLutData(self, buffer, filepath): + def getLutData(self, filepath): # This implementation is only to demonstrate the functionality. # Simulate that the LUT are coming from some kind of in-memory location. - + buffer = OCIO.vector_of_uint8_t() if filepath == os.path.join('my_unique_luts', 'my_unique_lut1.clf'): encoded_c1_lut = C1_LUT.encode('utf-8') - lst = list(encoded_c1_lut) - for c in lst: + data = bytearray(encoded_c1_lut) + for c in data: buffer.append(c) elif filepath == os.path.join('my_unique_luts', 'my_unique_lut2.clf'): encoded_c2_lut = C2_LUT.encode('utf-8') - lst = list(encoded_c2_lut) - for c in lst: + data = bytearray(encoded_c2_lut) + for c in data: buffer.append(c) + return buffer def getFastLutFileHash(self, filepath): # This implementation is only to demonstrate the functionality. diff --git a/tests/python/OCIOZArchiveTest.py b/tests/python/OCIOZArchiveTest.py index 8ee8a4edad..ea447fce3b 100644 --- a/tests/python/OCIOZArchiveTest.py +++ b/tests/python/OCIOZArchiveTest.py @@ -4,7 +4,11 @@ import unittest import os import platform +import sys + import tempfile +if sys.version_info.major < 3: + import shutil import PyOpenColorIO as OCIO from UnitTestUtils import (SIMPLE_CONFIG, TEST_DATAFILES_DIR) @@ -366,15 +370,22 @@ def test_extract_config_and_compare_to_original(self): """ # Testing CreateFromFile and ExtractOCIOZArchive on a successful path. + temp_dir_name = "" try: - temp_ocioz_directory = tempfile.TemporaryDirectory() + if sys.version_info.major >= 3: + # Python 3. + self.temp_ocioz_directory = tempfile.TemporaryDirectory() + temp_dir_name = self.temp_ocioz_directory.name + else: + # Python 2. + temp_dir_name = tempfile.mkdtemp() # 1 - Extract the OCIOZ archive to temporary directory. - OCIO.ExtractOCIOZArchive(self.ORIGNAL_ARCHIVED_CONFIG_PATH, temp_ocioz_directory.name) + OCIO.ExtractOCIOZArchive(self.ORIGNAL_ARCHIVED_CONFIG_PATH, temp_dir_name) # 2 - Create config from extracted OCIOZ archive. new_config = OCIO.Config().CreateFromFile( - os.path.join(temp_ocioz_directory.name, "config.ocio") + os.path.join(temp_dir_name, "config.ocio") ) new_config.validate() @@ -396,4 +407,10 @@ def test_extract_config_and_compare_to_original(self): # 5 - Compare serialization self.assertEqual(self.ORIGNAL_ARCHIVED_CONFIG.serialize(), new_config.serialize()) finally: - temp_ocioz_directory.cleanup() \ No newline at end of file + if sys.version_info.major >= 3: + # Python 3. + self.temp_ocioz_directory.cleanup() + else: + # Python 2. + if not temp_dir_name: + shutil.rmtree(temp_dir_name) \ No newline at end of file From 83a26aaaaed72a6bcea5dabc4a17a7712b006b25 Mon Sep 17 00:00:00 2001 From: Cedrik Fuoco Date: Wed, 12 Oct 2022 12:48:25 -0400 Subject: [PATCH 4/8] - Remove the minizip-ng patch as it doesn't work and further discussion needs to happend. - Responding to Remi's comments. Notable changes: - Comments typo and styles fix. - Unit tests fixes. - Removing the usage of std::ifstream in getLutData and getConfig since it is not needed at all - It was slowing down getLutData. - Implicitly converting python list to std::vector (for getLutData) - When archiving a config, the file won't added (overwridden) multiple times if the file matches multiple supported format. Signed-off-by: Cedrik Fuoco --- include/OpenColorIO/OpenColorIO.h | 6 +- share/cmake/modules/Findminizip-ng.cmake | 2 - .../patches/PatchMinizip-ngCMakelists.cmake | 12 ---- src/OpenColorIO/Config.cpp | 10 +--- src/OpenColorIO/OCIOZArchive.cpp | 60 ++++--------------- src/OpenColorIO/OCIOZArchive.h | 6 +- src/bindings/python/PyConfig.cpp | 6 +- src/bindings/python/PyConfigIOProxy.cpp | 5 +- tests/cpu/Config_tests.cpp | 4 +- tests/cpu/OCIOZArchive_tests.cpp | 13 ++-- tests/cpu/UnitTestUtils.cpp | 6 +- tests/python/ConfigTest.py | 17 ++---- tests/python/OCIOZArchiveTest.py | 9 +-- 13 files changed, 44 insertions(+), 112 deletions(-) delete mode 100644 share/cmake/patches/PatchMinizip-ngCMakelists.cmake diff --git a/include/OpenColorIO/OpenColorIO.h b/include/OpenColorIO/OpenColorIO.h index 8793867bf0..7d6a0351cc 100644 --- a/include/OpenColorIO/OpenColorIO.h +++ b/include/OpenColorIO/OpenColorIO.h @@ -175,7 +175,7 @@ extern OCIOEXPORT void LogMessage(LoggingLevel level, const char * message); /** * \brief Set the Compute Hash Function to use; otherwise, use the default. * - * This is not used when using CreateFromFile with an OCIOZ archive or CreateFromConfigProxyIO. + * This is not used when using CreateFromFile with an OCIOZ archive or CreateFromConfigIOProxy. * * \param ComputeHashFunction */ @@ -3593,7 +3593,7 @@ class OCIOEXPORT ConfigIOProxy * * \return String with the config Yaml. */ - virtual const std::string getConfigData() const = 0; + virtual std::string getConfigData() const = 0; /** * \brief Provide a fast unique ID for a LUT file. @@ -3612,7 +3612,7 @@ class OCIOEXPORT ConfigIOProxy * * \return The file hash string. */ - virtual const std::string getFastLutFileHash(const char * filepath) const = 0; + virtual std::string getFastLutFileHash(const char * filepath) const = 0; }; } // namespace OCIO_NAMESPACE diff --git a/share/cmake/modules/Findminizip-ng.cmake b/share/cmake/modules/Findminizip-ng.cmake index ed507681db..528d1d0d3e 100644 --- a/share/cmake/modules/Findminizip-ng.cmake +++ b/share/cmake/modules/Findminizip-ng.cmake @@ -187,8 +187,6 @@ if(NOT minizip-ng_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) GIT_TAG "${minizip-ng_VERSION}" GIT_CONFIG advice.detachedHead=false GIT_SHALLOW TRUE - PATCH_COMMAND - ${CMAKE_COMMAND} -P "${CMAKE_SOURCE_DIR}/share/cmake/patches/PatchMinizip-ngCMakelists.cmake" PREFIX "${_EXT_BUILD_ROOT}/libminizip-ng" BUILD_BYPRODUCTS ${minizip-ng_LIBRARY} CMAKE_ARGS ${MINIZIP-NG_CMAKE_ARGS} diff --git a/share/cmake/patches/PatchMinizip-ngCMakelists.cmake b/share/cmake/patches/PatchMinizip-ngCMakelists.cmake deleted file mode 100644 index c2243d6360..0000000000 --- a/share/cmake/patches/PatchMinizip-ngCMakelists.cmake +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright Contributors to the OpenColorIO Project. - -# Minizip-ng cmake minimum version has to be decrease to 3.12 since the container image -# Linux CentOS 7 VFX CY2019 image has cmake 3.12.4. -# CMake is easy to update. The image should probably be updated to a more recent version. - -# cmake_minimum_required(VERSION 3.13) -> cmake_minimum_required(VERSION 3.12) -file(READ CMakeLists.txt _minizip-ng_CMAKE_DATA) -string(REPLACE "cmake_minimum_required(VERSION 3.13)" "cmake_minimum_required(VERSION 3.12)" - _minizip-ng_CMAKE_DATA "${_minizip-ng_CMAKE_DATA}") -file(WRITE CMakeLists.txt "${_minizip-ng_CMAKE_DATA}") \ No newline at end of file diff --git a/src/OpenColorIO/Config.cpp b/src/OpenColorIO/Config.cpp index 26d8b949e6..42d5269771 100644 --- a/src/OpenColorIO/Config.cpp +++ b/src/OpenColorIO/Config.cpp @@ -1171,18 +1171,14 @@ ConstConfigRcPtr Config::CreateFromFile(const char * filename) // The file should be an OCIOZ archive file. - // Using new because OCIO doesn't have access to make_unique since OCIO still supports - // C++11. - std::shared_ptr ciop = std::shared_ptr( - new CIOPOciozArchive() - ); + auto ciop = std::make_shared(); // Store archive absolute path. ciop->setArchiveAbsPath(filename); // Build the entries map. ciop->buildEntries(); return CreateFromConfigIOProxy(ciop); } - } + } // Not an OCIOZ archive. Continue as usual. ifstream.clear(); @@ -4941,7 +4937,7 @@ bool Config::isArchivable() const // Check that FileTransform sources are not absolute nor have context variables outside of // config working directory. - for (auto path : files) + for (const auto & path : files) { if (!validatePathForArchiving(path)) { diff --git a/src/OpenColorIO/OCIOZArchive.cpp b/src/OpenColorIO/OCIOZArchive.cpp index 6d172b7373..fb551b7518 100644 --- a/src/OpenColorIO/OCIOZArchive.cpp +++ b/src/OpenColorIO/OCIOZArchive.cpp @@ -179,11 +179,9 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo FormatRegistry & formatRegistry = FormatRegistry::GetInstance(); FileFormatVector possibleFormats; formatRegistry.getFileFormatForExtension(ext, possibleFormats); - FileFormatVector::const_iterator endFormat = possibleFormats.end(); - FileFormatVector::const_iterator itFormat = possibleFormats.begin(); - while(itFormat != endFormat) + if (!possibleFormats.empty()) { - // Valid format. Archive current file. + // The extension is supported. Add the current file to the OCIOZ archive. if (mz_zip_writer_add_path( archiver, absPath.c_str(), configWorkingDirectory, 0, 1) != MZ_OK) @@ -195,7 +193,6 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo os << "Could not write LUT file " << absPath << " to in-memory archive."; throw Exception(os.str().c_str()); } - ++itFormat; } } } @@ -431,7 +428,7 @@ std::vector getFileBufferByExtension(void * reader, mz_zip_file & info, /** * @brief Get the content of a file inside an OCIOZ archive as a buffer. * - * The two possbiles callbacks are defined above: + * The two possibles callbacks are defined above: * getFileBufferByPath and getFileBufferByExtension. * * @param buffer Buffer of uint8_t @@ -567,68 +564,31 @@ std::vector CIOPOciozArchive::getLutData(const char * filepath) const // instead of a std::istream (max 5%). But the following iterations are just as fast due to // the FileTransform cache. - std::ifstream ociozStream = Platform::CreateInputFileStream( - m_archiveAbsPath.c_str(), - std::ios_base::in | std::ios_base::binary - ); - - if (ociozStream.fail()) - { - std::ostringstream os; - os << "Error could not open OCIOZ archive: " << m_archiveAbsPath; - throw Exception (os.str().c_str()); - } - - ociozStream.seekg(0, std::ios::end); - std::streamsize size = ociozStream.tellg(); - ociozStream.seekg(0, std::ios::beg); - - std::vector archiveBuffer(size); std::vector buffer; - if (ociozStream.read(archiveBuffer.data(), size)) - { - std::string fpath = pystring::os::path::normpath(filepath); - buffer = getFileBufferFromArchive(fpath, m_archiveAbsPath); - } - + buffer = getFileBufferFromArchive(pystring::os::path::normpath(filepath), m_archiveAbsPath); return buffer; } -const std::string CIOPOciozArchive::getConfigData() const +std::string CIOPOciozArchive::getConfigData() const { // In order to ease the implementation and to facilitate a future Python binding, this method // returns a std::string instead of a std::istream. // // It is expected that in most cases, the compiler will be able to avoid copying the buffer // (RVO/NRVO). - - std::ifstream ociozStream = Platform::CreateInputFileStream( - m_archiveAbsPath.c_str(), - std::ios_base::in | std::ios_base::binary - ); - - if (ociozStream.fail()) - { - std::ostringstream os; - os << "Error could not read OCIOZ archive: " << m_archiveAbsPath; - throw Exception (os.str().c_str()); - } - + std::string configData = ""; std::string configFilename = std::string(OCIO_CONFIG_DEFAULT_NAME) + std::string(OCIO_CONFIG_DEFAULT_FILE_EXT); std::vector configBuffer = getFileBufferFromArchive(configFilename, m_archiveAbsPath); if (configBuffer.size() > 0) { - // Create an string stream from the config array of chars. - // Specifiy the size of the string since the char * returned by minizip-ng function - // (mz_zip_reader_entry_save_buffer) is not null-terminated. - return std::string(reinterpret_cast(configBuffer.data()), configBuffer.size()); + configData = std::string(configBuffer.begin(), configBuffer.end()); } - return ""; + return configData; } -const std::string CIOPOciozArchive::getFastLutFileHash(const char * filepath) const +std::string CIOPOciozArchive::getFastLutFileHash(const char * filepath) const { std::string hash = ""; // Check into the map to check if the file exists in the archive. @@ -647,7 +607,7 @@ const std::string CIOPOciozArchive::getFastLutFileHash(const char * filepath) co return hash; } -void CIOPOciozArchive::setArchiveAbsPath(std::string absPath) +void CIOPOciozArchive::setArchiveAbsPath(const std::string & absPath) { m_archiveAbsPath = absPath; } diff --git a/src/OpenColorIO/OCIOZArchive.h b/src/OpenColorIO/OCIOZArchive.h index 1055bfda12..b5d976da56 100644 --- a/src/OpenColorIO/OCIOZArchive.h +++ b/src/OpenColorIO/OCIOZArchive.h @@ -79,9 +79,9 @@ class CIOPOciozArchive : public ConfigIOProxy // See OpenColorIO.h for informations on these five methods. std::vector getLutData(const char * filepath) const override; - const std::string getConfigData() const override; + std::string getConfigData() const override; // Currently using the filepath of the file + the CRC32. - const std::string getFastLutFileHash(const char * filepath) const override; + std::string getFastLutFileHash(const char * filepath) const override; // Following methods are specific to CIOPOciozArchive. /** @@ -89,7 +89,7 @@ class CIOPOciozArchive : public ConfigIOProxy * * @param absPath Absolute path to OCIOZ archive. */ - void setArchiveAbsPath(std::string absPath); + void setArchiveAbsPath(const std::string & absPath); /** * @brief Build a map of the zip file table of contents for the files in the archive. diff --git a/src/bindings/python/PyConfig.cpp b/src/bindings/python/PyConfig.cpp index f9a38911fc..cfca6ea454 100644 --- a/src/bindings/python/PyConfig.cpp +++ b/src/bindings/python/PyConfig.cpp @@ -734,13 +734,13 @@ void bindPyConfig(py::module & m) // Archiving .def("isArchivable", &Config::isArchivable, DOC(Config, isArchivable)) - .def("archive", [](ConfigRcPtr & self, const std::string filepath) + .def("archive", [](ConfigRcPtr & self, const char * filepath) { - std::ofstream f(filepath.c_str(), std::ofstream::out | std::ofstream::binary); + std::ofstream f(filepath, std::ofstream::out | std::ofstream::binary); self->archive(f); f.close(); }, - DOC(Config, isArchivable)) + DOC(Config, archive)) // Conversion to string .def("__str__", [](ConfigRcPtr & self) diff --git a/src/bindings/python/PyConfigIOProxy.cpp b/src/bindings/python/PyConfigIOProxy.cpp index 5d04161f05..2916e205df 100644 --- a/src/bindings/python/PyConfigIOProxy.cpp +++ b/src/bindings/python/PyConfigIOProxy.cpp @@ -29,7 +29,7 @@ struct PyConfigIOProxy : ConfigIOProxy ); } - const std::string getConfigData() const override + std::string getConfigData() const override { PYBIND11_OVERRIDE_PURE( std::string, @@ -38,7 +38,7 @@ struct PyConfigIOProxy : ConfigIOProxy ); } - const std::string getFastLutFileHash(const char * filepath) const override + std::string getFastLutFileHash(const char * filepath) const override { PYBIND11_OVERRIDE_PURE( std::string, // Return type @@ -52,6 +52,7 @@ struct PyConfigIOProxy : ConfigIOProxy void bindPyConfigIOProxy(py::module & m) { py::bind_vector>(m, "vector_of_uint8_t"); + py::implicitly_convertible>(); py::class_, PyConfigIOProxy>(m, "PyConfigIOProxy") .def(py::init()) diff --git a/tests/cpu/Config_tests.cpp b/tests/cpu/Config_tests.cpp index 31c60d1dc6..e33de6f785 100644 --- a/tests/cpu/Config_tests.cpp +++ b/tests/cpu/Config_tests.cpp @@ -8982,7 +8982,7 @@ OCIO_ADD_TEST(Config, create_from_config_io_proxy) class CIOPTest : public OCIO::ConfigIOProxy { public: - inline const std::string getConfigData() const override + inline std::string getConfigData() const override { // Get config data from filesystem, database, memory, etc. // In this example, the config is simply coming from the filesystem. @@ -9034,7 +9034,7 @@ OCIO_ADD_TEST(Config, create_from_config_io_proxy) return buffer; } - const std::string getFastLutFileHash(const char * filename) const override + inline std::string getFastLutFileHash(const char * filename) const override { std::vector paths = { std::string(OCIO::GetTestFilesDir()), diff --git a/tests/cpu/OCIOZArchive_tests.cpp b/tests/cpu/OCIOZArchive_tests.cpp index 5a9b171ab6..289048f829 100644 --- a/tests/cpu/OCIOZArchive_tests.cpp +++ b/tests/cpu/OCIOZArchive_tests.cpp @@ -76,7 +76,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable) OCIO::ConfigRcPtr cfg; OCIO_CHECK_NO_THROW(cfg = OCIO::Config::CreateFromStream(iss)->createEditableCopy()); - // Since a working directory is needed to archive a config, settting a fake working directory + // Since a working directory is needed to archive a config, setting a fake working directory // in order to test the search paths and FileTransform source logic. #ifdef _WIN32 cfg->setWorkingDir(R"(C:\fake_working_dir)"); @@ -92,10 +92,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable) /* * Legal scenarios */ - - // No search path. - OCIO_CHECK_EQUAL(true, cfg->isArchivable()); - + // Valid search path. cfg->setSearchPath("luts"); OCIO_CHECK_EQUAL(true, cfg->isArchivable()); @@ -499,8 +496,8 @@ OCIO_ADD_TEST(OCIOZArchive, extract_config_and_compare_to_original) * This test is doing the following : * * 1 - Create a config object from context_test1_windows.ocioz. - * 1 - Extract the config context_test1_windows.ocioz. - * 2 - Create a config object using the extracted config in step 1. + * 2 - Extract the config context_test1_windows.ocioz. + * 3 - Create a config object using the extracted config in step 1. * 4 - Compare different elements between the two configs. * * Testing CreateFromFile and ExtractOCIOZArchive on a successful path. @@ -537,7 +534,7 @@ OCIO_ADD_TEST(OCIOZArchive, extract_config_and_compare_to_original) pystring::os::path::join(dGuard.m_directoryPath, "config.ocio").c_str() )); OCIO_REQUIRE_ASSERT(configFromExtractedArchive); - OCIO_CHECK_NO_THROW(configFromArchive->validate()); + OCIO_CHECK_NO_THROW(configFromExtractedArchive->validate()); // 4 - Compare config cacheID - configFromArchive vs configFromExtractedArchive. OCIO::ConstContextRcPtr context; diff --git a/tests/cpu/UnitTestUtils.cpp b/tests/cpu/UnitTestUtils.cpp index 87a7b72cb7..9456e882d1 100644 --- a/tests/cpu/UnitTestUtils.cpp +++ b/tests/cpu/UnitTestUtils.cpp @@ -75,15 +75,15 @@ std::string CreateTemporaryDirectory(const std::string & name) { int nError = 0; - #if defined(_WIN32) +#if defined(_WIN32) std::string sPath = GetEnvVariable("TEMP"); static const std::string directory = pystring::os::path::join(sPath, name); nError = _mkdir(directory.c_str()); - #else +#else std::string sPath = "/tmp"; const std::string directory = pystring::os::path::join(sPath, name); nError = mkdir(directory.c_str(), 0777); - #endif +#endif if (nError != 0) { diff --git a/tests/python/ConfigTest.py b/tests/python/ConfigTest.py index 57c09bdc04..90a07d96d2 100644 --- a/tests/python/ConfigTest.py +++ b/tests/python/ConfigTest.py @@ -1028,7 +1028,7 @@ def test_create_from_archive(self): with self.assertRaises(OCIO.Exception): OCIO.Config.CreateFromFile(ocioz_file) - # Missing config file but contains LUTs files. + # Missing LUT files but contains LUTs files. ocioz_file = os.path.normpath( os.path.join( TEST_DATAFILES_DIR, 'configs', 'ocioz_archive_configs', 'config_missing_luts.ocioz' @@ -1114,18 +1114,13 @@ def getLutData(self, filepath): # This implementation is only to demonstrate the functionality. # Simulate that the LUT are coming from some kind of in-memory location. - buffer = OCIO.vector_of_uint8_t() + # Create an empty list + data = list() if filepath == os.path.join('my_unique_luts', 'my_unique_lut1.clf'): - encoded_c1_lut = C1_LUT.encode('utf-8') - data = bytearray(encoded_c1_lut) - for c in data: - buffer.append(c) + data = list(C1_LUT.encode('utf-8')) elif filepath == os.path.join('my_unique_luts', 'my_unique_lut2.clf'): - encoded_c2_lut = C2_LUT.encode('utf-8') - data = bytearray(encoded_c2_lut) - for c in data: - buffer.append(c) - return buffer + data = list(C2_LUT.encode('utf-8')) + return data def getFastLutFileHash(self, filepath): # This implementation is only to demonstrate the functionality. diff --git a/tests/python/OCIOZArchiveTest.py b/tests/python/OCIOZArchiveTest.py index ea447fce3b..1f364101cb 100644 --- a/tests/python/OCIOZArchiveTest.py +++ b/tests/python/OCIOZArchiveTest.py @@ -20,7 +20,7 @@ def setUp(self): Initialize the CONFIG """ self.CONFIG = OCIO.Config().CreateFromStream(SIMPLE_CONFIG) - # Since a working directory is needed to archive a config, settting a fake working directory + # Since a working directory is needed to archive a config, setting a fake working directory # in order to test the search paths and FileTransform source logic. if platform.system() == 'Windows': self.CONFIG.setWorkingDir('C:\\fake_working_dir') @@ -39,9 +39,6 @@ def test_is_archivable(self): # Testing search paths ############################### - # No search path. - self.assertEqual(True, self.CONFIG.isArchivable()) - # Valid search path. self.CONFIG.setSearchPath('luts') self.assertEqual(True, self.CONFIG.isArchivable()) @@ -123,7 +120,7 @@ def test_is_archivable(self): # Function to facilitate adding a new FileTransform to a config. def addFTAndTestIsArchivable(cfg, path, isArchivable): - fullPath = os.path.join(path, "fake_lut.cls") + fullPath = os.path.join(path, "fake_lut.clf") ft = OCIO.FileTransform() ft.setSrc(fullPath) @@ -326,7 +323,7 @@ def setUp(self): self.ORIGNAL_ARCHIVED_CONFIG = OCIO.Config().CreateFromFile( self.ORIGNAL_ARCHIVED_CONFIG_PATH ) - self.ORIGINAL_CONFIG.validate() + self.ORIGNAL_ARCHIVED_CONFIG.validate() def tearDown(self): self.CONFIG = None From 88f53369bbbb6455b36e3a17c297d98fa5e5bdf3 Mon Sep 17 00:00:00 2001 From: Cedrik Fuoco Date: Wed, 12 Oct 2022 14:36:41 -0400 Subject: [PATCH 5/8] - Added implicit conversion from bytearray to std::vector - In ConfigIOProxy Python Unit test, changed getLutData to return a bytearray instead of a list. I think this makes more sense and it works with Python 2 and 3. Signed-off-by: Cedrik Fuoco --- src/bindings/python/PyConfigIOProxy.cpp | 1 + tests/python/ConfigTest.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/bindings/python/PyConfigIOProxy.cpp b/src/bindings/python/PyConfigIOProxy.cpp index 2916e205df..544b81bff2 100644 --- a/src/bindings/python/PyConfigIOProxy.cpp +++ b/src/bindings/python/PyConfigIOProxy.cpp @@ -53,6 +53,7 @@ void bindPyConfigIOProxy(py::module & m) { py::bind_vector>(m, "vector_of_uint8_t"); py::implicitly_convertible>(); + py::implicitly_convertible>(); py::class_, PyConfigIOProxy>(m, "PyConfigIOProxy") .def(py::init()) diff --git a/tests/python/ConfigTest.py b/tests/python/ConfigTest.py index 90a07d96d2..c7cebf03d7 100644 --- a/tests/python/ConfigTest.py +++ b/tests/python/ConfigTest.py @@ -1114,12 +1114,12 @@ def getLutData(self, filepath): # This implementation is only to demonstrate the functionality. # Simulate that the LUT are coming from some kind of in-memory location. - # Create an empty list - data = list() + # Create an empty bytearray. + data = bytearray() if filepath == os.path.join('my_unique_luts', 'my_unique_lut1.clf'): - data = list(C1_LUT.encode('utf-8')) + data = bytearray(C1_LUT.encode('utf-8')) elif filepath == os.path.join('my_unique_luts', 'my_unique_lut2.clf'): - data = list(C2_LUT.encode('utf-8')) + data = bytearray(C2_LUT.encode('utf-8')) return data def getFastLutFileHash(self, filepath): From 234a6f08aac2cdea2d99e98b646cbe4956a1ae05 Mon Sep 17 00:00:00 2001 From: Cedrik Fuoco Date: Thu, 13 Oct 2022 09:36:38 -0400 Subject: [PATCH 6/8] Update comments to be in line with the current implementations Signed-off-by: Cedrik Fuoco --- include/OpenColorIO/OpenColorIO.h | 5 ++- src/OpenColorIO/OCIOZArchive.cpp | 51 +++++++++++++------------ src/OpenColorIO/OCIOZArchive.h | 39 ++++++++++--------- src/OpenColorIO/Platform.h | 2 +- src/apps/ocioarchive/main.cpp | 4 +- src/bindings/python/PyConfigIOProxy.cpp | 22 +++++------ tests/cpu/OCIOZArchive_tests.cpp | 4 +- tests/cpu/UnitTestUtils.h | 10 ++--- tests/python/ConfigTest.py | 8 ++-- 9 files changed, 75 insertions(+), 70 deletions(-) diff --git a/include/OpenColorIO/OpenColorIO.h b/include/OpenColorIO/OpenColorIO.h index 7d6a0351cc..e747089f54 100644 --- a/include/OpenColorIO/OpenColorIO.h +++ b/include/OpenColorIO/OpenColorIO.h @@ -3580,11 +3580,12 @@ class OCIOEXPORT ConfigIOProxy /** * \brief Provide the contents of a LUT file as a buffer of uint8_t data. * - * \param buffer Contents of the LUT file. * \param filepath Fully resolved path to the "file." * * The file path is based on the Config's current working directory and is the same absolute * path that would have been provided to the file system. + * + * \return Vector of uint8 with the content of the LUT. */ virtual std::vector getLutData(const char * filepath) const = 0; @@ -3605,7 +3606,7 @@ class OCIOEXPORT ConfigIOProxy * If the "file" does not exist, in other words, if the proxy is unable to supply the requested * file contents, the function must return an empty string. * - * \param filepath Fully resolve path to the "file." + * \param filepath Fully resolve the path to the "file." * * The file path is based on the Config's current working directory and is the same absolute * path that would have been provided to the file system. diff --git a/src/OpenColorIO/OCIOZArchive.cpp b/src/OpenColorIO/OCIOZArchive.cpp index fb551b7518..52e292ef77 100644 --- a/src/OpenColorIO/OCIOZArchive.cpp +++ b/src/OpenColorIO/OCIOZArchive.cpp @@ -64,7 +64,7 @@ struct ArchiveOptions { }; /** - * @brief Guard against early throws with Minizip-ng objects. + * \brief Guard against early throws with Minizip-ng objects. * */ struct MinizipNgHandlerGuard @@ -141,9 +141,9 @@ struct MinizipNgMemStreamGuard /** * Utility function for archived Configs. * - * @param archiver Minizip-ng handle object. - * @param path Path of the file or the folder to add inside the OCIOZ archive. - * @param configWorkingDirectory Working directory of the current config. + * \param archiver Minizip-ng handle object. + * \param path Path of the file or the folder to add inside the OCIOZ archive. + * \param configWorkingDirectory Working directory of the current config. */ void addSupportedFiles(void * archiver, const char * path, const char * configWorkingDirectory) { @@ -199,6 +199,7 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo mz_os_close_dir(dir); } } +////////////////////////////////////////////////////////////////////////////////////// void archiveConfig(std::ostream & ostream, const Config & config, const char * configWorkingDirectory) { @@ -316,7 +317,7 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c } /** - * @brief Extract the specified OCIOZ archive. + * \brief Extract the specified OCIOZ archive. * * This function can only be used with the OCIOZ archive format (not arbitrary zip files). * @@ -374,15 +375,16 @@ void ExtractOCIOZArchive(const char * archivePath, const char * destination) } /** - * @brief Callback function for getFileStringFromArchiveStream in order to get the contents of a + * \brief Callback function for getFileStringFromArchiveStream in order to get the contents of a * file inside an OCIOZ archive as a buffer. * * The file is retrieved by comparing the paths. * - * @param buffer Buffer of uint8_t - * @param reader Minizip-ng handle object. - * @param info File information. - * @param filepath Path to find. + * \param reader Minizip-ng handle object. + * \param info File information. + * \param filepath Path to find. + * + * \return Vector of uint8 with the content of the specified file from an OCIOZ archive. */ std::vector getFileBufferByPath(void * reader, mz_zip_file & info, std::string filepath) { @@ -401,15 +403,16 @@ std::vector getFileBufferByPath(void * reader, mz_zip_file & info, std: } /** - * @brief Callback function for getFileStringFromArchiveStream in order to Get the content of a + * \brief Callback function for getFileStringFromArchiveStream in order to Get the content of a * file inside an OCIOZ archive as a buffer. * * The file is retrieved by comparing only the extension. (Used for the .ocio file.) * - * @param buffer Buffer of uint8_t - * @param reader Minizip-ng reader object. - * @param info File information - * @param extension Extension to find + * \param reader Minizip-ng reader object. + * \param info File information + * \param extension Extension to find + * + * \return Vector of uint8 with the content of the file from an OCIOZ archive. */ std::vector getFileBufferByExtension(void * reader, mz_zip_file & info, std::string extension) { @@ -426,15 +429,16 @@ std::vector getFileBufferByExtension(void * reader, mz_zip_file & info, } /** - * @brief Get the content of a file inside an OCIOZ archive as a buffer. + * \brief Get the content of a file inside an OCIOZ archive as a buffer. * - * The two possibles callbacks are defined above: + * The two possible callbacks are defined above: * getFileBufferByPath and getFileBufferByExtension. * - * @param buffer Buffer of uint8_t - * @param filepath File to retrieve from the OCIOZ archive. - * @param archivePath Path to the archive. - * @param fn Callback function to get the (file) buffer by path or by extension. + * \param filepath File to retrieve from the OCIOZ archive. + * \param archivePath Path to the archive. + * \param fn Callback function to get the (file) buffer by path or by extension. + * + * \return Vector of uint8 with the content of the specified file from an OCIOZ archive. */ std::vector getFileStringFromArchiveFile(const std::string & filepath, const std::string & archivePath, @@ -450,7 +454,7 @@ std::vector getFileStringFromArchiveFile(const std::string & filepath, MinizipNgHandlerGuard extracterGuard(reader, false, true); - // Open the zip in memory. + // Open the OCIOZ archive from the filesystem. err = mz_zip_reader_open_file(reader, archivePath.c_str()); if (err != MZ_OK) { @@ -488,19 +492,16 @@ std::vector getFileStringFromArchiveFile(const std::string & filepath, // API section ////////////////////////////////////////////////////////////////////////////////////// -// See header file for information. std::vector getFileBufferFromArchive(const std::string & filepath, const std::string & archivePath) { return getFileStringFromArchiveFile(filepath, archivePath, &getFileBufferByPath); } -// See header file for information. std::vector getFileBufferFromArchiveByExtension(const std::string & extension, const std::string & archivePath) { return getFileStringFromArchiveFile(extension, archivePath, &getFileBufferByExtension); } -// See header file for information. void getEntriesMappingFromArchiveFile(const std::string & archivePath, std::map & map) { diff --git a/src/OpenColorIO/OCIOZArchive.h b/src/OpenColorIO/OCIOZArchive.h index b5d976da56..5bdbc6315b 100644 --- a/src/OpenColorIO/OCIOZArchive.h +++ b/src/OpenColorIO/OCIOZArchive.h @@ -16,13 +16,13 @@ namespace OCIO_NAMESPACE { /** - * @brief Archive a config into an OCIOZ file. + * \brief Archive a config into an OCIOZ file. * * Note: The config file inside the archive is hardcoded to "config.ocio". * - * @param ostream Output stream to write the data into. - * @param config Config object. - * @param configWorkingDirectory Working directory of the current config. + * \param ostream Output stream to write the data into. + * \param config Config object. + * \param configWorkingDirectory Working directory of the current config. */ void archiveConfig( std::ostream & ostream, @@ -30,41 +30,43 @@ void archiveConfig( const char * configWorkingDirectory); /** - * @brief Get the content of a file inside an OCIOZ archive as a buffer. + * \brief Get the content of a file inside an OCIOZ archive as a buffer. * * The file is retrieve by comparing the paths. * - * @param buffer Buffer - * @param filepath Path to find. - * @param archivePath Path to archive + * \param filepath Path to find. + * \param archivePath Path to archive + * + * \return Vector of uint8 with the content of the specified file from an OCIOZ archive. */ std::vector getFileBufferFromArchive( const std::string & filepath, const std::string & archivePath); /** - * @brief Get the content of a file inside an OCIOZ archive as a buffer. + * \brief Get the content of a file inside an OCIOZ archive as a buffer. * * The file is retrieve by comparing the extensions. * - * @param buffer Buffer - * @param extension Extension to find - * @param archivePath Path to archive + * \param extension Extension to find + * \param archivePath Path to archive + * + * \return Vector of uint8 with the content of the specified file from an OCIOZ archive. */ std::vector getFileBufferFromArchiveByExtension( const std::string & extension, const std::string & archivePath); /** - * @brief Get the Entries from OCIOZ archive + * \brief Get the Entries from OCIOZ archive * * Populate a std::map object with the following information: * key => value : full_path_of_the_file_inside_archive => calculated_hash_of_the_file * * The hash is calculated using the full path of the file inside the archive and its CRC32. * - * @param buffer Path to archive. - * @param map std::map object to be populated + * \param archivePath Path to archive. + * \param map std::map object to be populated */ void getEntriesMappingFromArchiveFile( const std::string & archivePath, @@ -78,6 +80,7 @@ class CIOPOciozArchive : public ConfigIOProxy CIOPOciozArchive() = default; // See OpenColorIO.h for informations on these five methods. + std::vector getLutData(const char * filepath) const override; std::string getConfigData() const override; // Currently using the filepath of the file + the CRC32. @@ -85,14 +88,14 @@ class CIOPOciozArchive : public ConfigIOProxy // Following methods are specific to CIOPOciozArchive. /** - * @brief Set the OCIOZ archive absolute path. + * \brief Set the OCIOZ archive absolute path. * - * @param absPath Absolute path to OCIOZ archive. + * \param absPath Absolute path to OCIOZ archive. */ void setArchiveAbsPath(const std::string & absPath); /** - * @brief Build a map of the zip file table of contents for the files in the archive. + * \brief Build a map of the zip file table of contents for the files in the archive. * * The structure is a std::map with the key as the full path of the file and the value as a * calculated hash. diff --git a/src/OpenColorIO/Platform.h b/src/OpenColorIO/Platform.h index 1fcf019956..67413c76f3 100644 --- a/src/OpenColorIO/Platform.h +++ b/src/OpenColorIO/Platform.h @@ -99,7 +99,7 @@ void OpenInputFileStream(std::ifstream & stream, const char * filename, std::ios // Returns the specified filename string as a UTF16 wstring for Windows. const std::wstring filenameToUTF(const std::string & str); #else - // Return the specified filename string as is for Unix-like OS. + // Returns the specified filename string as is for Unix-like OS. const std::string filenameToUTF(const std::string & str); #endif diff --git a/src/apps/ocioarchive/main.cpp b/src/apps/ocioarchive/main.cpp index 02bf8dcb9c..bf222fc8aa 100644 --- a/src/apps/ocioarchive/main.cpp +++ b/src/apps/ocioarchive/main.cpp @@ -64,14 +64,14 @@ int main(int argc, const char **argv) " # Extract myarchive.ocioz into new directory named myarchive\n" " ocioarchive --extract myarchive.ocioz\n\n" " # Extract myarchive.ocioz into new directory named ocio_config\n" - " ocioarchive --extract myarchive.ocioz --newdir ocio_config\n\n" + " ocioarchive --extract myarchive.ocioz --dir ocio_config\n\n" " # List the files inside myarchive.ocioz\n" " ocioarchive --list myarchive.ocioz\n", "%*", parse_end_args, "", "", "Options:", "--iconfig %s", &configFilename, "Config to archive (takes precedence over $OCIO)", "--extract", &extract, "Extract an OCIOZ config archive", - "--dir %s", &extractDestination, "Path where to extract the files (folder are created if missing)", + "--dir %s", &extractDestination, "Path where to extract the files (folders are created if missing)", "--list", &list, "List the files inside an archive without extracting it", "--help", &help, "Display the help and exit", "-h", &help, "Display the help and exit", diff --git a/src/bindings/python/PyConfigIOProxy.cpp b/src/bindings/python/PyConfigIOProxy.cpp index 544b81bff2..2c9aac83fd 100644 --- a/src/bindings/python/PyConfigIOProxy.cpp +++ b/src/bindings/python/PyConfigIOProxy.cpp @@ -22,29 +22,29 @@ struct PyConfigIOProxy : ConfigIOProxy std::vector getLutData(const char * filepath) const override { PYBIND11_OVERRIDE_PURE( - std::vector, - ConfigIOProxy, // Parent class - getLutData, // Name of function in C++ (must match Python name) - filepath // Argument(s) + std::vector, // Return type. + ConfigIOProxy, // Parent class. + getLutData, // Name of function in C++ (must match Python name). + filepath // Argument. ); } std::string getConfigData() const override { PYBIND11_OVERRIDE_PURE( - std::string, - ConfigIOProxy, // Parent class - getConfigData, // Name of function in C++ (must match Python name) + std::string, // Return type. + ConfigIOProxy, // Parent class. + getConfigData, // Name of function in C++ (must match Python name). ); } std::string getFastLutFileHash(const char * filepath) const override { PYBIND11_OVERRIDE_PURE( - std::string, // Return type - ConfigIOProxy, // Parent class - getFastLutFileHash, // Name of function in C++ (must match Python name) - filepath // Argument(s) + std::string, // Return type. + ConfigIOProxy, // Parent class. + getFastLutFileHash, // Name of function in C++ (must match Python name). + filepath // Argument. ); } }; diff --git a/tests/cpu/OCIOZArchive_tests.cpp b/tests/cpu/OCIOZArchive_tests.cpp index 289048f829..4906618b31 100644 --- a/tests/cpu/OCIOZArchive_tests.cpp +++ b/tests/cpu/OCIOZArchive_tests.cpp @@ -45,7 +45,7 @@ struct DirectoryCreationGuard OCIO_ADD_TEST(OCIOZArchive, is_config_archivable) { - // This test primarly tests the isArchivable method from the Config object + // This test primarily tests the isArchivable method from the Config object const std::string CONFIG = "ocio_profile_version: 2\n" @@ -192,7 +192,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable) * Legal scenarios */ - // Valid FileTransfrom path. + // Valid FileTransform path. addFTAndTestIsArchivable("luts", true); addFTAndTestIsArchivable(R"(luts/myluts1)", true); addFTAndTestIsArchivable(R"(luts\myluts1)", true); diff --git a/tests/cpu/UnitTestUtils.h b/tests/cpu/UnitTestUtils.h index 07cd7a1372..0fa467e9d0 100644 --- a/tests/cpu/UnitTestUtils.h +++ b/tests/cpu/UnitTestUtils.h @@ -128,17 +128,17 @@ struct EnvironmentVariableGuard }; /** - * @brief Create a Temporary Directory + * \brief Create a Temporary Directory * - * @param name Name of the directory - * @return std::string Full path to the directory + * \param name Name of the directory + * \return Full path to the directory */ std::string CreateTemporaryDirectory(const std::string & name); /** - * @brief Remove the directory specified in the path. + * \brief Remove the directory specified in the path. * - * @param directoryPath Path to the directory + * \param directoryPath Path to the directory */ void RemoveTemporaryDirectory(const std::string & directoryPath); diff --git a/tests/python/ConfigTest.py b/tests/python/ConfigTest.py index c7cebf03d7..0c53947000 100644 --- a/tests/python/ConfigTest.py +++ b/tests/python/ConfigTest.py @@ -990,7 +990,7 @@ def test_create_from_archive(self): # Simple check on the number of color spaces in the test config. self.assertEqual(len(config.getColorSpaceNames()), 13) - # Simple test to exercice ConfigIOProxy. + # Simple test to exercise ConfigIOProxy. processor = config.getProcessor("plain_lut1_cs", "shot1_lut1_cs") processor.getDefaultCPUProcessor() @@ -1006,7 +1006,7 @@ def test_create_from_archive(self): # Simple check on the number of color spaces in the test config. self.assertEqual(len(config.getColorSpaceNames()), 13) - # Simple test to exercice ConfigIOProxy. + # Simple test to exercise ConfigIOProxy. processor = config.getProcessor("plain_lut1_cs", "shot1_lut1_cs") processor.getDefaultCPUProcessor() @@ -1028,7 +1028,7 @@ def test_create_from_archive(self): with self.assertRaises(OCIO.Exception): OCIO.Config.CreateFromFile(ocioz_file) - # Missing LUT files but contains LUTs files. + # Missing LUT files but contains config file. ocioz_file = os.path.normpath( os.path.join( TEST_DATAFILES_DIR, 'configs', 'ocioz_archive_configs', 'config_missing_luts.ocioz' @@ -1146,7 +1146,7 @@ def lutExists(filepath): # Simple check on the number of color spaces in the test config. self.assertEqual(len(config.getColorSpaceNames()), 3) - # Simple test to exercice ConfigIOProxy. + # Simple test to exercise ConfigIOProxy. processor = config.getProcessor("c1", "c2") processor.getDefaultCPUProcessor() From 0788e9e5fa2db0ad194df62adf75c45424c0d82a Mon Sep 17 00:00:00 2001 From: Cedrik Fuoco Date: Mon, 24 Oct 2022 17:18:52 -0400 Subject: [PATCH 7/8] - Changing ci workflow in order to install cmake version 3.13.3 for Linux (matrix.vfx-cy == '2019'). Signed-off-by: Cedrik Fuoco --- .github/workflows/ci_workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci_workflow.yml b/.github/workflows/ci_workflow.yml index d911e869d3..890021741b 100644 --- a/.github/workflows/ci_workflow.yml +++ b/.github/workflows/ci_workflow.yml @@ -252,6 +252,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + # minizip-ng requires CMake 3.13+ but VFX2019 image ships with 3.12 + - name: Upgrade VFX2019 CMake + run: pip install cmake==3.13.3 + if: matrix.vfx-cy == '2019' - name: Install docs env run: share/ci/scripts/linux/yum/install_docs_env.sh if: matrix.build-docs == 'ON' From d7e0737a337ec2474a6ddbbc4ab5a417663a12e7 Mon Sep 17 00:00:00 2001 From: Cedrik Fuoco Date: Tue, 25 Oct 2022 11:42:05 -0400 Subject: [PATCH 8/8] - Fixing issue with the install location of zlib library - Removing unused CMAKE variables from ZLIB_CMAKE_ARGS Signed-off-by: Cedrik Fuoco --- share/cmake/modules/Findzlib.cmake | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/share/cmake/modules/Findzlib.cmake b/share/cmake/modules/Findzlib.cmake index a305939da4..546cea341a 100644 --- a/share/cmake/modules/Findzlib.cmake +++ b/share/cmake/modules/Findzlib.cmake @@ -77,8 +77,12 @@ if(NOT ZLIB_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) set(_ZLIB_LIB_SUFFIX "d") endif() + # ZLIB doesn't use CMAKE_INSTALL_LIBDIR + # It is hardcoded to ${CMAKE_INSTALL_PREFIX}/lib + # https://github.com/madler/zlib/blob/master/CMakeLists.txt#L9 + set(_ZLIB_INSTALL_LIBDIR "lib") set(ZLIB_LIBRARIES - "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_ZLIB_STATIC_LIB_NAME}${_ZLIB_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + "${_EXT_DIST_ROOT}/${_ZLIB_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_ZLIB_STATIC_LIB_NAME}${_ZLIB_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") if(_ZLIB_TARGET_CREATE) set(ZLIB_CMAKE_ARGS @@ -90,10 +94,6 @@ if(NOT ZLIB_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} -DCMAKE_INSTALL_MESSAGE=${CMAKE_INSTALL_MESSAGE} -DCMAKE_INSTALL_PREFIX=${_EXT_DIST_ROOT} - -DCMAKE_INSTALL_BINDIR=${CMAKE_INSTALL_BINDIR} - -DCMAKE_INSTALL_DATADIR=${CMAKE_INSTALL_DATADIR} - -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} - -DCMAKE_INSTALL_INCLUDEDIR=${CMAKE_INSTALL_INCLUDEDIR} -DCMAKE_OBJECT_PATH_MAX=${CMAKE_OBJECT_PATH_MAX} ) @@ -146,7 +146,7 @@ if(NOT ZLIB_FOUND AND NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL NONE) ZLIB_install zlib_remove_dll COMMENT "Remove zlib.lib and zlib.dll, leaves only zlibstatic.lib" DEPENDEES install - COMMAND ${CMAKE_COMMAND} -E remove -f ${_EXT_DIST_ROOT}/lib/zlib.lib ${_EXT_DIST_ROOT}/bin/zlib.dll + COMMAND ${CMAKE_COMMAND} -E remove -f ${_EXT_DIST_ROOT}/${_ZLIB_INSTALL_LIBDIR}/zlib.lib ${_EXT_DIST_ROOT}/bin/zlib.dll ) add_library(ZLIB::ZLIB STATIC IMPORTED GLOBAL)