Skip to content

[tmva][sofie] Generate ONNX models and refs instead of binaries in repo #22799

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
guitargeek wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[tmva][sofie] Generate ONNX models and refs instead of binaries in repo #22799

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 93 additions & 58 deletions tmva/sofie/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,56 +10,107 @@
############################################################################


# Look for needed Python modules
ROOT_FIND_PYTHON_MODULE(torch)

# onnx 1.19.0 has a bug that makes this version unusable:
# https://github.com/onnx/onnx/issues/7249
# 1.19.1 instead could be used
# In that case, we have to disable the "TestSofieModels" test,
# which imports onnx indirectly via torch.onnx
ROOT_FIND_PYTHON_MODULE(onnx)
if (ROOT_ONNX_FOUND AND DEFINED ROOT_ONNX_VERSION)
if(ROOT_ONNX_VERSION VERSION_EQUAL "1.19.0")
message(WARNING "Found broken onnx version ${ROOT_ONNX_VERSION} (see https://github.com/onnx/onnx/issues/7249). Some TMVA SOFIE tests will be disabled.")
set(broken_onnx TRUE)
endif()
endif()

# The ONNX input models are not checked into the repository: they are created
# by generate_input_models.py, which runs as the SofieGenerateModels_ONNX test
# that the other ONNX tests depend on. Set ONNX_MODELS_DIR to use pre-existing
# models from a custom directory instead.
if (NOT ONNX_MODELS_DIR)
set(ONNX_MODELS_DIR input_models)
if (ROOT_ONNX_FOUND AND NOT broken_onnx)
set(ONNX_MODELS_DIR ${CMAKE_CURRENT_BINARY_DIR}/input_models)
set(generate_onnx_models TRUE)
# Get the model names from the generator script (--list does not require
# the onnx module, only a Python interpreter).
execute_process(
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_input_models.py --list
OUTPUT_VARIABLE _onnx_model_names
OUTPUT_STRIP_TRAILING_WHITESPACE
RESULT_VARIABLE _onnx_list_status)
if (NOT _onnx_list_status EQUAL 0)
message(FATAL_ERROR "Failed to list the SOFIE ONNX test models with generate_input_models.py")
endif()
string(REPLACE "\n" ";" ONNX_MODEL_NAMES "${_onnx_model_names}")
else()
message(WARNING "The onnx Python module is not usable: the ONNX input models for the TMVA SOFIE tests cannot be generated and the corresponding tests are disabled.")
endif()
else()
get_filename_component(ONNX_MODELS_DIR ${ONNX_MODELS_DIR} ABSOLUTE)
file(GLOB ONNX_FILES "${ONNX_MODELS_DIR}/*.onnx")
set(ONNX_MODEL_NAMES "")
foreach(onnx_file ${ONNX_FILES})
get_filename_component(fname ${onnx_file} NAME_WE)
list(APPEND ONNX_MODEL_NAMES ${fname})
endforeach()
endif()

# Finding .onnx files to be parsed and creating the appropriate code to
# parse all file. It is much faster to combine all parsing in a single executable
# which will avoid initialization time (especially when using ROOT)
set(CAPTURE_STR "EmitModel( \"@1\", \"@2\");")
set(ALL_CAPTURES "")
# Finding .onnx files to be parsed and creating the appropriate command
file(GLOB ONNX_FILES "${ONNX_MODELS_DIR}/*.onnx")
foreach(onnx_file ${ONNX_FILES})
get_filename_component(fname ${onnx_file} NAME_WE)
get_filename_component(fdir ${onnx_file} DIRECTORY)
string(REPLACE "@1" ${onnx_file} cap ${CAPTURE_STR})
string(REPLACE "@2" ${fname} cap ${cap})
list(APPEND ALL_CAPTURES ${cap})
endforeach()
string(REPLACE ";" ";\n" EMIT_CAPTURES "${ALL_CAPTURES}")
configure_file(EmitFromONNX.cxx.in EmitFromONNX_all.cxx @ONLY)

ROOTTEST_GENERATE_EXECUTABLE(emitFromONNX EmitFromONNX_all.cxx
LIBRARIES protobuf::libprotobuf ROOTTMVASofie ROOTTMVASofieParser
FIXTURES_SETUP sofie-compile-models-onnx-build)

# silence protobuf warnings seen in version 3.0 and 3.6. Not needed from protobuf version 3.17
target_compile_options(emitFromONNX PRIVATE -Wno-unused-parameter -Wno-array-bounds)

ROOTTEST_ADD_TEST(SofieCompileModels_ONNX
COMMAND ${CMAKE_COMMAND} -E env ROOTIGNOREPREFIX=1 ./emitFromONNX ${onnx_file} ${CMAKE_CURRENT_BINARY_DIR}/${fname}
FIXTURES_REQUIRED sofie-compile-models-onnx-build
FIXTURES_SETUP sofie-compile-models-onnx
)

# Creating a Google Test
if (BLAS_FOUND) # we need BLAS for compiling the models
ROOT_EXECUTABLE(TestCustomModelsFromONNX TestCustomModelsFromONNX.cxx
LIBRARIES Core GTest::gtest GTest::gtest_main
if (ONNX_MODELS_DIR)
# Creating the appropriate code to parse all ONNX model files. It is much
# faster to combine all parsing in a single executable which will avoid
# initialization time (especially when using ROOT)
set(CAPTURE_STR "EmitModel( \"@1\", \"@2\");")
set(ALL_CAPTURES "")
foreach(fname ${ONNX_MODEL_NAMES})
string(REPLACE "@1" ${ONNX_MODELS_DIR}/${fname}.onnx cap ${CAPTURE_STR})
string(REPLACE "@2" ${fname} cap ${cap})
list(APPEND ALL_CAPTURES ${cap})
endforeach()
string(REPLACE ";" ";\n" EMIT_CAPTURES "${ALL_CAPTURES}")
configure_file(EmitFromONNX.cxx.in EmitFromONNX_all.cxx @ONLY)

ROOTTEST_GENERATE_EXECUTABLE(emitFromONNX EmitFromONNX_all.cxx
LIBRARIES protobuf::libprotobuf ROOTTMVASofie ROOTTMVASofieParser
FIXTURES_SETUP sofie-compile-models-onnx-build)

# silence protobuf warnings seen in version 3.0 and 3.6. Not needed from protobuf version 3.17
target_compile_options(emitFromONNX PRIVATE -Wno-unused-parameter -Wno-array-bounds)

set(compile_models_fixtures sofie-compile-models-onnx-build)
if (generate_onnx_models)
ROOTTEST_ADD_TEST(SofieGenerateModels_ONNX
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_input_models.py --outdir ${ONNX_MODELS_DIR}
FIXTURES_SETUP sofie-generate-models-onnx
)
list(APPEND compile_models_fixtures sofie-generate-models-onnx)
endif()

ROOTTEST_ADD_TEST(SofieCompileModels_ONNX
COMMAND ${CMAKE_COMMAND} -E env ROOTIGNOREPREFIX=1 ./emitFromONNX
FIXTURES_REQUIRED ${compile_models_fixtures}
FIXTURES_SETUP sofie-compile-models-onnx
)
ROOTTEST_ADD_TEST(TestCustomModelsFromONNX
EXEC ./TestCustomModelsFromONNX
FIXTURES_REQUIRED sofie-compile-models-onnx)

if (clad)
ROOT_EXECUTABLE(TestCladAutodiff TestCladAutodiff.cxx
# Creating a Google Test
if (BLAS_FOUND) # we need BLAS for compiling the models
ROOT_EXECUTABLE(TestCustomModelsFromONNX TestCustomModelsFromONNX.cxx
LIBRARIES Core GTest::gtest GTest::gtest_main
)
ROOTTEST_ADD_TEST(TestCladAutodiff
EXEC ./TestCladAutodiff
ROOTTEST_ADD_TEST(TestCustomModelsFromONNX
EXEC ./TestCustomModelsFromONNX
FIXTURES_REQUIRED sofie-compile-models-onnx)

if (clad)
ROOT_EXECUTABLE(TestCladAutodiff TestCladAutodiff.cxx
LIBRARIES Core GTest::gtest GTest::gtest_main
)
ROOTTEST_ADD_TEST(TestCladAutodiff
EXEC ./TestCladAutodiff
FIXTURES_REQUIRED sofie-compile-models-onnx)
endif()
endif()
endif()

Expand All @@ -70,22 +121,6 @@ if (BLAS_FOUND)
endif()
endif()

# Look for needed Python modules
ROOT_FIND_PYTHON_MODULE(torch)

# onnx 1.19.0 has a bug that makes this version unusable:
# https://github.com/onnx/onnx/issues/7249
# 1.19.1 instead could be used
# In that case, we have to disable the "TestSofieModels" test,
# which imports onnx indirectly via torch.onnx
ROOT_FIND_PYTHON_MODULE(onnx)
if (ROOT_ONNX_FOUND AND DEFINED ROOT_ONNX_VERSION)
if(ROOT_ONNX_VERSION VERSION_EQUAL "1.19.0")
message(WARNING "Found broken onnx version ${ROOT_ONNX_VERSION} (see https://github.com/onnx/onnx/issues/7249). Some TMVA SOFIE tests will be disabled.")
set(broken_onnx TRUE)
endif()
endif()

if (ROOT_TORCH_FOUND AND ROOT_ONNX_FOUND AND NOT broken_onnx)
configure_file(Conv1dModelGenerator.py Conv1dModelGenerator.py COPYONLY)
configure_file(Conv2dModelGenerator.py Conv2dModelGenerator.py COPYONLY)
Expand Down
18 changes: 4 additions & 14 deletions tmva/sofie/test/TestCladAutodiff.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ constexpr auto modelHeaderSuffix = "_FromONNX_unoptimized.hxx";
constexpr auto modelDataSuffix = "_FromONNX_unoptimized.dat";
#include "test_helpers.h"

#include "input_models/references/Linear_16.ref.hxx"

#include "gtest/gtest.h"

// Test differentiating a fully-connected neural network with Clad.
Expand All @@ -12,9 +10,9 @@ TEST(ONNXClad, Linear16)
{
constexpr float TOLERANCE = DEFAULT_TOLERANCE;

// Preparing the standard all-ones input
std::vector<float> input(1600);
std::fill_n(input.data(), input.size(), 1.0f);
SofieReference ref = readReference("Linear_16");
// Mutable copy: the numeric differentiation below perturbs the input values
std::vector<float> input = ref.f32("input0");

ASSERT_INCLUDE_AND_RUN(std::vector<float>, "Linear_16", input);

Expand Down Expand Up @@ -135,13 +133,5 @@ float Linear_16_wrapper_num_diff(TMVA_SOFIE_Linear_16::Session const &session, f
ADD_FAILURE() << "Further mismatches suppressed (total mismatches: " << mismatchCount << ")";
}

// Checking output size
EXPECT_EQ(output.size(), sizeof(Linear_16_ExpectedOutput::all_ones) / sizeof(float));

float *correct = Linear_16_ExpectedOutput::all_ones;

// Checking every output value, one by one
for (size_t i = 0; i < output.size(); ++i) {
EXPECT_LE(std::abs(output[i] - correct[i]), TOLERANCE);
}
expectNear(output, ref.f32("output0"), TOLERANCE);
}
Loading
Loading