[ML] Add a script to run each unit test separately

.buildkite/pipelines/build_linux.json.py

@@ -72,7 +72,6 @@ def main(args):
                   "RUN_TESTS": "true",
                   "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
                 },
-                "artifact_paths": "*/**/unittest/boost_test_results.junit",
                 "plugins": {
                   "test-collector#v1.2.0": {                                                              
                     "files": "*/*/unittest/boost_test_results.junit",

.buildkite/scripts/steps/build_and_test.sh

@@ -106,7 +106,7 @@ fi
 if [[ -z "$CPP_CROSS_COMPILE" ]] ; then 
   OS=$(uname -s | tr "A-Z" "a-z")
   TEST_RESULTS_ARCHIVE=${OS}-${HARDWARE_ARCH}-unit_test_results.tgz
-  find . -path  "*/**/ml_test_*.out" -o -path "*/**/*.junit" | xargs tar cvzf ${TEST_RESULTS_ARCHIVE}
+  find . \( -path "*/**/ml_test_*.out" -o -path "*/**/*.junit" \) -print0 | tar czf ${TEST_RESULTS_ARCHIVE} --null -T -
   buildkite-agent artifact upload "${TEST_RESULTS_ARCHIVE}"
 fi
 

3rd_party/3rd_party.cmake

@@ -25,6 +25,10 @@ if(NOT INSTALL_DIR)
   message(FATAL_ERROR "INSTALL_DIR not specified")
 endif()
 
+STRING(REPLACE "//" "/" INSTALL_DIR ${INSTALL_DIR})
+
+message(STATUS "3rd_party: CMAKE_CXX_COMPILER_VERSION_MAJOR=${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
+
 string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} HOST_SYSTEM_NAME)
 message(STATUS "3rd_party: HOST_SYSTEM_NAME=${HOST_SYSTEM_NAME}")
 
@@ -43,7 +47,9 @@ set(ARCH ${HOST_SYSTEM_PROCESSOR})
 if ("${HOST_SYSTEM_NAME}" STREQUAL "darwin")
   message(STATUS "3rd_party: Copying macOS 3rd party libraries")
   set(BOOST_LOCATION "/usr/local/lib")
-  set(BOOST_COMPILER "clang")
+  set(BOOST_COMPILER "clang-darwin${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
+  message(STATUS "3rd_party: BOOST_COMPILER=${BOOST_COMPILER}")
+
   if( "${ARCH}" STREQUAL "x86_64" )
     set(BOOST_ARCH "x64")
   else()
@@ -63,7 +69,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux")
   if(NOT DEFINED ENV{CPP_CROSS_COMPILE} OR "$ENV{CPP_CROSS_COMPILE}" STREQUAL "")
     message(STATUS "3rd_party: NOT cross compiling. Copying Linux 3rd party libraries")
     set(BOOST_LOCATION "/usr/local/gcc133/lib")
-    set(BOOST_COMPILER "gcc")
+    set(BOOST_COMPILER "gcc${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
     if( "${ARCH}" STREQUAL "aarch64" )
       set(BOOST_ARCH "a64")
     else()
@@ -93,7 +99,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux")
     message(STATUS "3rd_party: Cross compile for macosx: Copying macOS 3rd party libraries")
     set(SYSROOT "/usr/local/sysroot-x86_64-apple-macosx10.14")
     set(BOOST_LOCATION "${SYSROOT}/usr/local/lib")
-    set(BOOST_COMPILER "clang")
+    set(BOOST_COMPILER "clang-darwin${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
     set(BOOST_EXTENSION "mt-x64-1_86.dylib")
     set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
     set(XML_LOCATION)
@@ -108,7 +114,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux")
     message(STATUS "3rd_party: Cross compile for linux-aarch64: Copying Linux 3rd party libraries")
     set(SYSROOT "/usr/local/sysroot-$ENV{CPP_CROSS_COMPILE}-linux-gnu")
     set(BOOST_LOCATION "${SYSROOT}/usr/local/gcc133/lib")
-    set(BOOST_COMPILER "gcc")
+    set(BOOST_COMPILER "gcc${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
     if("$ENV{CPP_CROSS_COMPILE}" STREQUAL "aarch64")
       set(BOOST_ARCH "a64")
     else()
@@ -188,6 +194,9 @@ function(install_libs _target _source_dir _prefix _postfix)
 
   set(LIBRARIES ${ARGN})
 
+  message(STATUS "_target=${_target} _source_dir=${_source_dir} _prefix=${_prefix} _postfix=${_postfix} LIBRARIES=${LIBRARIES}")
+
+
   file(GLOB _LIBS ${_source_dir}/*${_prefix}*${_postfix})
 
   if(_LIBS)
@@ -219,7 +228,7 @@ function(install_libs _target _source_dir _prefix _postfix)
             endif()
             file(CHMOD ${INSTALL_DIR}/${_LIB} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
           else()
-            file(COPY ${_RESOLVED_PATH} DESTINATION ${INSTALL_DIR})
+            file(COPY ${_RESOLVED_PATH} DESTINATION "${INSTALL_DIR}")
             file(CHMOD ${INSTALL_DIR}/${_RESOLVED_LIB} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
           endif()
         endforeach()

3rd_party/CMakeLists.txt

@@ -22,7 +22,7 @@ add_custom_target(licenses ALL
 # as part of the CMake configuration step - avoiding
 # the need for it to be done on every build
 execute_process(
-    COMMAND ${CMAKE_COMMAND} -DINSTALL_DIR=${INSTALL_DIR} -P ./3rd_party.cmake
+    COMMAND ${CMAKE_COMMAND} -DINSTALL_DIR=${INSTALL_DIR} -DCMAKE_CXX_COMPILER_VERSION_MAJOR=${CMAKE_CXX_COMPILER_VERSION_MAJOR} -P ./3rd_party.cmake
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 

CONTRIBUTING.md

@@ -92,6 +92,17 @@ Note that we configure the build to be of type `RelWithDebInfo` in order to obta
     1. It is also possible to control the behaviour of the test framework by passing any other arbitrary flags via the
        `TEST_FLAGS` environment variable , e.g. `TEST_FLAGS="--random" cmake --build cmake-build-relwithdebinfo -t test`
        (use TEST_FLAGS="--help" to see the full list).
+1. On Linux and maOS it is possible to run individual tests within a Boost Test suite in separate processes.
+   1. This is convenient for several reasons:
+      1. Isolation: Prevent one test's failures (e.g., memory corruption, unhandled exceptions) from affecting subsequent tests.
+      1. Resource Management: Clean up of resources (memory, file handles, network connections) between tests more effectively.
+      1. Stability: Improve the robustness of test suites, especially for long-running or complex tests.
+      1. Parallelization: A means to run individual test cases in parallel has been provided:
+         1. For all tests associated with a library or executable, e.g.
+         `cmake --build cmake-build-relwithdebinfo -j 8 -t test_api_individually`
+         1. For all tests in the `ml-cpp` repo:
+         `cmake --build cmake-build-relwithdebinfo -j 8 -t test_individually`
+   1. **Care should be taken that tests don't modify common resources.**
 1. As a convenience, there exists a `precommit` target that both formats the code and runs the entire test suite, e.g.
     1. `./gradlew precommit`
     1. `cmake --build cmake-build-relwithdebinfo -j 8 -t precommit`

cmake/compiler/clang.cmake

@@ -16,7 +16,7 @@ set(CMAKE_RANLIB  "ranlib")
 set(CMAKE_STRIP  "strip")
 
 
-list(APPEND ML_C_FLAGS 
+list(APPEND ML_C_FLAGS
   ${CROSS_FLAGS}
   ${ARCHCFLAGS}
   "-fstack-protector"

cmake/functions.cmake

@@ -392,6 +392,13 @@ function(ml_add_test_executable _target)
       COMMENT "Running test: ml_test_${_target}"
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
       )
+
+    add_custom_target(test_${_target}_individually
+      DEPENDS ml_test_${_target}
+      COMMAND ${CMAKE_SOURCE_DIR}/run_tests_as_seperate_processes.sh ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR} test_${_target}
+      COMMENT "Running test: ml_test_${_target}_individually"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    )
   endif()
 endfunction()
 
@@ -420,8 +427,10 @@ function(ml_add_test _directory _target)
   add_subdirectory(../${_directory} ${_directory})
   list(APPEND ML_BUILD_TEST_DEPENDS ml_test_${_target})
   list(APPEND ML_TEST_DEPENDS test_${_target})
+  list(APPEND ML_TEST_INDIVIDUALLY_DEPENDS test_${_target}_individually)
   set(ML_BUILD_TEST_DEPENDS ${ML_BUILD_TEST_DEPENDS} PARENT_SCOPE)
   set(ML_TEST_DEPENDS ${ML_TEST_DEPENDS} PARENT_SCOPE)
+  set(ML_TEST_INDIVIDUALLY_DEPENDS ${ML_TEST_INDIVIDUALLY_DEPENDS} PARENT_SCOPE)
 endfunction()
 
 

cmake/test-runner.cmake

@@ -9,30 +9,44 @@
 # limitation.
 #
 
-if(TEST_NAME STREQUAL "ml_test_seccomp")
-  execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS} --logger=HRF,all --report_format=HRF --show_progress=no --no_color_output  OUTPUT_FILE ${TEST_DIR}/${TEST_NAME}.out ERROR_FILE ${TEST_DIR}/${TEST_NAME}.out RESULT_VARIABLE TEST_SUCCESS)
-else()
-  # Turn the TEST_FLAGS environment variable into a CMake list variable
-  if (DEFINED ENV{TEST_FLAGS} AND NOT "$ENV{TEST_FLAGS}" STREQUAL "")
-    string(REPLACE " " ";" TEST_FLAGS $ENV{TEST_FLAGS})
-  endif()
+execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f ${TEST_DIR}/*.out)
+execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f ${TEST_DIR}/*.failed)
+execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f boost_test_results*.xml)
+execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f boost_test_results*.junit)
 
-  # Special case for specifying a subset of tests to run (can be regex)
-  if (DEFINED ENV{TESTS} AND NOT "$ENV{TESTS}" STREQUAL "")
-    set(TESTS "--run_test=$ENV{TESTS}")
-  endif()
+# Turn the TEST_FLAGS environment variable into a CMake list variable
+if (DEFINED ENV{TEST_FLAGS} AND NOT "$ENV{TEST_FLAGS}" STREQUAL "")
+  string(REPLACE " " ";" TEST_FLAGS $ENV{TEST_FLAGS})
+endif()
+
+set(SAFE_TEST_NAME "")
+set(TESTS "")
+# Special case for specifying a subset of tests to run (can be regex)
+if (DEFINED ENV{TESTS} AND NOT "$ENV{TESTS}" STREQUAL "")
+  set(TESTS "--run_test=$ENV{TESTS}")
+  string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" SAFE_TEST_NAME "$ENV{TESTS}")
+  set(SAFE_TEST_NAME "_${SAFE_TEST_NAME}")
+endif()
+
+string(REPLACE "boost_test_results" "boost_test_results${SAFE_TEST_NAME}" BOOST_TEST_OUTPUT_FORMAT_FLAGS "$ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS}")
+set(OUTPUT_FILE "${TEST_DIR}/${TEST_NAME}${SAFE_TEST_NAME}.out")
+set(FAILED_FILE "${TEST_DIR}/${TEST_NAME}${SAFE_TEST_NAME}.failed")
 
-  # If any special command line args are present run the tests in the foreground
-  if (DEFINED TEST_FLAGS OR DEFINED TESTS)
-    message(STATUS "executing process ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS}")
-    execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS} RESULT_VARIABLE TEST_SUCCESS)
+# If env var RUN_BOOST_TESTS_IN_FOREGROUND is defined run the tests in the foreground
+if(TEST_NAME STREQUAL "ml_test_seccomp")
+  execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS} --logger=HRF,all --report_format=HRF --show_progress=no --no_color_output  OUTPUT_FILE ${OUTPUT_FILE} ERROR_FILE ${OUTPUT_FILE} RESULT_VARIABLE TEST_SUCCESS)
+else()
+  if(NOT DEFINED ENV{RUN_BOOST_TESTS_IN_FOREGROUND})
+    execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS} --no_color_output  OUTPUT_FILE ${OUTPUT_FILE} ERROR_FILE ${OUTPUT_FILE}  RESULT_VARIABLE TEST_SUCCESS)
   else()
-    execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} $ENV{TEST_FLAGS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS}
-      --no_color_output  OUTPUT_FILE ${TEST_DIR}/${TEST_NAME}.out ERROR_FILE ${TEST_DIR}/${TEST_NAME}.out RESULT_VARIABLE TEST_SUCCESS)
+    execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS}  RESULT_VARIABLE TEST_SUCCESS)
   endif()
 endif()
 
 if (NOT TEST_SUCCESS EQUAL 0)
-  execute_process(COMMAND ${CMAKE_COMMAND} -E cat ${TEST_DIR}/${TEST_NAME}.out)
-  file(WRITE "${TEST_DIR}/${TEST_NAME}.failed" "")
+  if (EXISTS ${TEST_DIR}/${TEST_NAME})
+    execute_process(COMMAND ${CMAKE_COMMAND} -E cat ${OUTPUT_FILE})
+    file(WRITE "${FAILED_FILE}" "")
+  endif()
 endif()
+

dev-tools/docker/docker_entrypoint.sh

@@ -66,6 +66,6 @@ if [ "x$1" = "x--test" ] ; then
     # failure is the unit tests, and then the detailed test results can be
     # copied from the image
     echo passed > build/test_status.txt
-    cmake --build cmake-build-docker ${CMAKE_VERBOSE} -j`nproc` -t test || echo failed > build/test_status.txt
+    cmake --build cmake-build-docker ${CMAKE_VERBOSE} -j $(nproc) -t test_individually || echo failed > build/test_status.txt
 fi
 

dev-tools/docker_test.sh

@@ -92,7 +92,10 @@ do
     # Using tar to copy the build and test artifacts out of the container seems
     # more reliable than docker cp, and also means the files end up with the
     # correct uid/gid
-    docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c "find . $EXTRACT_FIND | xargs tar cf - $EXTRACT_EXPLICIT && sleep 30" | tar xvf -
+    docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c "find . \( $EXTRACT_FIND \) -print0 |  tar cf - $EXTRACT_EXPLICIT --null -T -" | tar xvf -
+    if [ $? != 0 ]; then
+      echo "Copying build and test artifacts from docker container failed"
+    fi
     docker rmi --force $TEMP_TAG
     # The image build is set to return zero (i.e. succeed as far as Docker is
     # concerned) when the only problem is that the unit tests fail, as this

lib/api/unittest/CMultiFileDataAdderTest.cc

@@ -37,6 +37,8 @@
 #include <ios>
 #include <iterator>
 #include <memory>
+#include <random> // For random number generation facilities
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -100,7 +102,16 @@ void detectorPersistHelper(const std::string& configFileName,
 
     // Persist the detector state to file(s)
 
-    std::string baseOrigOutputFilename(ml::test::CTestTmpDir::tmpDir() + "/orig");
+    // Create a random number to use to generate a unique file name for each test
+    // this allows tests to be run successfully in parallel
+    std::random_device rd;
+    std::mt19937 gen(rd());
+    std::uniform_int_distribution<> distrib(1, 100);
+    std::ostringstream oss;
+    oss << distrib(gen);
+
+    std::string baseOrigOutputFilename(ml::test::CTestTmpDir::tmpDir() +
+                                       "/orig_" + oss.str());
     {
         // Clean up any leftovers of previous failures
         boost::filesystem::path origDir(baseOrigOutputFilename);
@@ -152,7 +163,8 @@ void detectorPersistHelper(const std::string& configFileName,
 
     // Finally, persist the new detector state to a file
 
-    std::string baseRestoredOutputFilename(ml::test::CTestTmpDir::tmpDir() + "/restored");
+    std::string baseRestoredOutputFilename(ml::test::CTestTmpDir::tmpDir() +
+                                           "/restored_" + oss.str());
     {
         // Clean up any leftovers of previous failures
         boost::filesystem::path restoredDir(baseRestoredOutputFilename);

lib/core/unittest/CLoggerTest.cc

@@ -49,20 +49,21 @@ class CTestFixture {
     }
 };
 
-std::function<void()> makeReader(std::ostringstream& loggedData) {
-    return [&loggedData] {
+std::function<void()> makeReader(std::ostringstream& loggedData, const std::string& pipeName) {
+    return [&loggedData, &pipeName]() {
+
         for (std::size_t attempt = 1; attempt <= 100; ++attempt) {
             // wait a bit so that pipe has been created
             std::this_thread::sleep_for(std::chrono::milliseconds(50));
-            std::ifstream strm(TEST_PIPE_NAME);
+            std::ifstream strm(pipeName);
             if (strm.is_open()) {
                 std::copy(std::istreambuf_iterator<char>(strm),
                           std::istreambuf_iterator<char>(),
                           std::ostreambuf_iterator<char>(loggedData));
                 return;
             }
         }
-        BOOST_FAIL("Failed to connect to logging pipe within a reasonable time");
+        BOOST_FAIL("Failed to connect to logging pipe " + pipeName + " within a reasonable time");
     };
 }
 
@@ -204,12 +205,13 @@ BOOST_FIXTURE_TEST_CASE(testNonAsciiJsonLogging, CTestFixture) {
                      "Non-iso8859-15: 编码 test", "surrogate pair: 𐐷 test"};
 
     std::ostringstream loggedData;
-    std::thread reader(makeReader(loggedData));
+    const std::string& pipeName = std::string{TEST_PIPE_NAME} + "_testNonAsciiJsonLogging";
+    std::thread reader(makeReader(loggedData, pipeName));
 
     ml::core::CLogger& logger = ml::core::CLogger::instance();
     // logger might have been reconfigured in previous tests, so reset and reconfigure it
     logger.reset();
-    logger.reconfigure(TEST_PIPE_NAME, "");
+    logger.reconfigure(pipeName, "");
 
     for (const auto& m : messages) {
         LOG_INFO(<< m);
@@ -225,14 +227,16 @@ BOOST_FIXTURE_TEST_CASE(testNonAsciiJsonLogging, CTestFixture) {
 BOOST_FIXTURE_TEST_CASE(testWarnAndErrorThrottling, CTestFixture) {
 
     std::ostringstream loggedData;
-    std::thread reader{makeReader(loggedData)};
+    const std::string& pipeName = std::string{TEST_PIPE_NAME} + "_testWarnAndErrorThrottling";
+
+    std::thread reader{makeReader(loggedData, pipeName)};
 
     TStrVec messages{"Warn should only be seen once", "Error should only be seen once"};
 
     ml::core::CLogger& logger = ml::core::CLogger::instance();
     // logger might have been reconfigured in previous tests, so reset and reconfigure it
     logger.reset();
-    logger.reconfigure(TEST_PIPE_NAME, "");
+    logger.reconfigure(pipeName, "");
 
     for (std::size_t i = 0; i < 10; ++i) {
         LOG_WARN(<< messages[0]);