Usage of ccache in the test suite (#364)

* test: add -cache option to the test script to accelerate the build

* build: Add option -DIdefix_PROBLEM_DIR to allow using the standard out-of-source build way of cmake

---------

Co-authored-by: Geoffroy Lesur <geoffroy.lesur@univ-grenoble-alpes.fr>

Author: svalat

.github/workflows/idefix-ci.yml

@@ -28,15 +28,15 @@ jobs:
     name: CPU Jobs (intel OneApi)
     uses: ./.github/workflows/idefix-ci-jobs.yml
     with:
-      TESTME_OPTIONS: -intel -Werror
+      TESTME_OPTIONS: -intel -Werror -ccache
       IDEFIX_COMPILER: icc
 
   gcc-jobs:
     needs: Linter
     name: CPU Jobs (gcc)
     uses: ./.github/workflows/idefix-ci-jobs.yml
     with:
-      TESTME_OPTIONS: -Werror
+      TESTME_OPTIONS: -Werror -ccache
       IDEFIX_COMPILER: gcc
 
   cuda-jobs:

CMakeLists.txt

@@ -16,6 +16,7 @@ option(Idefix_DEBUG "Enable Idefix debug features (makes the code very slow)" OF
 option(Idefix_RUNTIME_CHECKS "Enable runtime sanity checks" OFF)
 option(Idefix_WERROR "Treat compiler warnings as errors" OFF)
 option(Idefix_PYTHON "Enable python bindings (requires pybind11)" OFF)
+set(Idefix_PROBLEM_DIR "${CMAKE_BINARY_DIR}" CACHE STRING "Problem directory to build for.")
 set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag")
 set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file")
 option(Idefix_CUSTOM_EOS "Use custom equation of state" OFF)
@@ -34,7 +35,6 @@ set_property(CACHE Idefix_PRECISION PROPERTY STRINGS Double Single)
 set(Idefix_LOOP_PATTERN "Default" CACHE STRING "Loop pattern for idefix_for")
 set_property(CACHE Idefix_LOOP_PATTERN PROPERTY STRINGS Default SIMD Range MDRange TeamPolicy TeamPolicyInnerVector)
 
-
 # load git revision tools
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
 include(GetGitRevisionDescription)
@@ -51,7 +51,7 @@ if(Kokkos_ENABLE_CUDA)
 endif()
 
 # Add kokkos CMAKE files (required early since these set compiler options)
-add_subdirectory(src/kokkos build/kokkos)
+add_subdirectory(src/kokkos ${CMAKE_BINARY_DIR}/build/kokkos)
 include_directories(${Kokkos_INCLUDE_DIRS_RET})
 
 # Add Idefix CXX Flags
@@ -75,20 +75,22 @@ endif()
 add_executable(idefix)
 add_subdirectory(src build)
 
-if(EXISTS ${PROJECT_BINARY_DIR}/setup.cpp)
-  target_sources(idefix PUBLIC ${PROJECT_BINARY_DIR}/setup.cpp)
+# make absolute
+get_filename_component(Idefix_PROBLEM_DIR_ABS ${Idefix_PROBLEM_DIR} ABSOLUTE BASE_DIR ${PROJECT_BINARY_DIR})
+
+if(EXISTS ${Idefix_PROBLEM_DIR_ABS}/setup.cpp)
+  target_sources(idefix PUBLIC ${Idefix_PROBLEM_DIR_ABS}/setup.cpp)
 else()
   message(WARNING "No specific setup.cpp found in the problem directory (this message can be ignored if using python to define your problem)")
 endif()
 
 # If a CMakeLists.txt is in the problem dir (for problem-specific source files)
 # then read it
-if(EXISTS ${PROJECT_BINARY_DIR}/CMakeLists.txt)
-  message(STATUS "Including problem-specific CMakeLists in '${PROJECT_BINARY_DIR}'")
-  add_subdirectory(${PROJECT_BINARY_DIR} build/setup)
+if(EXISTS ${Idefix_PROBLEM_DIR_ABS}/CMakeLists.txt)
+  message(STATUS "Including problem-specific CMakeLists in '${Idefix_PROBLEM_DIR_ABS}'")
+  add_subdirectory(${Idefix_PROBLEM_DIR_ABS} build/setup)
 endif()
 
-
 if(Idefix_MHD)
   add_compile_definitions("MHD=YES")
 else()
@@ -210,7 +212,7 @@ if(${Idefix_PRECISION} STREQUAL "Single")
 endif()
 
 target_include_directories(idefix PUBLIC
-                           "${PROJECT_BINARY_DIR}"
+                           "${Idefix_PROBLEM_DIR_ABS}"
                            )
 target_include_directories(idefix PUBLIC
                            src/kokkos/core/src
@@ -249,6 +251,7 @@ message(STATUS "    Python: ${Idefix_PYTHON}")
 message(STATUS "    Reconstruction: ${Idefix_RECONSTRUCTION}")
 message(STATUS "    Precision: ${Idefix_PRECISION}")
 message(STATUS "    Version: ${Idefix_VERSION}")
+message(STATUS "    Problem directory: '${Idefix_PROBLEM_DIR}'")
 message(STATUS "    Problem definitions: '${Idefix_DEFS}'")
 if(Idefix_CUSTOM_EOS)
   message(STATUS "    EOS: Custom file '${Idefix_CUSTOM_EOS_FILE}'")

doc/source/reference/makefile.rst

@@ -59,6 +59,12 @@ Several options can be enabled from the command line (or are accessible with ``c
     The number of ghost cells is automatically adjusted as a function of the order of the reconstruction scheme.
     *Idefix* uses 2 ghost cells when ``ORDER < 4`` and 3 ghost cells when ``ORDER = 4``
 
+``-D Idefix_PROBLEM_DIR=.``
+    Specify where to find the problem directory to build *Idefix* out of source.
+    Place yourself in the ``build`` directory you want to build in and call the ``cmake`` by :
+      + pointing the *Idefix* source directory via ``IDEFIX_DIR`` as usual.
+      + using ``-D Idefix_PROBLEM_DIR`` to point the problem to build.
+
 ``-D Kokkos_ENABLE_OPENMP=ON``
     Enable OpenMP parallelisation on supported compilers. Note that this can be enabled simultaneously with MPI, resulting in a hybrid MPI+OpenMP compilation.
 

pytools/idfx_test.py

@@ -99,6 +99,9 @@ def __init__ (self):
                     help="Consider warnings as errors",
                     action="store_true")
 
+    parser.add_argument("-ccache",
+                    help="Use ccache to reduce the build time over multiple run of the test suite.",
+                    action="store_true")
 
     args, unknown=parser.parse_known_args()
 
@@ -107,11 +110,19 @@ def __init__ (self):
     self.referenceDirectory = os.path.join(idefix_dir_env,"reference")
     # current directory relative to $IDEFIX_DIR/test (used to retrieve the path ot reference files)
     self.testDir=os.path.relpath(os.curdir,os.path.join(idefix_dir_env,"test"))
+    # build directory, currently inside the test named build-test
+    self.buildDir=os.path.abspath("./build-test")
+    # store the full path of problem directory
+    self.problemDir=os.path.abspath("./")
 
   def configure(self,definitionFile=""):
     comm=["cmake"]
     # add source directory
     comm.append(self.idefixDir)
+
+    # we will build in ./build-test so problem dir is parent
+    comm.append("-DIdefix_PROBLEM_DIR="+self.problemDir)
+
     # add specific options
     for opt in self.cmake:
       comm.append("-D"+opt)
@@ -155,7 +166,7 @@ def configure(self,definitionFile=""):
     else:
       self.definitions="definitions.hpp"
 
-    comm.append("-DIdefix_DEFS="+self.definitions)
+    comm.append("-DIdefix_DEFS="+os.path.join(self.problemDir, self.definitions))
 
     if(self.mpi):
       comm.append("-DIdefix_MPI=ON")
@@ -169,19 +180,32 @@ def configure(self,definitionFile=""):
     elif(self.reconstruction==4):
       comm.append("-DIdefix_RECONSTRUCTION=Parabolic")
 
+    # export ccache env
+    if self.ccache:
+      comm.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache")
 
     try:
-        cmake=subprocess.run(comm)
+        # clean before configuring again
+        self.clean()
+
+        # call cmake
+        cmake=subprocess.run(comm, cwd=os.path.abspath(self.buildDir))
         cmake.check_returncode()
     except subprocess.CalledProcessError as e:
         print(bcolors.FAIL+"***************************************************")
         print("Cmake failed")
         print("***************************************************"+bcolors.ENDC)
         raise e
 
+  def clean(self):
+    # remove the build directory before re-creating it
+    if os.path.exists(self.buildDir):
+      shutil.rmtree(self.buildDir)
+    os.makedirs(self.buildDir, exist_ok=False)
+
   def compile(self,jobs=8):
     try:
-        make=subprocess.run(["make","-j"+str(jobs)])
+        make=subprocess.run(["make","-j"+str(jobs)], cwd=os.path.abspath(self.buildDir))
         make.check_returncode()
     except subprocess.CalledProcessError as e:
         print(bcolors.FAIL+"***************************************************")
@@ -190,7 +214,7 @@ def compile(self,jobs=8):
         raise e
 
   def run(self, inputFile="", np=2, nowrite=False, restart=-1):
-      comm=["./idefix"]
+      comm=[os.path.join(self.buildDir,"idefix")]
       if inputFile:
           comm.append("-i")
           comm.append(inputFile)