From 9a1c3c47f204d0c1a13b8416f15e07120341f967 Mon Sep 17 00:00:00 2001 From: Rok Cesnovar Date: Wed, 23 Sep 2020 19:02:21 +0200 Subject: [PATCH 1/3] refactor jenkins pipeline --- Jenkinsfile | 77 +++++++++++++++++------------------------------------ 1 file changed, 25 insertions(+), 52 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 264abf81971..8a23b9423fd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,6 +61,7 @@ pipeline { string(defaultValue: '', name: 'stan_pr', description: 'PR to test Stan upstream against e.g. PR-630') booleanParam(defaultValue: false, name: 'withRowVector', description: 'Run additional distribution tests on RowVectors (takes 5x as long)') booleanParam(defaultValue: false, name: 'gpu_async', description: 'Run the OpenCL tests on both a sync (AMD) GPU and an async (NVIDIA) one.') + booleanParam(defaultValue: false, name: 'run_win_tests', description: 'Run full unit tests on Windows.') } options { skipDefaultCheckout() @@ -138,6 +139,7 @@ pipeline { stash 'MathSetup' sh "echo CXX=${env.CXX} -Werror > make/local" sh "echo BOOST_PARALLEL_JOBS=${env.PARALLEL} >> make/local" + failFast true parallel( CppLint: { sh "make cpplint" }, Dependencies: { sh """#!/bin/bash @@ -220,6 +222,7 @@ pipeline { !skipRemainingStages } } + failFast true parallel { stage('MPI tests') { agent { label 'linux && mpi' } @@ -310,31 +313,38 @@ pipeline { agent any steps { script { - if (isUnix()) { - deleteDir() - unstash 'MathSetup' - sh "echo CXX=${env.CXX} -Werror > make/local" - sh "echo CPPFLAGS+=-DSTAN_THREADS >> make/local" - sh "export STAN_NUM_THREADS=4" + deleteDir() + unstash 'MathSetup' + sh "echo CXX=${env.CXX} -Werror > make/local" + sh "echo STAN_THREADS=true >> make/local" + sh "export STAN_NUM_THREADS=4" + if (isBranch('develop') || isBranch('master')) { + runTests("test/unit") + sh "find . -name *_test.xml | xargs rm" + } else { runTests("test/unit -f thread") sh "find . -name *_test.xml | xargs rm" runTests("test/unit -f map_rect") sh "find . -name *_test.xml | xargs rm" - runTests("test/unit -f reduce_sum") - } else { - deleteDirWin() - unstash 'MathSetup' - bat "echo CXX=${env.CXX} -Werror > make/local" - bat "echo CXXFLAGS+=-DSTAN_THREADS >> make/local" - runTestsWin("test/unit -f thread", false) - runTestsWin("test/unit -f map_rect", false) - runTestsWin("test/unit -f reduce_sum", false) + runTests("test/unit -f reduce_sum") } } } post { always { retry(3) { deleteDir() } } } } stage('Windows Headers & Unit') { + when { + allOf { + anyOf { + branch 'develop' + branch 'master' + expression { params.run_win_tests } + } + expression { + !skipRemainingStages + } + } + } agent { label 'windows' } steps { deleteDirWin() @@ -346,43 +356,6 @@ pipeline { } } } - stage('Additional merge tests') { - when { - allOf { - anyOf { - branch 'develop' - branch 'master' - } - expression { - !skipRemainingStages - } - } - } - parallel { - stage('Linux Unit with Threading') { - agent { label 'linux' } - steps { - deleteDir() - unstash 'MathSetup' - sh "echo CXX=${GCC} >> make/local" - sh "echo CXXFLAGS=-DSTAN_THREADS >> make/local" - runTests("test/unit") - } - post { always { retry(3) { deleteDir() } } } - } - stage('Mac Unit with Threading') { - agent { label 'osx' } - steps { - deleteDir() - unstash 'MathSetup' - sh "echo CC=${env.CXX} -Werror > make/local" - sh "echo CXXFLAGS+=-DSTAN_THREADS >> make/local" - runTests("test/unit") - } - post { always { retry(3) { deleteDir() } } } - } - } - } stage('Upstream tests') { when { allOf { From c3aef7212e6357762e11aee05f2bd0adf6eb4411 Mon Sep 17 00:00:00 2001 From: Rok Cesnovar Date: Thu, 24 Sep 2020 15:47:26 +0200 Subject: [PATCH 2/3] fix for jenkins yaml --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8a23b9423fd..47eee9d9794 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -139,7 +139,6 @@ pipeline { stash 'MathSetup' sh "echo CXX=${env.CXX} -Werror > make/local" sh "echo BOOST_PARALLEL_JOBS=${env.PARALLEL} >> make/local" - failFast true parallel( CppLint: { sh "make cpplint" }, Dependencies: { sh """#!/bin/bash From 438ca77cd62c01bd2a715b656f229b7da0149f7a Mon Sep 17 00:00:00 2001 From: Rok Cesnovar Date: Fri, 2 Oct 2020 19:57:58 +0200 Subject: [PATCH 3/3] move Windows and OpenCL header checks to Github actions --- .github/workflows/header_checks.yml | 65 +++++++++++++++++++++++++++++ Jenkinsfile | 3 -- 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/header_checks.yml diff --git a/.github/workflows/header_checks.yml b/.github/workflows/header_checks.yml new file mode 100644 index 00000000000..fb84287946b --- /dev/null +++ b/.github/workflows/header_checks.yml @@ -0,0 +1,65 @@ +name: Header checks + +on: + pull_request: + branches: [ develop, master ] + push: + branches: [ develop, master ] + paths-ignore: + - 'doygen/**' + - 'hooks/**' + - 'licenses/**' + - 'LICENSE.md' + - 'README.md' + - 'RELEASE-NOTES.txt' + +jobs: + windows: + name: Windows + runs-on: windows-latest + + steps: + - uses: n1hility/cancel-previous-runs@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + workflow: c-cpp.yml + if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'" + + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '2.x' + + - name: Print g++/mingw32-make version and path + run: | + g++ --version + Get-Command g++ | Select-Object -ExpandProperty Definition + mingw32-make --version + Get-Command mingw32-make | Select-Object -ExpandProperty Definition + shell: powershell + + - name: Build Math libs + shell: powershell + run: mingw32-make -f make/standalone math-libs + + - name: Run header tests + shell: powershell + run: make -j2 test-headers + + opencl: + name: OpenCL + runs-on: ubuntu-latest + + steps: + - uses: n1hility/cancel-previous-runs@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + workflow: c-cpp.yml + if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'" + + - uses: actions/checkout@v2 + + - name: Run header tests + run: | + echo "STAN_OPENCL=true" > make/local + make -j2 test-headers \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 47eee9d9794..527bb2d0eae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -245,7 +245,6 @@ pipeline { sh "echo STAN_OPENCL=true>> make/local" sh "echo OPENCL_PLATFORM_ID=0>> make/local" sh "echo OPENCL_DEVICE_ID=${OPENCL_DEVICE_ID}>> make/local" - sh "make -j${env.PARALLEL} test-headers" runTests("test/unit/math/opencl") runTests("test/unit/math/prim/fun/gp_exp_quad_cov_test") runTests("test/unit/math/prim/fun/mdivide_left_tri_test") @@ -270,7 +269,6 @@ pipeline { sh "echo STAN_OPENCL=true>> make/local" sh "echo OPENCL_PLATFORM_ID=0>> make/local" sh "echo OPENCL_DEVICE_ID=${OPENCL_DEVICE_ID}>> make/local" - sh "make -j${env.PARALLEL} test-headers" runTests("test/unit/math/opencl") runTests("test/unit/math/prim/fun/gp_exp_quad_cov_test") runTests("test/unit/math/prim/fun/mdivide_left_tri_test") @@ -349,7 +347,6 @@ pipeline { deleteDirWin() unstash 'MathSetup' bat "mingw32-make.exe -f make/standalone math-libs" - bat "mingw32-make -j${env.PARALLEL} test-headers" runTestsWin("test/unit", false, true) } }