From 9082c94a88793a004fba2e7088d50aca58ec3a3a Mon Sep 17 00:00:00 2001 From: tqchen Date: Tue, 3 Jun 2025 18:16:42 -0400 Subject: [PATCH] [CI] Further robustify is_last_build check This PR further robustifies is_last_build check so it does not involve getJob API which may be denied. Hopefully will resolve some of the spot not rerun issue recently. --- ci/jenkins/generated/arm_jenkinsfile.groovy | 36 +++++++------ ci/jenkins/generated/cpu_jenkinsfile.groovy | 46 +++++++++-------- .../generated/docker_jenkinsfile.groovy | 13 +++-- ci/jenkins/generated/gpu_jenkinsfile.groovy | 51 +++++++++++-------- .../generated/hexagon_jenkinsfile.groovy | 36 +++++++------ ci/jenkins/generated/i386_jenkinsfile.groovy | 51 +++++++++++-------- ci/jenkins/generated/lint_jenkinsfile.groovy | 13 +++-- ci/jenkins/generated/wasm_jenkinsfile.groovy | 36 +++++++------ ci/jenkins/templates/utils/Prepare.groovy.j2 | 11 ++-- ci/jenkins/templates/utils/macros.j2 | 28 +++++----- 10 files changed, 182 insertions(+), 139 deletions(-) diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index 03ea3a028040..9e4afc8f1393 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-02-15T20:02:41.820729 +// Generated at 2025-06-03T18:16:35.851073 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -280,10 +280,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { @@ -530,17 +533,18 @@ def build() { try { run_build('ARM-GRAVITON3-SPOT') } catch (Throwable ex) { - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' - currentBuild.result = 'SUCCESS' - run_build('ARM-GRAVITON3') - } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' - throw ex - } + echo 'Exception during SPOT run ' + ex.toString() + if (is_last_build()) { + // retry if we are currently at last build + // mark the current stage as success + // and try again via on demand node + echo 'Retry on-demand given it is last build' + currentBuild.result = 'SUCCESS' + run_build('ARM-GRAVITON3') + } else { + echo 'Exit since it is not last build' + throw ex + } } } } diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index e93400f6d637..daadc16c7631 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-05-09T10:31:17.078676 +// Generated at 2025-06-03T18:16:35.861918 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -280,10 +280,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { @@ -530,17 +533,18 @@ def build() { try { run_build('CPU-SPOT') } catch (Throwable ex) { - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' - throw ex - } + echo 'Exception during SPOT run ' + ex.toString() + if (is_last_build()) { + // retry if we are currently at last build + // mark the current stage as success + // and try again via on demand node + echo 'Retry on-demand given it is last build' + currentBuild.result = 'SUCCESS' + run_build('CPU') + } else { + echo 'Exit since it is not last build' + throw ex + } } } } @@ -648,15 +652,16 @@ def test() { try { shard_run_unittest_CPU_1_of_2('CPU-SMALL-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' shard_run_unittest_CPU_1_of_2('CPU-SMALL') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } } @@ -665,15 +670,16 @@ def test() { try { shard_run_unittest_CPU_2_of_2('CPU-SMALL-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' shard_run_unittest_CPU_2_of_2('CPU-SMALL') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } } diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy index daad2188ff26..2391d9a87ab8 100644 --- a/ci/jenkins/generated/docker_jenkinsfile.groovy +++ b/ci/jenkins/generated/docker_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-02-15T10:12:52.000152 +// Generated at 2025-06-03T18:16:35.797894 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -280,10 +280,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index 20f016dcdee3..1fc4348c6f1c 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-02-15T12:03:28.800680 +// Generated at 2025-06-03T18:16:35.885417 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -280,10 +280,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { @@ -536,17 +539,18 @@ def build() { try { run_build('CPU-SPOT') } catch (Throwable ex) { - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' - throw ex - } + echo 'Exception during SPOT run ' + ex.toString() + if (is_last_build()) { + // retry if we are currently at last build + // mark the current stage as success + // and try again via on demand node + echo 'Retry on-demand given it is last build' + currentBuild.result = 'SUCCESS' + run_build('CPU') + } else { + echo 'Exit since it is not last build' + throw ex + } } } } @@ -728,15 +732,16 @@ def test() { try { shard_run_unittest_GPU_1_of_2('GPU-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' shard_run_unittest_GPU_1_of_2('GPU') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } } @@ -745,15 +750,16 @@ def test() { try { shard_run_unittest_GPU_2_of_2('GPU-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' shard_run_unittest_GPU_2_of_2('GPU') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } } @@ -762,15 +768,16 @@ def test() { try { shard_run_docs_GPU_1_of_1('GPU-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' shard_run_docs_GPU_1_of_1('GPU') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } } diff --git a/ci/jenkins/generated/hexagon_jenkinsfile.groovy b/ci/jenkins/generated/hexagon_jenkinsfile.groovy index a9014337a74a..173506fcce7e 100644 --- a/ci/jenkins/generated/hexagon_jenkinsfile.groovy +++ b/ci/jenkins/generated/hexagon_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-02-15T19:31:36.031215 +// Generated at 2025-06-03T18:16:35.839798 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -280,10 +280,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { @@ -534,17 +537,18 @@ def build() { try { run_build('CPU-SPOT') } catch (Throwable ex) { - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' - throw ex - } + echo 'Exception during SPOT run ' + ex.toString() + if (is_last_build()) { + // retry if we are currently at last build + // mark the current stage as success + // and try again via on demand node + echo 'Retry on-demand given it is last build' + currentBuild.result = 'SUCCESS' + run_build('CPU') + } else { + echo 'Exit since it is not last build' + throw ex + } } } } diff --git a/ci/jenkins/generated/i386_jenkinsfile.groovy b/ci/jenkins/generated/i386_jenkinsfile.groovy index 993f1d3e8323..3ef2b532bae1 100644 --- a/ci/jenkins/generated/i386_jenkinsfile.groovy +++ b/ci/jenkins/generated/i386_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-02-15T10:14:10.142167 +// Generated at 2025-06-03T18:16:35.814567 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -280,10 +280,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { @@ -530,17 +533,18 @@ def build() { try { run_build('CPU-SPOT') } catch (Throwable ex) { - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' - throw ex - } + echo 'Exception during SPOT run ' + ex.toString() + if (is_last_build()) { + // retry if we are currently at last build + // mark the current stage as success + // and try again via on demand node + echo 'Retry on-demand given it is last build' + currentBuild.result = 'SUCCESS' + run_build('CPU') + } else { + echo 'Exit since it is not last build' + throw ex + } } } } @@ -702,15 +706,16 @@ def test() { try { shard_run_python_i386_1_of_3('CPU-SMALL-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' shard_run_python_i386_1_of_3('CPU-SMALL') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } } @@ -719,15 +724,16 @@ def test() { try { shard_run_python_i386_2_of_3('CPU-SMALL-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' shard_run_python_i386_2_of_3('CPU-SMALL') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } } @@ -736,15 +742,16 @@ def test() { try { shard_run_python_i386_3_of_3('CPU-SMALL-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' shard_run_python_i386_3_of_3('CPU-SMALL') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } } diff --git a/ci/jenkins/generated/lint_jenkinsfile.groovy b/ci/jenkins/generated/lint_jenkinsfile.groovy index a1750eb853d5..c347ddc40862 100644 --- a/ci/jenkins/generated/lint_jenkinsfile.groovy +++ b/ci/jenkins/generated/lint_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-02-15T10:12:51.981152 +// Generated at 2025-06-03T18:16:35.827692 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -280,10 +280,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy index 407f4c80049d..d214fb3710f3 100644 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-02-15T10:14:10.202706 +// Generated at 2025-06-03T18:16:35.874501 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -280,10 +280,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { @@ -532,17 +535,18 @@ def build() { try { run_build('CPU-SPOT') } catch (Throwable ex) { - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' - throw ex - } + echo 'Exception during SPOT run ' + ex.toString() + if (is_last_build()) { + // retry if we are currently at last build + // mark the current stage as success + // and try again via on demand node + echo 'Retry on-demand given it is last build' + currentBuild.result = 'SUCCESS' + run_build('CPU') + } else { + echo 'Exit since it is not last build' + throw ex + } } } } diff --git a/ci/jenkins/templates/utils/Prepare.groovy.j2 b/ci/jenkins/templates/utils/Prepare.groovy.j2 index 68e6569d4e46..f4c5193232b0 100644 --- a/ci/jenkins/templates/utils/Prepare.groovy.j2 +++ b/ci/jenkins/templates/utils/Prepare.groovy.j2 @@ -156,10 +156,13 @@ def cancel_previous_build() { } def is_last_build() { - // whether it is last build - def job = Jenkins.instance.getItem(env.JOB_NAME) - def lastBuild = job.getLastBuild() - return lastBuild.getNumber() == env.BUILD_NUMBER + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } } def checkout_trusted_files() { diff --git a/ci/jenkins/templates/utils/macros.j2 b/ci/jenkins/templates/utils/macros.j2 index ee90b043ccbc..662d9aef111c 100644 --- a/ci/jenkins/templates/utils/macros.j2 +++ b/ci/jenkins/templates/utils/macros.j2 @@ -96,17 +96,18 @@ def build() { try { run_build('{{ node }}-SPOT') } catch (Throwable ex) { - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' - currentBuild.result = 'SUCCESS' - run_build('{{ node }}') - } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' - throw ex - } + echo 'Exception during SPOT run ' + ex.toString() + if (is_last_build()) { + // retry if we are currently at last build + // mark the current stage as success + // and try again via on demand node + echo 'Retry on-demand given it is last build' + currentBuild.result = 'SUCCESS' + run_build('{{ node }}') + } else { + echo 'Exit since it is not last build' + throw ex + } } } } @@ -125,15 +126,16 @@ def test() { try { {{ method_name }}('{{ node }}-SPOT') } catch (Throwable ex) { + echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { // retry if at last build // mark the current stage as success // and try again via on demand node - echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand' + echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' {{ method_name }}('{{ node }}') } else { - echo 'Exception during SPOT run ' + ex.toString() + ' exit since it is not last build' + echo 'Exit since it is not last build' throw ex } }