From 9d192e8381d7f554a8a3ecddaf34db7cf27fe4fd Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Tue, 8 Jul 2025 08:12:05 +0000 Subject: [PATCH 1/3] Merge current waive list with TOT waive list Signed-off-by: Yiqing Yan --- jenkins/L0_MergeRequest.groovy | 179 +++++++++++++++++++++++++++++---- jenkins/L0_Test.groovy | 15 +++ jenkins/mergeWaiveList.py | 50 +++++++++ 3 files changed, 222 insertions(+), 22 deletions(-) create mode 100644 jenkins/mergeWaiveList.py diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 583bfe80c9bf..9448b0db74f1 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -141,11 +141,14 @@ def CACHED_CHANGED_FILE_LIST = "cached_changed_file_list" def ACTION_INFO = "action_info" @Field def IMAGE_KEY_TO_TAG = "image_key_to_tag" +@Field +def TARGET_BRANCH = "target_branch" def globalVars = [ (GITHUB_PR_API_URL): gitlabParamsFromBot.get('github_pr_api_url', null), (CACHED_CHANGED_FILE_LIST): null, (ACTION_INFO): gitlabParamsFromBot.get('action_info', null), (IMAGE_KEY_TO_TAG): [:], + (TARGET_BRANCH): gitlabParamsFromBot.get('target_branch', null), ] // If not running all test stages in the L0 pre-merge, we will not update the GitLab status at the end. @@ -300,31 +303,163 @@ def echoNodeAndGpuInfo(pipeline, stageName) } def setupPipelineEnvironment(pipeline, testFilter, globalVars) +{ + sh "env | sort" + updateGitlabCommitStatus name: "${BUILD_STATUS_NAME}", state: 'running' + echo "Using GitLab repo: ${LLM_REPO}." + sh "git config --global --add safe.directory \"*\"" + // NB: getContainerURIs reads files in ${LLM_ROOT}/jenkins/ + if (env.gitlabMergeRequestLastCommit) { + env.gitlabCommit = env.gitlabMergeRequestLastCommit + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + } else { + branch = env.gitlabBranch ? env.gitlabBranch : "main" + trtllm_utils.checkoutSource(LLM_REPO, branch, LLM_ROOT, true, true) + checkoutCommit = sh (script: "cd ${LLM_ROOT} && git rev-parse HEAD",returnStdout: true).trim() + env.gitlabCommit = checkoutCommit + } + echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." + echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." + testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) + testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) + testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) + getContainerURIs().each { k, v -> + globalVars[k] = v + } +} + +def getMergeRequestOneFileChangesGitlab(pipeline, filePath) { + withCredentials([ + usernamePassword( + credentialsId: 'svc_tensorrt_gitlab_read_api_token', + usernameVariable: 'GITLAB_API_USER', + passwordVariable: 'GITLAB_API_TOKEN' + ), + string(credentialsId: 'default-git-url', variable: 'DEFAULT_GIT_URL') + ]) { + def diff = "" + def pageId = 0 + while(true) { + pageId += 1 + def rawDataJson = pipeline.sh( + script: """ + curl --header "PRIVATE-TOKEN: $GITLAB_API_TOKEN" \ + --url "https://${DEFAULT_GIT_URL}/api/v4/projects/${env.gitlabMergeRequestTargetProjectId}/merge_requests/${env.gitlabMergeRequestIid}/diffs?page=${pageId}&per_page=20" + """, + returnStdout: true + ) + def rawDataList = readJSON text: rawDataJson, returnPojo: true + def found = rawDataList.find { rawData -> + if (rawData.get("new_path") == filePath || rawData.get("old_path") == filePath) { + diff = rawData.get("diff") + return true + } + return false + } + if (!rawDataList || diff != "") { break } + } + pipeline.echo("The change of ${filePath} is: ${diff}") + return diff + } +} + +def getMergeRequestOneFileChangesGithub(pipeline, githubPrApiUrl, filePath) { + def diff = "" + def pageId = 0 + withCredentials([ + string( + credentialsId: 'github-token-trtllm-ci', + variable: 'GITHUB_API_TOKEN' + ), + ]) { + while(true) { + pageId += 1 + def rawDataJson = pipeline.sh( + script: """ + curl --header "Authorization: Bearer $GITHUB_API_TOKEN" \ + --url "${githubPrApiUrl}/files?page=${pageId}&per_page=20" + """, + returnStdout: true + ) + echo "rawDataJson: ${rawDataJson}" + def rawDataList = readJSON text: rawDataJson, returnPojo: true + def found = rawDataList.find { rawData -> + if (rawData.get("filename") == filePath || rawData.get("previous_filename") == filePath) { + diff = rawData.get("patch") + return true + } + return false + } + if (!rawDataList || diff != "") { break } + } + } + pipeline.echo("The change of ${filePath} is: ${diff}") + return diff +} + +def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) { + def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) + if (env.alternativeTRT || isOfficialPostMergeJob) { + pipeline.echo("Force set waive list diff to empty.") + return "" + } + + def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] + def diff = "" + + try { + if (githubPrApiUrl != null) { + diff = getMergeRequestOneFileChangesGithub(pipeline, githubPrApiUrl, filePath) + } else { + diff = getMergeRequestOneFileChangesGitlab(pipeline, filePath) + } + return diff + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + pipeline.echo("Get merge request one changed file diff failed. Error: ${e.toString()}") + return "" + } +} + +def mergeWaiveList(pipeline, globalVars) +{ + sh "git config --global --add safe.directory \"*\"" + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabMergeRequestLastCommit, LLM_ROOT, true, true) + sh "cp ${LLM_ROOT}/tests/integration/test_lists/waives.txt ./waives_CUR_${env.gitlabMergeRequestLastCommit}.txt" + sh "cp ${LLM_ROOT}/jenkins/mergeWaiveList.py ./" + + targetBranch = env.gitlabTargetBranch ? env.gitlabTargetBranch : globalVars[TARGET_BRANCH] + echo "Target branch: ${targetBranch}" + trtllm_utils.checkoutSource(LLM_REPO, targetBranch, LLM_ROOT, true, true) + targetBranchTOTCommit = sh (script: "cd ${LLM_ROOT} && git rev-parse HEAD", returnStdout: true).trim() + echo "Target branch TOT commit: ${targetBranchTOTCommit}" + sh "cp ${LLM_ROOT}/tests/integration/test_lists/waives.txt ./waives_TOT_${targetBranchTOTCommit}.txt" + // sh "cp ${LLM_ROOT}/jenkins/mergeWaiveList.py ./" + + def diff = getMergeRequestOneFileChanges(pipeline, globalVars, "tests/integration/test_lists/waives.txt") + + sh """ + python3 mergeWaiveList.py \ + --cur-waive-list=waives_CUR_${env.gitlabMergeRequestLastCommit}.txt \ + --latest-waive-list=waives_TOT_${targetBranchTOTCommit}.txt \ + --diff='${diff}' \ + --output-file=waives.txt + """ + trtllm_utils.uploadArtifacts("waives*.txt", "${UPLOAD_PATH}/waive_list/") + echo "Waive list: https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/waive_list/" +} + +def preparation(pipeline, testFilter, globalVars) { image = "urm.nvidia.com/docker/golang:1.22" setupPipelineSpec = createKubernetesPodConfig(image, "package") trtllm_utils.launchKubernetesPod(pipeline, setupPipelineSpec, "trt-llm", { - sh "env | sort" - updateGitlabCommitStatus name: "${BUILD_STATUS_NAME}", state: 'running' - echo "Using GitLab repo: ${LLM_REPO}." - sh "git config --global --add safe.directory \"*\"" - // NB: getContainerURIs reads files in ${LLM_ROOT}/jenkins/ - if (env.gitlabMergeRequestLastCommit) { - env.gitlabCommit = env.gitlabMergeRequestLastCommit - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) - } else { - branch = env.gitlabBranch ? env.gitlabBranch : "main" - trtllm_utils.checkoutSource(LLM_REPO, branch, LLM_ROOT, true, true) - checkoutCommit = sh (script: "cd ${LLM_ROOT} && git rev-parse HEAD",returnStdout: true).trim() - env.gitlabCommit = checkoutCommit + stage("Setup environment") { + setupPipelineEnvironment(pipeline, testFilter, globalVars) } - echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." - echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." - testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) - testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) - testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) - getContainerURIs().each { k, v -> - globalVars[k] = v + stage("Merge Waive List") { + mergeWaiveList(pipeline, globalVars) } }) } @@ -1136,12 +1271,12 @@ pipeline { } } stages { - stage("Setup environment") + stage("Preparation") { steps { script { - setupPipelineEnvironment(this, testFilter, globalVars) + preparation(this, testFilter, globalVars) println globalVars globalVars[ACTION_INFO] = trtllm_utils.setupPipelineDescription(this, globalVars[ACTION_INFO]) echo "enableFailFast is: ${enableFailFast}" diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 47326f5012f5..e7899c1ed219 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1290,6 +1290,21 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && wget -nv ${llmTarfile}") sh "cd ${llmPath} && tar -zxf ${tarName}" + // download waives.txt + def waivesTxt = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/waive_list/waives.txt" + try { + trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${waivesTxt}") + if (fileExists("waives.txt")) { + sh "rm ${llmSrc}/tests/integration/test_lists/waives.txt" + sh "mv waives.txt ${llmSrc}/tests/integration/test_lists/waives.txt" + } + echo "Download merged waives.txt successfully" + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + echo "Failed to download merged waives.txt, use the default waives.txt. Error: ${e.message}" + } + // install python package if (env.alternativeTRT) { sh "cd ${llmSrc} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" diff --git a/jenkins/mergeWaiveList.py b/jenkins/mergeWaiveList.py new file mode 100644 index 000000000000..202fa02aea13 --- /dev/null +++ b/jenkins/mergeWaiveList.py @@ -0,0 +1,50 @@ +import argparse +import sys + + +def get_remove_lines_from_diff(diff): + lines = diff.split('\n') + remove_lines = [ + line[1:] + '\n' for line in lines if len(line) > 1 and line[0] == '-' + ] + return remove_lines + + +def parse_waive_txt(waive_txt): + with open(waive_txt, 'r') as f: + lines = f.readlines() + waive_list = [line for line in lines if len(line) != 0] + return waive_list + + +def write_waive_list(waive_list, output_file): + with open(output_file, 'w') as f: + for line in waive_list: + f.write(line) + + +def merge_waive_list(cur_list, main_list, remove_lines, output_file): + merged = list(dict.fromkeys(cur_list + main_list)) + for line in reversed(remove_lines): + for i in range(len(merged) - 1, -1, -1): + if merged[i] == line: + merged.pop(i) + break + write_waive_list(merged, output_file) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--cur-waive-list', + required=True, + help='Current waive list') + parser.add_argument('--latest-waive-list', + required=True, + help='Latest waive list') + parser.add_argument('--diff', required=True, help='Diff of the waive list') + parser.add_argument('--output-file', required=True, help='Output file') + args = parser.parse_args(sys.argv[1:]) + cur_list = parse_waive_txt(args.cur_waive_list) + main_list = parse_waive_txt(args.latest_waive_list) + remove_lines = get_remove_lines_from_diff(args.diff) + merge_waive_list(cur_list, main_list, remove_lines, args.output_file) From 6b7c74f0cfe244605c6f1b320401792806111b62 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Tue, 22 Jul 2025 09:20:16 +0000 Subject: [PATCH 2/3] fix for review Signed-off-by: Yiqing Yan --- jenkins/L0_MergeRequest.groovy | 205 ++++++++++-------------- jenkins/L0_Test.groovy | 9 +- jenkins/{ => scripts}/mergeWaiveList.py | 9 +- 3 files changed, 96 insertions(+), 127 deletions(-) rename jenkins/{ => scripts}/mergeWaiveList.py (83%) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 9448b0db74f1..3c59e0d77edd 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -328,126 +328,35 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) } } -def getMergeRequestOneFileChangesGitlab(pipeline, filePath) { - withCredentials([ - usernamePassword( - credentialsId: 'svc_tensorrt_gitlab_read_api_token', - usernameVariable: 'GITLAB_API_USER', - passwordVariable: 'GITLAB_API_TOKEN' - ), - string(credentialsId: 'default-git-url', variable: 'DEFAULT_GIT_URL') - ]) { - def diff = "" - def pageId = 0 - while(true) { - pageId += 1 - def rawDataJson = pipeline.sh( - script: """ - curl --header "PRIVATE-TOKEN: $GITLAB_API_TOKEN" \ - --url "https://${DEFAULT_GIT_URL}/api/v4/projects/${env.gitlabMergeRequestTargetProjectId}/merge_requests/${env.gitlabMergeRequestIid}/diffs?page=${pageId}&per_page=20" - """, - returnStdout: true - ) - def rawDataList = readJSON text: rawDataJson, returnPojo: true - def found = rawDataList.find { rawData -> - if (rawData.get("new_path") == filePath || rawData.get("old_path") == filePath) { - diff = rawData.get("diff") - return true - } - return false - } - if (!rawDataList || diff != "") { break } - } - pipeline.echo("The change of ${filePath} is: ${diff}") - return diff - } -} - -def getMergeRequestOneFileChangesGithub(pipeline, githubPrApiUrl, filePath) { - def diff = "" - def pageId = 0 - withCredentials([ - string( - credentialsId: 'github-token-trtllm-ci', - variable: 'GITHUB_API_TOKEN' - ), - ]) { - while(true) { - pageId += 1 - def rawDataJson = pipeline.sh( - script: """ - curl --header "Authorization: Bearer $GITHUB_API_TOKEN" \ - --url "${githubPrApiUrl}/files?page=${pageId}&per_page=20" - """, - returnStdout: true - ) - echo "rawDataJson: ${rawDataJson}" - def rawDataList = readJSON text: rawDataJson, returnPojo: true - def found = rawDataList.find { rawData -> - if (rawData.get("filename") == filePath || rawData.get("previous_filename") == filePath) { - diff = rawData.get("patch") - return true - } - return false - } - if (!rawDataList || diff != "") { break } - } - } - pipeline.echo("The change of ${filePath} is: ${diff}") - return diff -} - -def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) { - def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || isOfficialPostMergeJob) { - pipeline.echo("Force set waive list diff to empty.") - return "" - } - - def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] - def diff = "" - - try { - if (githubPrApiUrl != null) { - diff = getMergeRequestOneFileChangesGithub(pipeline, githubPrApiUrl, filePath) - } else { - diff = getMergeRequestOneFileChangesGitlab(pipeline, filePath) - } - return diff - } catch (InterruptedException e) { - throw e - } catch (Exception e) { - pipeline.echo("Get merge request one changed file diff failed. Error: ${e.toString()}") - return "" - } -} - def mergeWaiveList(pipeline, globalVars) { + // Get current waive list sh "git config --global --add safe.directory \"*\"" - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabMergeRequestLastCommit, LLM_ROOT, true, true) - sh "cp ${LLM_ROOT}/tests/integration/test_lists/waives.txt ./waives_CUR_${env.gitlabMergeRequestLastCommit}.txt" - sh "cp ${LLM_ROOT}/jenkins/mergeWaiveList.py ./" + sh "cp ${LLM_ROOT}/tests/integration/test_lists/waives.txt ./waives_CUR_${env.gitlabCommit}.txt" + sh "cp ${LLM_ROOT}/jenkins/scripts/mergeWaiveList.py ./" + // Get TOT waive list + LLM_TOT_ROOT = "llm-tot" targetBranch = env.gitlabTargetBranch ? env.gitlabTargetBranch : globalVars[TARGET_BRANCH] echo "Target branch: ${targetBranch}" - trtllm_utils.checkoutSource(LLM_REPO, targetBranch, LLM_ROOT, true, true) - targetBranchTOTCommit = sh (script: "cd ${LLM_ROOT} && git rev-parse HEAD", returnStdout: true).trim() + trtllm_utils.checkoutSource(LLM_REPO, targetBranch, LLM_TOT_ROOT, true, true) + targetBranchTOTCommit = sh (script: "cd ${LLM_TOT_ROOT} && git rev-parse HEAD", returnStdout: true).trim() echo "Target branch TOT commit: ${targetBranchTOTCommit}" - sh "cp ${LLM_ROOT}/tests/integration/test_lists/waives.txt ./waives_TOT_${targetBranchTOTCommit}.txt" - // sh "cp ${LLM_ROOT}/jenkins/mergeWaiveList.py ./" + sh "cp ${LLM_TOT_ROOT}/tests/integration/test_lists/waives.txt ./waives_TOT_${targetBranchTOTCommit}.txt" + // Get waive list diff in current MR def diff = getMergeRequestOneFileChanges(pipeline, globalVars, "tests/integration/test_lists/waives.txt") + // Merge waive lists sh """ python3 mergeWaiveList.py \ - --cur-waive-list=waives_CUR_${env.gitlabMergeRequestLastCommit}.txt \ + --cur-waive-list=waives_CUR_${env.gitlabCommit}.txt \ --latest-waive-list=waives_TOT_${targetBranchTOTCommit}.txt \ --diff='${diff}' \ --output-file=waives.txt """ trtllm_utils.uploadArtifacts("waives*.txt", "${UPLOAD_PATH}/waive_list/") - echo "Waive list: https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/waive_list/" + echo "New merged test waive list: https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/waive_list/waives.txt" } def preparation(pipeline, testFilter, globalVars) @@ -455,10 +364,10 @@ def preparation(pipeline, testFilter, globalVars) image = "urm.nvidia.com/docker/golang:1.22" setupPipelineSpec = createKubernetesPodConfig(image, "package") trtllm_utils.launchKubernetesPod(pipeline, setupPipelineSpec, "trt-llm", { - stage("Setup environment") { + stage("Setup Environment") { setupPipelineEnvironment(pipeline, testFilter, globalVars) } - stage("Merge Waive List") { + stage("Merge Test Waive List") { mergeWaiveList(pipeline, globalVars) } }) @@ -550,8 +459,8 @@ def launchReleaseCheck(pipeline) }) } -def getMergeRequestChangedFileListGitlab(pipeline) { - def changedFileList = [] +def getGitlabMRChangedFile(pipeline, function, filePath="") { + def result = null def pageId = 0 withCredentials([ usernamePassword( @@ -571,19 +480,34 @@ def getMergeRequestChangedFileListGitlab(pipeline) { returnStdout: true ) def rawDataList = readJSON text: rawDataJson, returnPojo: true - rawDataList.each { rawData -> - changedFileList += [rawData.get("old_path"), rawData.get("new_path")] + if (function == "getOneFileChanges") { + if (result == null) { + result = "" + } + rawDataList.find { rawData -> + if (rawData.get("new_path") == filePath || rawData.get("old_path") == filePath) { + result = rawData.get("diff") + return true + } + return false + } + if (result != "") { break } + } else if (function == "getChangedFileList") { + if (result == null) { + result = [] + } + rawDataList.each { rawData -> + result += [rawData.get("old_path"), rawData.get("new_path")] + } } if (!rawDataList) { break } } } - def changedFileListStr = changedFileList.join(",\n") - pipeline.echo("The changeset of this MR is: ${changedFileListStr}.") - return changedFileList + return result } -def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { - def changedFileList = [] +def getGithubMRChangedFile(pipeline, githubPrApiUrl, function, filePath="") { + def result = null def pageId = 0 withCredentials([ string( @@ -602,15 +526,30 @@ def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { ) echo "rawDataJson: ${rawDataJson}" def rawDataList = readJSON text: rawDataJson, returnPojo: true - rawDataList.each { rawData -> - changedFileList += [rawData.get("filename"), rawData.get("previous_filename")].findAll { it } + if (function == "getOneFileChanges") { + if (result == null) { + result = "" + } + rawDataList.find { rawData -> + if (rawData.get("filename") == filePath || rawData.get("previous_filename") == filePath) { + result = rawData.get("patch") + return true + } + return false + } + if (result != "") { break } + } else if (function == "getChangedFileList") { + if (result == null) { + result = [] + } + rawDataList.each { rawData -> + result += [rawData.get("filename"), rawData.get("previous_filename")].findAll { it } + } } if (!rawDataList) { break } } } - def changedFileListStr = changedFileList.join(",\n") - pipeline.echo("The changeset of this PR is: ${changedFileListStr}.") - return changedFileList + return result } def getMergeRequestChangedFileList(pipeline, globalVars) { @@ -626,11 +565,15 @@ def getMergeRequestChangedFileList(pipeline, globalVars) { return globalVars[CACHED_CHANGED_FILE_LIST] } try { + def changedFileList = [] if (githubPrApiUrl != null) { - globalVars[CACHED_CHANGED_FILE_LIST] = getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) + changedFileList = getGithubMRChangedFile(pipeline, githubPrApiUrl, "getChangedFileList") } else { - globalVars[CACHED_CHANGED_FILE_LIST] = getMergeRequestChangedFileListGitlab(pipeline) + changedFileList = getGitlabMRChangedFile(pipeline, "getChangedFileList") } + def changedFileListStr = changedFileList.join(",\n") + pipeline.echo("The changeset of this MR is: ${changedFileListStr}.") + globalVars[CACHED_CHANGED_FILE_LIST] = changedFileList return globalVars[CACHED_CHANGED_FILE_LIST] } catch (InterruptedException e) { throw e @@ -641,6 +584,26 @@ def getMergeRequestChangedFileList(pipeline, globalVars) { } } +def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) { + def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] + def diff = "" + + try { + if (githubPrApiUrl != null) { + diff = getGithubMRChangedFile(pipeline, githubPrApiUrl, "getOneFileChanges", filePath) + } else { + diff = getGitlabMRChangedFile(pipeline, "getOneFileChanges", filePath) + } + pipeline.echo("The change of ${filePath} is: ${diff}") + return diff + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + pipeline.echo("Get merge request one changed file diff failed. Error: ${e.toString()}") + return "" + } +} + def getAutoTriggerTagList(pipeline, testFilter, globalVars) { def autoTriggerTagList = [] def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e7899c1ed219..ffa1fa3267c1 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1290,14 +1290,15 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && wget -nv ${llmTarfile}") sh "cd ${llmPath} && tar -zxf ${tarName}" - // download waives.txt + // Download the new merged waives.txt def waivesTxt = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/waive_list/waives.txt" try { trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${waivesTxt}") - if (fileExists("waives.txt")) { - sh "rm ${llmSrc}/tests/integration/test_lists/waives.txt" - sh "mv waives.txt ${llmSrc}/tests/integration/test_lists/waives.txt" + if (!fileExists("waives.txt")) { + error "There is no merged waives.txt file, use the default waives.txt." } + sh "rm ${llmSrc}/tests/integration/test_lists/waives.txt" + sh "mv waives.txt ${llmSrc}/tests/integration/test_lists/waives.txt" echo "Download merged waives.txt successfully" } catch (InterruptedException e) { throw e diff --git a/jenkins/mergeWaiveList.py b/jenkins/scripts/mergeWaiveList.py similarity index 83% rename from jenkins/mergeWaiveList.py rename to jenkins/scripts/mergeWaiveList.py index 202fa02aea13..909526a117f3 100644 --- a/jenkins/mergeWaiveList.py +++ b/jenkins/scripts/mergeWaiveList.py @@ -1,11 +1,16 @@ import argparse import sys +# Generate the merged waive list: +# 1. Parse the current MR waive list, and get the removed lines from the diff +# 2. Parse the TOT waive list +# 3. Merge the current MR waive list and TOT waive list, and remove the removed lines from the step 1 + def get_remove_lines_from_diff(diff): lines = diff.split('\n') remove_lines = [ - line[1:] + '\n' for line in lines if len(line) > 1 and line[0] == '-' + line[1:] + '\n' for line in lines if len(line) > 1 and line.startswith('-') ] return remove_lines @@ -13,7 +18,7 @@ def get_remove_lines_from_diff(diff): def parse_waive_txt(waive_txt): with open(waive_txt, 'r') as f: lines = f.readlines() - waive_list = [line for line in lines if len(line) != 0] + waive_list = [line for line in lines if line.strip()] return waive_list From 27225add6df7cb2eeaf5f2d1f989ea4fbe9f253f Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Tue, 22 Jul 2025 09:41:03 +0000 Subject: [PATCH 3/3] fix pre-commit check Signed-off-by: Yiqing Yan --- jenkins/scripts/mergeWaiveList.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins/scripts/mergeWaiveList.py b/jenkins/scripts/mergeWaiveList.py index 909526a117f3..418a603b9af7 100644 --- a/jenkins/scripts/mergeWaiveList.py +++ b/jenkins/scripts/mergeWaiveList.py @@ -10,7 +10,8 @@ def get_remove_lines_from_diff(diff): lines = diff.split('\n') remove_lines = [ - line[1:] + '\n' for line in lines if len(line) > 1 and line.startswith('-') + line[1:] + '\n' for line in lines + if len(line) > 1 and line.startswith('-') ] return remove_lines