From c1cdf5245d74c72ef167bd938f516fbe8e5de2a8 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 1 Sep 2022 18:13:24 -0500 Subject: [PATCH 01/66] Added verify_release_build.yml file with check_environment an create_pr test steps. --- .github/workflows/verify_release_build.yml | 69 ++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/verify_release_build.yml diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml new file mode 100644 index 000000000000..6239c8937410 --- /dev/null +++ b/.github/workflows/verify_release_build.yml @@ -0,0 +1,69 @@ +name: Verify Release Build +on: + workflow_dispatch: + inputs: + RELEASE_VER: + description: Beam version of current release + required: true +env: + WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? +jobs: + check_environment: + runs-on: ubuntu-latest + steps: + - name: Check environment + id: check_env_vars + env: + RELEASE_BRANCH: ${{ release-$RELEASE_VER }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Verify release build against branch: $RELEASE_BRANCH." + echo "Verify hub installation." + hub version + git version + create_release_pr: + needs: check_environment + runs-on: ubuntu-latest + steps: + - run: | + echo "[Current Task] Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins." + echo "This task will create a PR against apache/beam." + - name: Check out code + uses: actions/checkout@v3 + ref: ${{ $RELEASE_BRANCH }} + - name: Set Dataflow Python Batch Tests Release Version + # Commenting for the moment: sh ./set_version.sh $RELEASE_VER --git-add instead of echo "This is a test." >> TEST_FILE.md + run: | + git checkout -b $WORKING_BRANCH --quiet + echo "This is a test." >> TEST_FILE.md + git add TEST_FILE.md + # In case the version string was not changed, append a newline to CHANGES.md + echo "" >> CHANGES.md + git add CHANGES.md + git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet + git push origin $WORKING_BRANCH --quiet + hub pull-request -F- <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch + You can run many tests automatically using release/src/main/scripts/mass_comment.py." + echo "[NOTE]: Please make sure all test targets have been invoked." + echo "Please check the test results. If there is any failure, follow the policy in release guide." + working-directory: 'release/src/main/scripts' + +# trigger_phrases: +# needs: create_testing_pr +# runs-on: ubuntu-latest +# steps: +# - name: Trigger beam_Release_Gradle_Build and PostCommit jobs # Pending to include tests migrated to GH Actions (by Wizeline team) +# run: | +# sh ./mass_comment.py +# working-directory: 'release/src/main/scripts' + +# Commenting this clean-up step 'cause I think this step is not needed, as the final self-hosted implementation has ephemeral runners. +# clean-up: +# if: always() +# runs-on: ubuntu-latest +# steps: +# - name: Clean Temporal Directory +# run: | +# echo "==================== Final Cleanup ====================" +# rm -rf $LOCAL_BEAM_DIR +# echo "* Deleted workspace ${LOCAL_BEAM_DIR}" From ec9907246ad9609101647685bd44c3ee99c4d77f Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 1 Sep 2022 18:19:03 -0500 Subject: [PATCH 02/66] Fixing typo issue in checkout step in yml. --- .github/workflows/verify_release_build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 6239c8937410..30911cff4078 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -30,6 +30,7 @@ jobs: echo "This task will create a PR against apache/beam." - name: Check out code uses: actions/checkout@v3 + with: ref: ${{ $RELEASE_BRANCH }} - name: Set Dataflow Python Batch Tests Release Version # Commenting for the moment: sh ./set_version.sh $RELEASE_VER --git-add instead of echo "This is a test." >> TEST_FILE.md From aa17ac54ccdd335b940a736d125bf0b6377a8e25 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 08:56:42 -0500 Subject: [PATCH 03/66] Removing blank spaces --- .github/workflows/verify_release_build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 30911cff4078..f7ce2f0b836b 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ $RELEASE_BRANCH }} - - name: Set Dataflow Python Batch Tests Release Version + - name: Set Dataflow Python Batch Tests Release Version # Commenting for the moment: sh ./set_version.sh $RELEASE_VER --git-add instead of echo "This is a test." >> TEST_FILE.md run: | git checkout -b $WORKING_BRANCH --quiet @@ -47,8 +47,7 @@ jobs: You can run many tests automatically using release/src/main/scripts/mass_comment.py." echo "[NOTE]: Please make sure all test targets have been invoked." echo "Please check the test results. If there is any failure, follow the policy in release guide." - working-directory: 'release/src/main/scripts' - + working-directory: "release/src/main/scripts" # trigger_phrases: # needs: create_testing_pr # runs-on: ubuntu-latest From 6be03a3c6faaa2bf8ec4cf814272d5c8603686da Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 12:04:34 -0500 Subject: [PATCH 04/66] Adding RELEASE_BRANCH env as GITHUB_ENV variable. --- .github/workflows/verify_release_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index f7ce2f0b836b..bb2802af6554 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -14,9 +14,9 @@ jobs: - name: Check environment id: check_env_vars env: - RELEASE_BRANCH: ${{ release-$RELEASE_VER }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + echo "RELEASE_BRANCH=release-$RELEASE_VER" >> $GITHUB_ENV echo "Verify release build against branch: $RELEASE_BRANCH." echo "Verify hub installation." hub version From ea954fa7857e71ba8e1af4da29fd4e5faee23d95 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 12:07:17 -0500 Subject: [PATCH 05/66] Fixed ref RELASE_BRANCH --- .github/workflows/verify_release_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index bb2802af6554..31c9e73f1a75 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -31,7 +31,7 @@ jobs: - name: Check out code uses: actions/checkout@v3 with: - ref: ${{ $RELEASE_BRANCH }} + ref: ${{ env.RELEASE_BRANCH }} - name: Set Dataflow Python Batch Tests Release Version # Commenting for the moment: sh ./set_version.sh $RELEASE_VER --git-add instead of echo "This is a test." >> TEST_FILE.md run: | From ebfb62cdfa59c3182bf5448e88421bbf8fa1d28d Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 12:37:20 -0500 Subject: [PATCH 06/66] Added RELEASE_ENV as step env var. Set github config user --- .github/workflows/verify_release_build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 31c9e73f1a75..313158d0c8c0 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -5,8 +5,10 @@ on: RELEASE_VER: description: Beam version of current release required: true + default: vtest # remove default before merging env: - WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? + #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? + WORKING_BRANCH: verify-release-workflow jobs: check_environment: runs-on: ubuntu-latest @@ -15,6 +17,7 @@ jobs: id: check_env_vars env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_ENV: ${{ github.event.inputs.RELEASE_VER }} run: | echo "RELEASE_BRANCH=release-$RELEASE_VER" >> $GITHUB_ENV echo "Verify release build against branch: $RELEASE_BRANCH." @@ -32,6 +35,9 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ env.RELEASE_BRANCH }} + - name: Set git config + run: | + git config user.name $GITHUB_ACTOR - name: Set Dataflow Python Batch Tests Release Version # Commenting for the moment: sh ./set_version.sh $RELEASE_VER --git-add instead of echo "This is a test." >> TEST_FILE.md run: | From 00ef3797b54bfcb2ae8a06e28a82ad038611e01d Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 12:50:52 -0500 Subject: [PATCH 07/66] Fixed RELEASE_VER env var. --- .github/workflows/verify_release_build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 313158d0c8c0..7fa228729301 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -8,7 +8,7 @@ on: default: vtest # remove default before merging env: #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? - WORKING_BRANCH: verify-release-workflow + WORKING_BRANCH: verify-release-pr-test jobs: check_environment: runs-on: ubuntu-latest @@ -17,13 +17,13 @@ jobs: id: check_env_vars env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RELEASE_ENV: ${{ github.event.inputs.RELEASE_VER }} + RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | echo "RELEASE_BRANCH=release-$RELEASE_VER" >> $GITHUB_ENV - echo "Verify release build against branch: $RELEASE_BRANCH." + echo "Verify release build against branch: "$RELEASE_BRANCH" ." echo "Verify hub installation." - hub version git version + hub version create_release_pr: needs: check_environment runs-on: ubuntu-latest From bb032b7bf704cc67d5d03e4e6f93f0ff3d92e2e2 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 15:33:52 -0500 Subject: [PATCH 08/66] Added write pull_request permission. --- .github/workflows/verify_release_build.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 7fa228729301..9a0365c58849 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -20,17 +20,21 @@ jobs: RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | echo "RELEASE_BRANCH=release-$RELEASE_VER" >> $GITHUB_ENV - echo "Verify release build against branch: "$RELEASE_BRANCH" ." + echo "Verify release build against branch: $RELEASE_BRANCH" echo "Verify hub installation." git version hub version + + # Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins. + # This create_release_pr task will create a PR against apache/beam. create_release_pr: needs: check_environment runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - run: | - echo "[Current Task] Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins." - echo "This task will create a PR against apache/beam." + echo "RELEASE BRANCH: $RELEASE_BRANCH" - name: Check out code uses: actions/checkout@v3 with: From 5b81aac982e540f3a45696892a3c55ea2396c74d Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 15:42:29 -0500 Subject: [PATCH 09/66] Added write contents permission. Print RELEASE_BRANCH for confirmation only. --- .github/workflows/verify_release_build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 9a0365c58849..331f91f4784b 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -19,8 +19,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | + echo "Set RELEASE_BRANCH environment variable" echo "RELEASE_BRANCH=release-$RELEASE_VER" >> $GITHUB_ENV - echo "Verify release build against branch: $RELEASE_BRANCH" echo "Verify hub installation." git version hub version @@ -32,9 +32,11 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write + contents: write steps: - - run: | - echo "RELEASE BRANCH: $RELEASE_BRANCH" + - name: Test RELEASE_BRANCH env var + if: ${{ env.RELEASE_BRANCH == 'release-vtest' }} + run: echo "Release Branch $RELEASE_BRANCH" - name: Check out code uses: actions/checkout@v3 with: From a655b1d25f731f56d1b61b97ca2e350decfd3bd5 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 15:57:33 -0500 Subject: [PATCH 10/66] Adding base and header in hub pull-request command. --- .github/workflows/verify_release_build.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 331f91f4784b..e048fbb87111 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -19,8 +19,11 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | - echo "Set RELEASE_BRANCH environment variable" + echo "Set RELEASE_BRANCH environment variable " + echo "${{ env.RELEASE_VER }}" + echo $GITHUB_ACTOR echo "RELEASE_BRANCH=release-$RELEASE_VER" >> $GITHUB_ENV + echo "${{ env.RELEASE_BRANCH }}" echo "Verify hub installation." git version hub version @@ -55,10 +58,7 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - hub pull-request -F- <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch - You can run many tests automatically using release/src/main/scripts/mass_comment.py." - echo "[NOTE]: Please make sure all test targets have been invoked." - echo "Please check the test results. If there is any failure, follow the policy in release guide." + hub pull-request -b dannymartinm:${{ env.RELEASE_BRANCH }} -h dannymartinm:$WORKING_BRANCH -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." working-directory: "release/src/main/scripts" # trigger_phrases: # needs: create_testing_pr @@ -79,3 +79,7 @@ jobs: # echo "==================== Final Cleanup ====================" # rm -rf $LOCAL_BEAM_DIR # echo "* Deleted workspace ${LOCAL_BEAM_DIR}" + + +# echo "[NOTE]: Please make sure all test targets have been invoked." +# echo "Please check the test results. If there is any failure, follow the policy in release guide." \ No newline at end of file From a5fbfa8c4fbcd677c0b6c0bf355b16c5cbd7d7ae Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 16:09:47 -0500 Subject: [PATCH 11/66] Using env.VAR_NAME convention. --- .github/workflows/verify_release_build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index e048fbb87111..6073ee53c1d3 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -13,6 +13,7 @@ jobs: check_environment: runs-on: ubuntu-latest steps: + - run: echo "RELEASE_BRANCH=${release-$RELEASE_VER}" >> $GITHUB_ENV - name: Check environment id: check_env_vars env: @@ -20,10 +21,8 @@ jobs: RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | echo "Set RELEASE_BRANCH environment variable " - echo "${{ env.RELEASE_VER }}" - echo $GITHUB_ACTOR - echo "RELEASE_BRANCH=release-$RELEASE_VER" >> $GITHUB_ENV echo "${{ env.RELEASE_BRANCH }}" + echo $GITHUB_ACTOR echo "Verify hub installation." git version hub version @@ -58,7 +57,7 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - hub pull-request -b dannymartinm:${{ env.RELEASE_BRANCH }} -h dannymartinm:$WORKING_BRANCH -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." + hub pull-request -b ${{env.GITHUB_ACTOR}}:${{ env.RELEASE_BRANCH }} -h ${{env.GITHUB_ACTOR}}:$WORKING_BRANCH -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." working-directory: "release/src/main/scripts" # trigger_phrases: # needs: create_testing_pr From 384a41dff2740fea0ee74332394a3c19ae0c57ac Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 16:25:38 -0500 Subject: [PATCH 12/66] Print variables --- .github/workflows/verify_release_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 6073ee53c1d3..9e41dad269b6 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -21,7 +21,7 @@ jobs: RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | echo "Set RELEASE_BRANCH environment variable " - echo "${{ env.RELEASE_BRANCH }}" + echo ${{env.RELEASE_BRANCH}} $RELEASE_BRANCH $RELEASE_VER echo $GITHUB_ACTOR echo "Verify hub installation." git version @@ -57,7 +57,7 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - hub pull-request -b ${{env.GITHUB_ACTOR}}:${{ env.RELEASE_BRANCH }} -h ${{env.GITHUB_ACTOR}}:$WORKING_BRANCH -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." + hub pull-request -b ${{env.GITHUB_ACTOR}}:${{env.RELEASE_BRANCH}} -h ${{env.GITHUB_ACTOR}}:${{env.$WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." working-directory: "release/src/main/scripts" # trigger_phrases: # needs: create_testing_pr From b47dce224775cc6a5516244e6e8f993f7b62e0a0 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 16:27:56 -0500 Subject: [PATCH 13/66] Fixing env var syntaxis issue issue --- .github/workflows/verify_release_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 9e41dad269b6..e504980347cd 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -57,7 +57,7 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - hub pull-request -b ${{env.GITHUB_ACTOR}}:${{env.RELEASE_BRANCH}} -h ${{env.GITHUB_ACTOR}}:${{env.$WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." + hub pull-request -b ${{env.GITHUB_ACTOR}}:${{env.RELEASE_BRANCH}} -h ${{env.GITHUB_ACTOR}}:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." working-directory: "release/src/main/scripts" # trigger_phrases: # needs: create_testing_pr From 1216f940483f989ccf757adef7e718fba7afb236 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 16:39:24 -0500 Subject: [PATCH 14/66] Adding missing GITHUB_TOKEN env var in create PR job. --- .github/workflows/verify_release_build.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index e504980347cd..fe89889d2768 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -13,14 +13,14 @@ jobs: check_environment: runs-on: ubuntu-latest steps: - - run: echo "RELEASE_BRANCH=${release-$RELEASE_VER}" >> $GITHUB_ENV + #- run: echo "RELEASE_BRANCH=${release-$RELEASE_VER}" >> $GITHUB_ENV - name: Check environment - id: check_env_vars env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | - echo "Set RELEASE_BRANCH environment variable " + echo "Set RELEASE_BRANCH env variable." + RELEASE_BRANCH=release-${{env.RELEASE_VER}} + echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV echo ${{env.RELEASE_BRANCH}} $RELEASE_BRANCH $RELEASE_VER echo $GITHUB_ACTOR echo "Verify hub installation." @@ -47,9 +47,11 @@ jobs: run: | git config user.name $GITHUB_ACTOR - name: Set Dataflow Python Batch Tests Release Version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Commenting for the moment: sh ./set_version.sh $RELEASE_VER --git-add instead of echo "This is a test." >> TEST_FILE.md run: | - git checkout -b $WORKING_BRANCH --quiet + git checkout -b ${{env.WORKING_BRANCH}} --quiet echo "This is a test." >> TEST_FILE.md git add TEST_FILE.md # In case the version string was not changed, append a newline to CHANGES.md From 6d5a9abfa7fd7a04df27c0d76ec4a8fda93581b1 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 2 Sep 2022 16:46:57 -0500 Subject: [PATCH 15/66] Moved set RELEASE_BRANCH to create PR job. --- .github/workflows/verify_release_build.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index fe89889d2768..c3aac21ed96f 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -18,11 +18,6 @@ jobs: env: RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | - echo "Set RELEASE_BRANCH env variable." - RELEASE_BRANCH=release-${{env.RELEASE_VER}} - echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV - echo ${{env.RELEASE_BRANCH}} $RELEASE_BRANCH $RELEASE_VER - echo $GITHUB_ACTOR echo "Verify hub installation." git version hub version @@ -36,6 +31,11 @@ jobs: pull-requests: write contents: write steps: + - name: Set RELEASE_BRANCH env variable + run: | + RELEASE_BRANCH=release-${{env.RELEASE_VER}} + echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV + echo "RELEASE BRANCH: ${{env.RELEASE_BRANCH}}" - name: Test RELEASE_BRANCH env var if: ${{ env.RELEASE_BRANCH == 'release-vtest' }} run: echo "Release Branch $RELEASE_BRANCH" @@ -59,7 +59,8 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - hub pull-request -b ${{env.GITHUB_ACTOR}}:${{env.RELEASE_BRANCH}} -h ${{env.GITHUB_ACTOR}}:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." + hub pull-request -b ${{env.GITHUB_ACTOR}}:${{env.RELEASE_BRANCH}} -h ${{env.GITHUB_ACTOR}}:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + You can run many tests automatically using release/src/main/scripts/mass_comment.py." working-directory: "release/src/main/scripts" # trigger_phrases: # needs: create_testing_pr From e6a26950caaa4de8d20ff60d0750ceec6b09a1cb Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 14:56:10 -0500 Subject: [PATCH 16/66] Print both env.RB and $RB. --- .github/workflows/verify_release_build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index c3aac21ed96f..243b9ad2782c 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -35,7 +35,8 @@ jobs: run: | RELEASE_BRANCH=release-${{env.RELEASE_VER}} echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV - echo "RELEASE BRANCH: ${{env.RELEASE_BRANCH}}" + echo ${{env.RELEASE_BRANCH}} + echo $RELEASE_BRANCH - name: Test RELEASE_BRANCH env var if: ${{ env.RELEASE_BRANCH == 'release-vtest' }} run: echo "Release Branch $RELEASE_BRANCH" @@ -59,7 +60,7 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - hub pull-request -b ${{env.GITHUB_ACTOR}}:${{env.RELEASE_BRANCH}} -h ${{env.GITHUB_ACTOR}}:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py." working-directory: "release/src/main/scripts" # trigger_phrases: From fecb0084f6121979cfbedb22b75557a05eb7ec48 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 14:59:07 -0500 Subject: [PATCH 17/66] Print both RELASE_BRANCH env var --- .github/workflows/verify_release_build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 243b9ad2782c..46d1033a17cd 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -33,10 +33,11 @@ jobs: steps: - name: Set RELEASE_BRANCH env variable run: | + echo "Set RELEASE_BRANCH env variable." RELEASE_BRANCH=release-${{env.RELEASE_VER}} echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV - echo ${{env.RELEASE_BRANCH}} - echo $RELEASE_BRANCH + echo ${{env.RELEASE_BRANCH}} $RELEASE_BRANCH $RELEASE_VER + echo $GITHUB_ACTOR - name: Test RELEASE_BRANCH env var if: ${{ env.RELEASE_BRANCH == 'release-vtest' }} run: echo "Release Branch $RELEASE_BRANCH" From 3f4e76d8e79c4582148bec050d1854ff37146347 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 15:03:36 -0500 Subject: [PATCH 18/66] Adding RELEASE_VER env var in create_release_ir step. --- .github/workflows/verify_release_build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 46d1033a17cd..2c5171e3b791 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -15,8 +15,6 @@ jobs: steps: #- run: echo "RELEASE_BRANCH=${release-$RELEASE_VER}" >> $GITHUB_ENV - name: Check environment - env: - RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | echo "Verify hub installation." git version @@ -32,12 +30,14 @@ jobs: contents: write steps: - name: Set RELEASE_BRANCH env variable + env: + RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | - echo "Set RELEASE_BRANCH env variable." RELEASE_BRANCH=release-${{env.RELEASE_VER}} echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV - echo ${{env.RELEASE_BRANCH}} $RELEASE_BRANCH $RELEASE_VER + echo ${{env.RELEASE_BRANCH}} echo $GITHUB_ACTOR + echo $RELEASE_BRANCH - name: Test RELEASE_BRANCH env var if: ${{ env.RELEASE_BRANCH == 'release-vtest' }} run: echo "Release Branch $RELEASE_BRANCH" From ed4c1a72915701134b06b98d53cc7cb67adf45cb Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 15:06:53 -0500 Subject: [PATCH 19/66] Adding RELEASE_VER env var in create_release_ir step. --- .github/workflows/verify_release_build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 2c5171e3b791..c68d17914470 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -8,7 +8,7 @@ on: default: vtest # remove default before merging env: #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? - WORKING_BRANCH: verify-release-pr-test + WORKING_BRANCH: 'verify-release-pr-test' jobs: check_environment: runs-on: ubuntu-latest @@ -28,10 +28,10 @@ jobs: permissions: pull-requests: write contents: write + env: + RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} steps: - name: Set RELEASE_BRANCH env variable - env: - RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} run: | RELEASE_BRANCH=release-${{env.RELEASE_VER}} echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV From d484deea0fc1c63918cdb0845011a51b5de3db22 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 15:23:07 -0500 Subject: [PATCH 20/66] Cleaned code --- .github/workflows/verify_release_build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index c68d17914470..87b110f2eeed 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -35,9 +35,6 @@ jobs: run: | RELEASE_BRANCH=release-${{env.RELEASE_VER}} echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV - echo ${{env.RELEASE_BRANCH}} - echo $GITHUB_ACTOR - echo $RELEASE_BRANCH - name: Test RELEASE_BRANCH env var if: ${{ env.RELEASE_BRANCH == 'release-vtest' }} run: echo "Release Branch $RELEASE_BRANCH" From 0909c4c06780d2c81e33fdcc852c2e7cc324cfa1 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 17:09:51 -0500 Subject: [PATCH 21/66] Adding run set_version script before creating the new PR. --- .github/workflows/verify_release_build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 87b110f2eeed..99ebdae1526d 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -13,7 +13,6 @@ jobs: check_environment: runs-on: ubuntu-latest steps: - #- run: echo "RELEASE_BRANCH=${release-$RELEASE_VER}" >> $GITHUB_ENV - name: Check environment run: | echo "Verify hub installation." @@ -48,9 +47,9 @@ jobs: - name: Set Dataflow Python Batch Tests Release Version env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Commenting for the moment: sh ./set_version.sh $RELEASE_VER --git-add instead of echo "This is a test." >> TEST_FILE.md run: | git checkout -b ${{env.WORKING_BRANCH}} --quiet + sh ./set_version.sh "$RELEASE_VER" --git-add echo "This is a test." >> TEST_FILE.md git add TEST_FILE.md # In case the version string was not changed, append a newline to CHANGES.md @@ -58,9 +57,9 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. - You can run many tests automatically using release/src/main/scripts/mass_comment.py." working-directory: "release/src/main/scripts" +# hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. +# You can run many tests automatically using release/src/main/scripts/mass_comment.py." # trigger_phrases: # needs: create_testing_pr # runs-on: ubuntu-latest From 5cb9b653abb3f8b1765c7cd9f3f7acda041ee99f Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 17:28:07 -0500 Subject: [PATCH 22/66] Run the script using bash to fix the ubuntu issue. Removed unused print step. --- .github/workflows/verify_release_build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 99ebdae1526d..15088df17de1 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -34,9 +34,6 @@ jobs: run: | RELEASE_BRANCH=release-${{env.RELEASE_VER}} echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV - - name: Test RELEASE_BRANCH env var - if: ${{ env.RELEASE_BRANCH == 'release-vtest' }} - run: echo "Release Branch $RELEASE_BRANCH" - name: Check out code uses: actions/checkout@v3 with: @@ -49,7 +46,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git checkout -b ${{env.WORKING_BRANCH}} --quiet - sh ./set_version.sh "$RELEASE_VER" --git-add + /bin/bash ./set_version.sh "$RELEASE_VER" --git-add echo "This is a test." >> TEST_FILE.md git add TEST_FILE.md # In case the version string was not changed, append a newline to CHANGES.md From e65b4ff2b162e06aad01aebcfd48db03eb82b7b4 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 18:03:53 -0500 Subject: [PATCH 23/66] Updating gradle.properties path --- release/src/main/scripts/set_version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/src/main/scripts/set_version.sh b/release/src/main/scripts/set_version.sh index 387dd03d69ac..1195ebdc3244 100755 --- a/release/src/main/scripts/set_version.sh +++ b/release/src/main/scripts/set_version.sh @@ -71,10 +71,10 @@ fi if [[ -z "$IS_SNAPSHOT_VERSION" ]] ; then # Fixing a release version - sed -i -e "s/version=.*/version=$TARGET_VERSION/" gradle.properties + sed -i -e "s/version=.*/version=$TARGET_VERSION/" beam/gradle.properties sed -i -e "s/project.version = .*/project.version = '$TARGET_VERSION'/" buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy sed -i -e "s/^__version__ = .*/__version__ = '${TARGET_VERSION}'/" sdks/python/apache_beam/version.py - sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION/" gradle.properties + sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION/" beam/gradle.properties sed -i -e "s/SdkVersion = .*/SdkVersion = \"$TARGET_VERSION\"/" sdks/go/pkg/beam/core/core.go else # For snapshot version: From 423aed956ce0cdf2639a80b60c7b6fcc60bcc342 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 5 Sep 2022 18:06:48 -0500 Subject: [PATCH 24/66] Updating gradle.properties path. --- release/src/main/scripts/set_version.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/src/main/scripts/set_version.sh b/release/src/main/scripts/set_version.sh index 1195ebdc3244..ab6217d25ef4 100755 --- a/release/src/main/scripts/set_version.sh +++ b/release/src/main/scripts/set_version.sh @@ -82,15 +82,15 @@ else # In the Gradle plugin, the -SNAPSHOT is dynamic so we don't add it here # Python appends .dev # The Dataflow container remains unchanged as in this case it is beam-master- form - sed -i -e "s/version=.*/version=$TARGET_VERSION-SNAPSHOT/" gradle.properties + sed -i -e "s/version=.*/version=$TARGET_VERSION-SNAPSHOT/" beam/gradle.properties sed -i -e "s/project.version = .*/project.version = '$TARGET_VERSION'/" buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy sed -i -e "s/^__version__ = .*/__version__ = '${TARGET_VERSION}.dev'/" sdks/python/apache_beam/version.py - sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION.dev/" gradle.properties + sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION.dev/" beam/gradle.properties sed -i -e "s/SdkVersion = .*/SdkVersion = \"${TARGET_VERSION}.dev\"/" sdks/go/pkg/beam/core/core.go fi if [[ "$GIT_ADD" == yes ]] ; then - git add gradle.properties + git add beam/gradle.properties git add buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy git add sdks/python/apache_beam/version.py git add sdks/go/pkg/beam/core/core.go From c87b59be5e9508cd02ad059ee573245413a7e883 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 6 Sep 2022 12:11:09 -0500 Subject: [PATCH 25/66] Passing GITHUB_WORKSPACE to set_version script. --- .github/workflows/verify_release_build.yml | 3 ++- release/src/main/scripts/set_version.sh | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 15088df17de1..ace0559e0262 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -34,6 +34,7 @@ jobs: run: | RELEASE_BRANCH=release-${{env.RELEASE_VER}} echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV + echo $GITHUB_WORKSPACE - name: Check out code uses: actions/checkout@v3 with: @@ -46,7 +47,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git checkout -b ${{env.WORKING_BRANCH}} --quiet - /bin/bash ./set_version.sh "$RELEASE_VER" --git-add + /bin/bash ./set_version.sh "$RELEASE_VER" --git-add "$GITHUB_WORKSPACE" echo "This is a test." >> TEST_FILE.md git add TEST_FILE.md # In case the version string was not changed, append a newline to CHANGES.md diff --git a/release/src/main/scripts/set_version.sh b/release/src/main/scripts/set_version.sh index ab6217d25ef4..26899fa6b334 100755 --- a/release/src/main/scripts/set_version.sh +++ b/release/src/main/scripts/set_version.sh @@ -30,6 +30,12 @@ function usage() { IS_SNAPSHOT_VERSION=yes GIT_ADD=no +ROOT_DIR="$3" + +echo "Print root dir:" +echo ROOT_DIR "$3" +echo "Print folders" +ls while [[ $# -gt 0 ]] ; do arg="$1" @@ -71,10 +77,10 @@ fi if [[ -z "$IS_SNAPSHOT_VERSION" ]] ; then # Fixing a release version - sed -i -e "s/version=.*/version=$TARGET_VERSION/" beam/gradle.properties + sed -i -e "s/version=.*/version=$TARGET_VERSION/" "$ROOT_DIR"/gradle.properties sed -i -e "s/project.version = .*/project.version = '$TARGET_VERSION'/" buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy sed -i -e "s/^__version__ = .*/__version__ = '${TARGET_VERSION}'/" sdks/python/apache_beam/version.py - sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION/" beam/gradle.properties + sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION/" "$ROOT_DIR"/gradle.properties sed -i -e "s/SdkVersion = .*/SdkVersion = \"$TARGET_VERSION\"/" sdks/go/pkg/beam/core/core.go else # For snapshot version: @@ -82,15 +88,15 @@ else # In the Gradle plugin, the -SNAPSHOT is dynamic so we don't add it here # Python appends .dev # The Dataflow container remains unchanged as in this case it is beam-master- form - sed -i -e "s/version=.*/version=$TARGET_VERSION-SNAPSHOT/" beam/gradle.properties + sed -i -e "s/version=.*/version=$TARGET_VERSION-SNAPSHOT/" "$ROOT_DIR"/gradle.properties sed -i -e "s/project.version = .*/project.version = '$TARGET_VERSION'/" buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy sed -i -e "s/^__version__ = .*/__version__ = '${TARGET_VERSION}.dev'/" sdks/python/apache_beam/version.py - sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION.dev/" beam/gradle.properties + sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION.dev/" "$ROOT_DIR"/gradle.properties sed -i -e "s/SdkVersion = .*/SdkVersion = \"${TARGET_VERSION}.dev\"/" sdks/go/pkg/beam/core/core.go fi if [[ "$GIT_ADD" == yes ]] ; then - git add beam/gradle.properties + git add "$ROOT_DIR"/gradle.properties git add buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy git add sdks/python/apache_beam/version.py git add sdks/go/pkg/beam/core/core.go From 065995b1d726fde72a0bd23c849c8887d9e82902 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 6 Sep 2022 12:32:41 -0500 Subject: [PATCH 26/66] Removing working-directory in workflow and update set_version path instead. --- .github/workflows/verify_release_build.yml | 4 ++-- release/src/main/scripts/set_version.sh | 16 +++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index ace0559e0262..4269117a2300 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -47,7 +47,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git checkout -b ${{env.WORKING_BRANCH}} --quiet - /bin/bash ./set_version.sh "$RELEASE_VER" --git-add "$GITHUB_WORKSPACE" + /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add echo "This is a test." >> TEST_FILE.md git add TEST_FILE.md # In case the version string was not changed, append a newline to CHANGES.md @@ -55,7 +55,7 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - working-directory: "release/src/main/scripts" + #working-directory: "release/src/main/scripts" # hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. # You can run many tests automatically using release/src/main/scripts/mass_comment.py." # trigger_phrases: diff --git a/release/src/main/scripts/set_version.sh b/release/src/main/scripts/set_version.sh index 26899fa6b334..387dd03d69ac 100755 --- a/release/src/main/scripts/set_version.sh +++ b/release/src/main/scripts/set_version.sh @@ -30,12 +30,6 @@ function usage() { IS_SNAPSHOT_VERSION=yes GIT_ADD=no -ROOT_DIR="$3" - -echo "Print root dir:" -echo ROOT_DIR "$3" -echo "Print folders" -ls while [[ $# -gt 0 ]] ; do arg="$1" @@ -77,10 +71,10 @@ fi if [[ -z "$IS_SNAPSHOT_VERSION" ]] ; then # Fixing a release version - sed -i -e "s/version=.*/version=$TARGET_VERSION/" "$ROOT_DIR"/gradle.properties + sed -i -e "s/version=.*/version=$TARGET_VERSION/" gradle.properties sed -i -e "s/project.version = .*/project.version = '$TARGET_VERSION'/" buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy sed -i -e "s/^__version__ = .*/__version__ = '${TARGET_VERSION}'/" sdks/python/apache_beam/version.py - sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION/" "$ROOT_DIR"/gradle.properties + sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION/" gradle.properties sed -i -e "s/SdkVersion = .*/SdkVersion = \"$TARGET_VERSION\"/" sdks/go/pkg/beam/core/core.go else # For snapshot version: @@ -88,15 +82,15 @@ else # In the Gradle plugin, the -SNAPSHOT is dynamic so we don't add it here # Python appends .dev # The Dataflow container remains unchanged as in this case it is beam-master- form - sed -i -e "s/version=.*/version=$TARGET_VERSION-SNAPSHOT/" "$ROOT_DIR"/gradle.properties + sed -i -e "s/version=.*/version=$TARGET_VERSION-SNAPSHOT/" gradle.properties sed -i -e "s/project.version = .*/project.version = '$TARGET_VERSION'/" buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy sed -i -e "s/^__version__ = .*/__version__ = '${TARGET_VERSION}.dev'/" sdks/python/apache_beam/version.py - sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION.dev/" "$ROOT_DIR"/gradle.properties + sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION.dev/" gradle.properties sed -i -e "s/SdkVersion = .*/SdkVersion = \"${TARGET_VERSION}.dev\"/" sdks/go/pkg/beam/core/core.go fi if [[ "$GIT_ADD" == yes ]] ; then - git add "$ROOT_DIR"/gradle.properties + git add gradle.properties git add buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy git add sdks/python/apache_beam/version.py git add sdks/go/pkg/beam/core/core.go From 040523e687b42b50e740cbcb8fec21c65e2a0275 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 6 Sep 2022 16:04:39 -0500 Subject: [PATCH 27/66] Set new GITHUB_PR as env variable. --- .github/workflows/verify_release_build.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 4269117a2300..cdcdaf5c37f5 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -34,7 +34,6 @@ jobs: run: | RELEASE_BRANCH=release-${{env.RELEASE_VER}} echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV - echo $GITHUB_WORKSPACE - name: Check out code uses: actions/checkout@v3 with: @@ -48,16 +47,14 @@ jobs: run: | git checkout -b ${{env.WORKING_BRANCH}} --quiet /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add - echo "This is a test." >> TEST_FILE.md - git add TEST_FILE.md # In case the version string was not changed, append a newline to CHANGES.md echo "" >> CHANGES.md git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - #working-directory: "release/src/main/scripts" -# hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. -# You can run many tests automatically using release/src/main/scripts/mass_comment.py." + GITHUB_PR=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + You can run many tests automatically using release/src/main/scripts/mass_comment.py.") + echo $GITHUB_PR # trigger_phrases: # needs: create_testing_pr # runs-on: ubuntu-latest From 21b6067d8fdefb4cb8d53ed1a86567e2abfa5083 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 6 Sep 2022 16:45:05 -0500 Subject: [PATCH 28/66] Get the GH_PR_NUMBER and set it as env variable. Create PR comment via GH cli. --- .github/workflows/verify_release_build.yml | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index cdcdaf5c37f5..d0020b38ff3d 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -52,18 +52,24 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - GITHUB_PR=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") - echo $GITHUB_PR -# trigger_phrases: -# needs: create_testing_pr -# runs-on: ubuntu-latest -# steps: -# - name: Trigger beam_Release_Gradle_Build and PostCommit jobs # Pending to include tests migrated to GH Actions (by Wizeline team) -# run: | -# sh ./mass_comment.py -# working-directory: 'release/src/main/scripts' - + echo "GitHub PR URL:" + echo $GITHUB_PR_URL + GITHUB_PR_NUMBER=$(echo $GITHUB_PR_URL |cut -d"/" -f7) + echo $GITHUB_PR_NUMBER + echo "$GITHUB_PR_NUMBER" >> $GITHUB_ENV +comment_trigger_phrases: + needs: create_testing_pr + runs-on: ubuntu-latest + steps: + - run: | + echo ${{env.GITHUB_PR_NUMBER}} + echo 'Test' + echo $GITHUB_PR_NUMBER + gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Commenting this clean-up step 'cause I think this step is not needed, as the final self-hosted implementation has ephemeral runners. # clean-up: # if: always() From e408983954dd1abdf0a33aa76e8c8feefdf88cdb Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 6 Sep 2022 16:46:51 -0500 Subject: [PATCH 29/66] Fixing yml issue. --- .github/workflows/verify_release_build.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index d0020b38ff3d..db986a6c18f6 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -63,11 +63,12 @@ comment_trigger_phrases: needs: create_testing_pr runs-on: ubuntu-latest steps: - - run: | - echo ${{env.GITHUB_PR_NUMBER}} - echo 'Test' - echo $GITHUB_PR_NUMBER - gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." + - name: Create PR Comment + run: | + echo ${{env.GITHUB_PR_NUMBER}} + echo 'Test' + echo $GITHUB_PR_NUMBER + gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Commenting this clean-up step 'cause I think this step is not needed, as the final self-hosted implementation has ephemeral runners. From 4aef49436989893e7c6df96e057f0a0d6ecfd3aa Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 6 Sep 2022 16:47:59 -0500 Subject: [PATCH 30/66] Fixing yml format issue. --- .github/workflows/verify_release_build.yml | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index db986a6c18f6..cae7cc6cfca6 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -59,18 +59,18 @@ jobs: GITHUB_PR_NUMBER=$(echo $GITHUB_PR_URL |cut -d"/" -f7) echo $GITHUB_PR_NUMBER echo "$GITHUB_PR_NUMBER" >> $GITHUB_ENV -comment_trigger_phrases: - needs: create_testing_pr - runs-on: ubuntu-latest - steps: - - name: Create PR Comment - run: | - echo ${{env.GITHUB_PR_NUMBER}} - echo 'Test' - echo $GITHUB_PR_NUMBER - gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + comment_trigger_phrases: + needs: create_testing_pr + runs-on: ubuntu-latest + steps: + - name: Create PR Comment + run: | + echo ${{env.GITHUB_PR_NUMBER}} + echo 'Test' + echo $GITHUB_PR_NUMBER + gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Commenting this clean-up step 'cause I think this step is not needed, as the final self-hosted implementation has ephemeral runners. # clean-up: # if: always() From 2c66321c4c5e6a5b615d52f33a74bb9721b4a7b3 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 6 Sep 2022 16:49:00 -0500 Subject: [PATCH 31/66] comment_trigger_phrases step needs create_release_pr to run. --- .github/workflows/verify_release_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index cae7cc6cfca6..073fed68b292 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -60,7 +60,7 @@ jobs: echo $GITHUB_PR_NUMBER echo "$GITHUB_PR_NUMBER" >> $GITHUB_ENV comment_trigger_phrases: - needs: create_testing_pr + needs: create_release_pr runs-on: ubuntu-latest steps: - name: Create PR Comment From 6f7be00a9df3220d32aa0aefd86a3caeeecf66e8 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 6 Sep 2022 16:51:19 -0500 Subject: [PATCH 32/66] Fixed set GITHUB_PR_NUMBER GITHUB_ENV var. --- .github/workflows/verify_release_build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 073fed68b292..4c39795d9ac0 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -54,11 +54,9 @@ jobs: git push origin $WORKING_BRANCH --quiet GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") - echo "GitHub PR URL:" - echo $GITHUB_PR_URL GITHUB_PR_NUMBER=$(echo $GITHUB_PR_URL |cut -d"/" -f7) + echo "GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER}" >> $GITHUB_ENV echo $GITHUB_PR_NUMBER - echo "$GITHUB_PR_NUMBER" >> $GITHUB_ENV comment_trigger_phrases: needs: create_release_pr runs-on: ubuntu-latest From 3688e01add82935af3ca9a3d1608d3e74ed96dc4 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 7 Sep 2022 12:07:36 -0500 Subject: [PATCH 33/66] Adding PR Number as an output. --- .github/workflows/verify_release_build.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 4c39795d9ac0..2687f0026534 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -21,7 +21,7 @@ jobs: # Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins. # This create_release_pr task will create a PR against apache/beam. - create_release_pr: + update_create_release_pr: needs: check_environment runs-on: ubuntu-latest permissions: @@ -52,23 +52,32 @@ jobs: git add CHANGES.md git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet + - name: Create new Release PR + id: create_pr + run: | GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") GITHUB_PR_NUMBER=$(echo $GITHUB_PR_URL |cut -d"/" -f7) echo "GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER}" >> $GITHUB_ENV echo $GITHUB_PR_NUMBER + echo "::set-output name=git_pr_number::${GITHUB_PR_NUMBER}" + outputs: + git_pr_number: ${{ steps.create_pr.outputs.git_pr_number }} comment_trigger_phrases: - needs: create_release_pr + needs: update_create_release_pr runs-on: ubuntu-latest steps: - name: Create PR Comment run: | - echo ${{env.GITHUB_PR_NUMBER}} + echo ${{ env.GITHUB_PR_NUMBER }} echo 'Test' - echo $GITHUB_PR_NUMBER - gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." + echo ${{ needs.update_create_release_pr.outputs.git_pr_number }} + echo 'Test 2' + echo ${{ env.GH_PR_NUMBER}} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_PR_NUMBER: ${{ needs.create_release_pr.outputs.git_pr_number }} +# gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." # Commenting this clean-up step 'cause I think this step is not needed, as the final self-hosted implementation has ephemeral runners. # clean-up: # if: always() From 577666a6f41e948df081a47f56e2deccb5ab226c Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 7 Sep 2022 12:11:07 -0500 Subject: [PATCH 34/66] Fixed lint yml issue --- .github/workflows/verify_release_build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 2687f0026534..78a483dac945 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -18,7 +18,6 @@ jobs: echo "Verify hub installation." git version hub version - # Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins. # This create_release_pr task will create a PR against apache/beam. update_create_release_pr: @@ -61,8 +60,8 @@ jobs: echo "GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER}" >> $GITHUB_ENV echo $GITHUB_PR_NUMBER echo "::set-output name=git_pr_number::${GITHUB_PR_NUMBER}" - outputs: - git_pr_number: ${{ steps.create_pr.outputs.git_pr_number }} + outputs: + git_pr_number: ${{ steps.create_pr.outputs.git_pr_number }} comment_trigger_phrases: needs: update_create_release_pr runs-on: ubuntu-latest From b8e1d93c6fdd194d026165387ced53205e1561b3 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 7 Sep 2022 12:22:16 -0500 Subject: [PATCH 35/66] Passing GH_TOKEN to hub. --- .github/workflows/verify_release_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 78a483dac945..59013c95c443 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -41,8 +41,6 @@ jobs: run: | git config user.name $GITHUB_ACTOR - name: Set Dataflow Python Batch Tests Release Version - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git checkout -b ${{env.WORKING_BRANCH}} --quiet /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add @@ -53,6 +51,8 @@ jobs: git push origin $WORKING_BRANCH --quiet - name: Create new Release PR id: create_pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") From 6f172df5ce754c9297324e0cc8206506960eea11 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 7 Sep 2022 12:25:02 -0500 Subject: [PATCH 36/66] Fixed env vars in comment step. --- .github/workflows/verify_release_build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 59013c95c443..93050cf7432f 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -73,9 +73,11 @@ jobs: echo ${{ needs.update_create_release_pr.outputs.git_pr_number }} echo 'Test 2' echo ${{ env.GH_PR_NUMBER}} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_PR_NUMBER: ${{ needs.create_release_pr.outputs.git_pr_number }} + echo 'Test 3' + echo $GH_PR_NUMBER + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_PR_NUMBER: ${{ needs.create_release_pr.outputs.git_pr_number }} # gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." # Commenting this clean-up step 'cause I think this step is not needed, as the final self-hosted implementation has ephemeral runners. # clean-up: From c4b3a5c408c5ed02c735b1e7f2a4982dd2bf23eb Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 7 Sep 2022 12:32:08 -0500 Subject: [PATCH 37/66] Added command to comment new pr --- .github/workflows/verify_release_build.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 93050cf7432f..6d9593271cf9 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -58,7 +58,6 @@ jobs: You can run many tests automatically using release/src/main/scripts/mass_comment.py.") GITHUB_PR_NUMBER=$(echo $GITHUB_PR_URL |cut -d"/" -f7) echo "GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER}" >> $GITHUB_ENV - echo $GITHUB_PR_NUMBER echo "::set-output name=git_pr_number::${GITHUB_PR_NUMBER}" outputs: git_pr_number: ${{ steps.create_pr.outputs.git_pr_number }} @@ -68,17 +67,12 @@ jobs: steps: - name: Create PR Comment run: | - echo ${{ env.GITHUB_PR_NUMBER }} - echo 'Test' - echo ${{ needs.update_create_release_pr.outputs.git_pr_number }} - echo 'Test 2' - echo ${{ env.GH_PR_NUMBER}} - echo 'Test 3' - echo $GH_PR_NUMBER + GITHUB_PR_NUMBER=${{ needs.update_create_release_pr.outputs.git_pr_number }} + echo $GITHUB_PR_NUMBER + gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_PR_NUMBER: ${{ needs.create_release_pr.outputs.git_pr_number }} -# gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." +# # Commenting this clean-up step 'cause I think this step is not needed, as the final self-hosted implementation has ephemeral runners. # clean-up: # if: always() From e3ad54fe4b528da74ec8ee0bf4077df99ae62fef Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 7 Sep 2022 17:13:12 -0500 Subject: [PATCH 38/66] Updated gh pr command and passing the whole PR URL. --- .github/workflows/verify_release_build.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 6d9593271cf9..aa01b6c5064a 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -56,20 +56,17 @@ jobs: run: | GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") - GITHUB_PR_NUMBER=$(echo $GITHUB_PR_URL |cut -d"/" -f7) - echo "GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER}" >> $GITHUB_ENV - echo "::set-output name=git_pr_number::${GITHUB_PR_NUMBER}" + echo "::set-output name=git_pr::${GITHUB_PR_URL}" outputs: - git_pr_number: ${{ steps.create_pr.outputs.git_pr_number }} + git_pr: ${{ steps.create_pr.outputs.git_pr }} comment_trigger_phrases: needs: update_create_release_pr runs-on: ubuntu-latest steps: - name: Create PR Comment run: | - GITHUB_PR_NUMBER=${{ needs.update_create_release_pr.outputs.git_pr_number }} - echo $GITHUB_PR_NUMBER - gh pr comment $GITHUB_PR_NUMBER --body "Hi! This is a test from Verify Release Workflow." + GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} + gh pr comment $GITHUB_PR --body "Hi! This is a test from Verify Release Workflow." env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # From b1eef0d95e43d0f8d86435dbfdcce4e98bc05b9d Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 7 Sep 2022 17:16:25 -0500 Subject: [PATCH 39/66] Adding pull-requests write permission to comment_trigger_phrases job. --- .github/workflows/verify_release_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index aa01b6c5064a..56d9126e1f18 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -62,6 +62,8 @@ jobs: comment_trigger_phrases: needs: update_create_release_pr runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: Create PR Comment run: | From ede162b4711ac9fec75ad22d3fc2c23bde8fbfc4 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 9 Sep 2022 17:23:15 -0500 Subject: [PATCH 40/66] Run the comment_pr_trigger_phrases.sh script to comment the PR with the Jenkins Trigger Phrases. --- .github/workflows/verify_release_build.yml | 23 ++---- .../ci/release/comment_pr_trigger_phrases.sh | 9 +++ scripts/ci/release/example.txt | 4 + .../ci/release/jenkins_trigger_phrases.txt | 77 +++++++++++++++++++ 4 files changed, 97 insertions(+), 16 deletions(-) create mode 100755 scripts/ci/release/comment_pr_trigger_phrases.sh create mode 100644 scripts/ci/release/example.txt create mode 100644 scripts/ci/release/jenkins_trigger_phrases.txt diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 56d9126e1f18..c4cc04bec25b 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -6,6 +6,10 @@ on: description: Beam version of current release required: true default: vtest # remove default before merging + COMMENT_TRIGGER_PHRASES: + description: Flag to comment the Jenkins Trigger Phrases in the new PR + required: true + default: false env: #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? WORKING_BRANCH: 'verify-release-pr-test' @@ -60,6 +64,7 @@ jobs: outputs: git_pr: ${{ steps.create_pr.outputs.git_pr }} comment_trigger_phrases: + if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES }} needs: update_create_release_pr runs-on: ubuntu-latest permissions: @@ -68,21 +73,7 @@ jobs: - name: Create PR Comment run: | GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} - gh pr comment $GITHUB_PR --body "Hi! This is a test from Verify Release Workflow." + sh ./comment_pr_trigger_phrases.sh $GITHUB_PR env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# -# Commenting this clean-up step 'cause I think this step is not needed, as the final self-hosted implementation has ephemeral runners. -# clean-up: -# if: always() -# runs-on: ubuntu-latest -# steps: -# - name: Clean Temporal Directory -# run: | -# echo "==================== Final Cleanup ====================" -# rm -rf $LOCAL_BEAM_DIR -# echo "* Deleted workspace ${LOCAL_BEAM_DIR}" - - -# echo "[NOTE]: Please make sure all test targets have been invoked." -# echo "Please check the test results. If there is any failure, follow the policy in release guide." \ No newline at end of file + working-directory: scripts/ci/release diff --git a/scripts/ci/release/comment_pr_trigger_phrases.sh b/scripts/ci/release/comment_pr_trigger_phrases.sh new file mode 100755 index 000000000000..c7db76a783d6 --- /dev/null +++ b/scripts/ci/release/comment_pr_trigger_phrases.sh @@ -0,0 +1,9 @@ +#!/bin/bash +#file="./jenkins_trigger_phrases.txt" +file="./example.txt" +GITHUB_PR=$1 +while IFS= read -r trigger_phrase +do + #printf '%s\n' "$trigger_phrase" + gh pr comment "$GITHUB_PR" --body "$trigger_phrase" +done <"$file" \ No newline at end of file diff --git a/scripts/ci/release/example.txt b/scripts/ci/release/example.txt new file mode 100644 index 000000000000..f378a59a93b3 --- /dev/null +++ b/scripts/ci/release/example.txt @@ -0,0 +1,4 @@ +Run PythonFormatter PreCommit +Run PythonLint PreCommit +Run RAT PreCommit +Run Whitespace PreCommit \ No newline at end of file diff --git a/scripts/ci/release/jenkins_trigger_phrases.txt b/scripts/ci/release/jenkins_trigger_phrases.txt new file mode 100644 index 000000000000..e4884c76ded5 --- /dev/null +++ b/scripts/ci/release/jenkins_trigger_phrases.txt @@ -0,0 +1,77 @@ +Run CommunityMetrics PreCommit +Run Dataflow Runner Nexmark Tests +Run Dataflow Runner V2 Java 11 Nexmark Tests +Run Dataflow Runner V2 Java 17 Nexmark Tests +Run Dataflow Runner V2 Nexmark Tests +Run Dataflow Streaming ValidatesRunner +Run Dataflow ValidatesRunner Java 11 +Run Dataflow ValidatesRunner Java 17 +Run Dataflow ValidatesRunner +Run Direct Runner Nexmark Tests +Run Direct ValidatesRunner Java 11 +Run Direct ValidatesRunner Java 17 +Run Direct ValidatesRunner in Java 11 +Run Direct ValidatesRunner +Run Flink Runner Nexmark Tests +Run Flink ValidatesRunner Java 11 +Run Flink ValidatesRunner +Run Go Flink ValidatesRunner +Run Go PreCommit +Run Go Samza ValidatesRunner +Run Go Spark ValidatesRunner +Run GoPortable PreCommit +Run Java 11 Examples on Dataflow Runner V2 +Run Java 17 Examples on Dataflow Runner V2 +Run Java Dataflow V2 ValidatesRunner Streaming +Run Java Dataflow V2 ValidatesRunner +Run Java Examples on Dataflow Runner V2 +Run Java Examples_Direct +Run Java Examples_Flink +Run Java Examples_Spark +Run Java Flink PortableValidatesRunner Streaming +Run Java Portability examples on Dataflow with Java 11 +Run Java PreCommit +Run Java Samza PortableValidatesRunner +Run Java Spark PortableValidatesRunner Batch +Run Java Spark v2 PortableValidatesRunner Streaming +Run Java Spark v3 PortableValidatesRunner Streaming +Run Java examples on Dataflow Java 11 +Run Java examples on Dataflow Java 17 +Run Java examples on Dataflow with Java 11 +Run Java_Examples_Dataflow PreCommit +Run Java_Examples_Dataflow_Java11 PreCommit +Run Java_Examples_Dataflow_Java17 PreCommit +Run Java_PVR_Flink_Batch PreCommit +Run Java_PVR_Flink_Docker PreCommit +Run Portable_Python PreCommit +Run Python Dataflow V2 ValidatesRunner +Run Python Dataflow ValidatesContainer +Run Python Dataflow ValidatesRunner +Run Python Examples_Dataflow +Run Python Examples_Direct +Run Python Examples_Flink +Run Python Examples_Spark +Run Python Flink ValidatesRunner +Run Python PreCommit +Run Python Samza ValidatesRunner +Run Python Spark ValidatesRunner +Run PythonDocker PreCommit +Run PythonDocs PreCommit +Run PythonFormatter PreCommit +Run PythonLint PreCommit +Run Python_PVR_Flink PreCommit +Run RAT PreCommit +Run Release Gradle Build +Run SQL PreCommit +Run SQL_Java11 PreCommit +Run SQL_Java17 PreCommit +Run Samza ValidatesRunner +Run Spark Runner Nexmark Tests +Run Spark StructuredStreaming ValidatesRunner +Run Spark ValidatesRunner Java 11 +Run Spark ValidatesRunner +Run Spotless PreCommit +Run Twister2 ValidatesRunner +Run Typescript PreCommit +Run ULR Loopback ValidatesRunner +Run Whitespace PreCommit \ No newline at end of file From 18f4b29275b0333f437e650537e51a47a56a8f97 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 15 Sep 2022 12:23:51 -0500 Subject: [PATCH 41/66] List the files in working-directory for testing --- .github/workflows/verify_release_build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index c4cc04bec25b..d718f59ee7ad 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -64,7 +64,7 @@ jobs: outputs: git_pr: ${{ steps.create_pr.outputs.git_pr }} comment_trigger_phrases: - if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES }} + if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} needs: update_create_release_pr runs-on: ubuntu-latest permissions: @@ -73,7 +73,8 @@ jobs: - name: Create PR Comment run: | GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} - sh ./comment_pr_trigger_phrases.sh $GITHUB_PR + ls -la env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} working-directory: scripts/ci/release +#sh ./comment_pr_trigger_phrases.sh $GITHUB_PR \ No newline at end of file From 147fc7d0fd6b906290764e582f8f8a7f093c46d6 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 15 Sep 2022 13:08:08 -0500 Subject: [PATCH 42/66] Fixing linter issue in working-directory --- .github/workflows/verify_release_build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index d718f59ee7ad..cbe70ca464ac 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -73,8 +73,7 @@ jobs: - name: Create PR Comment run: | GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} - ls -la + sh ./comment_pr_trigger_phrases.sh $GITHUB_PR env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: scripts/ci/release -#sh ./comment_pr_trigger_phrases.sh $GITHUB_PR \ No newline at end of file + working-directory: 'scripts/ci/release' From 78d2a2bc5454ce10e9f1841fbda319d8accd82f5 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 19 Sep 2022 12:46:36 -0500 Subject: [PATCH 43/66] Moved comment_trigger_Phrases job into the update_create_release_pr job. --- .github/workflows/verify_release_build.yml | 42 ++++++++++++---------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index cbe70ca464ac..358c1a43ac9b 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -9,7 +9,7 @@ on: COMMENT_TRIGGER_PHRASES: description: Flag to comment the Jenkins Trigger Phrases in the new PR required: true - default: false + default: 'false' env: #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? WORKING_BRANCH: 'verify-release-pr-test' @@ -60,20 +60,26 @@ jobs: run: | GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") - echo "::set-output name=git_pr::${GITHUB_PR_URL}" - outputs: - git_pr: ${{ steps.create_pr.outputs.git_pr }} - comment_trigger_phrases: - if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} - needs: update_create_release_pr - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - name: Create PR Comment - run: | - GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} - sh ./comment_pr_trigger_phrases.sh $GITHUB_PR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: 'scripts/ci/release' + - name: Comment PR Trigger Phrases + if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} + run: | + sh ./comment_pr_trigger_phrases.sh $GITHUB_PR_URL + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: 'scripts/ci/release' +# outputs: +# git_pr: ${{ steps.create_pr.outputs.git_pr }} +# comment_trigger_phrases: +# if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} +# needs: update_create_release_pr +# runs-on: ubuntu-latest +# permissions: +# pull-requests: write +# steps: +# - name: Create PR Comment +# run: | +# GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} +# sh ./comment_pr_trigger_phrases.sh $GITHUB_PR +# env: +# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# working-directory: 'scripts/ci/release' From 2982e3022aca926a4419d037897d0e4f664d4d83 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 19 Sep 2022 13:00:05 -0500 Subject: [PATCH 44/66] Added checkout step in comment_trigger_phrases job. --- .github/workflows/verify_release_build.yml | 42 ++++++++++------------ 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 358c1a43ac9b..8109c11423d6 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -60,26 +60,22 @@ jobs: run: | GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") - - name: Comment PR Trigger Phrases - if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} - run: | - sh ./comment_pr_trigger_phrases.sh $GITHUB_PR_URL - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: 'scripts/ci/release' -# outputs: -# git_pr: ${{ steps.create_pr.outputs.git_pr }} -# comment_trigger_phrases: -# if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} -# needs: update_create_release_pr -# runs-on: ubuntu-latest -# permissions: -# pull-requests: write -# steps: -# - name: Create PR Comment -# run: | -# GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} -# sh ./comment_pr_trigger_phrases.sh $GITHUB_PR -# env: -# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# working-directory: 'scripts/ci/release' + echo "::set-output name=git_pr::${GITHUB_PR_URL}" + outputs: + git_pr: ${{ steps.create_pr.outputs.git_pr }} + comment_trigger_phrases: + if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} + needs: update_create_release_pr + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Check out code + uses: actions/checkout@v3 + - name: Create PR Comment + run: | + GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} + sh ./comment_pr_trigger_phrases.sh $GITHUB_PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: 'scripts/ci/release' From cc9e426bd97b968da0d0e60458f5ad69545b31da Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 19 Sep 2022 15:41:31 -0500 Subject: [PATCH 45/66] Check out code from RELEASE_BRANCH in Commenting Trigger Phrases. --- .github/workflows/verify_release_build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 8109c11423d6..416731cb68c4 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -72,7 +72,9 @@ jobs: steps: - name: Check out code uses: actions/checkout@v3 - - name: Create PR Comment + with: + ref: ${{ env.RELEASE_BRANCH }} + - name: Comment Trigger Phrases in PR run: | GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} sh ./comment_pr_trigger_phrases.sh $GITHUB_PR From 6640399fdaeb66d95433594e9cc6910a8a0a11b2 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 20 Sep 2022 09:57:51 -0500 Subject: [PATCH 46/66] Adding, Install hub cli tool step --- .github/workflows/verify_release_build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 416731cb68c4..853a3bd33404 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -17,10 +17,9 @@ jobs: check_environment: runs-on: ubuntu-latest steps: - - name: Check environment + - name: Install hub cli tool run: | - echo "Verify hub installation." - git version + sudo apt-get update && sudo apt-get install -y hub hub version # Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins. # This create_release_pr task will create a PR against apache/beam. From 45c0c6a0b97f19221bc9e6cd5565f9c95b6eb1a9 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 20 Sep 2022 12:02:54 -0500 Subject: [PATCH 47/66] Print gh actions variables --- .github/workflows/verify_release_build.yml | 80 ++++++++++++---------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 853a3bd33404..b17fd92a8ab1 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -42,41 +42,45 @@ jobs: ref: ${{ env.RELEASE_BRANCH }} - name: Set git config run: | - git config user.name $GITHUB_ACTOR - - name: Set Dataflow Python Batch Tests Release Version - run: | - git checkout -b ${{env.WORKING_BRANCH}} --quiet - /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add - # In case the version string was not changed, append a newline to CHANGES.md - echo "" >> CHANGES.md - git add CHANGES.md - git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet - git push origin $WORKING_BRANCH --quiet - - name: Create new Release PR - id: create_pr - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. - You can run many tests automatically using release/src/main/scripts/mass_comment.py.") - echo "::set-output name=git_pr::${GITHUB_PR_URL}" - outputs: - git_pr: ${{ steps.create_pr.outputs.git_pr }} - comment_trigger_phrases: - if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} - needs: update_create_release_pr - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - ref: ${{ env.RELEASE_BRANCH }} - - name: Comment Trigger Phrases in PR - run: | - GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} - sh ./comment_pr_trigger_phrases.sh $GITHUB_PR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: 'scripts/ci/release' + echo $GITHUB_ACTOR + echo $RUNNER_NAME + echo $RUNNER_OS + +#git config user.name $GITHUB_ACTOR +# - name: Set Dataflow Python Batch Tests Release Version +# run: | +# git checkout -b ${{env.WORKING_BRANCH}} --quiet +# /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add +# # In case the version string was not changed, append a newline to CHANGES.md +# echo "" >> CHANGES.md +# git add CHANGES.md +# git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet +# git push origin $WORKING_BRANCH --quiet +# - name: Create new Release PR +# id: create_pr +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# run: | +# GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. +# You can run many tests automatically using release/src/main/scripts/mass_comment.py.") +# echo "::set-output name=git_pr::${GITHUB_PR_URL}" +# outputs: +# git_pr: ${{ steps.create_pr.outputs.git_pr }} +# comment_trigger_phrases: +# if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} +# needs: update_create_release_pr +# runs-on: ubuntu-latest +# permissions: +# pull-requests: write +# steps: +# - name: Check out code +# uses: actions/checkout@v3 +# with: +# ref: ${{ env.RELEASE_BRANCH }} +# - name: Comment Trigger Phrases in PR +# run: | +# GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} +# sh ./comment_pr_trigger_phrases.sh $GITHUB_PR +# env: +# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# working-directory: 'scripts/ci/release' From daf09adf04717630d364098e6d5db1929f4022f2 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 20 Sep 2022 12:45:53 -0500 Subject: [PATCH 48/66] Print gh actions config --- .github/workflows/verify_release_build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index b17fd92a8ab1..8ac41e2a8cda 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -45,7 +45,8 @@ jobs: echo $GITHUB_ACTOR echo $RUNNER_NAME echo $RUNNER_OS - + git config user.name + git config user.email #git config user.name $GITHUB_ACTOR # - name: Set Dataflow Python Batch Tests Release Version # run: | From 25f94bd9490608c039e4ab3bdf22e3f37658bf87 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 20 Sep 2022 16:44:13 -0500 Subject: [PATCH 49/66] Using hub in GH-hosted runner. --- .github/workflows/verify_release_build.yml | 96 ++++++++++------------ 1 file changed, 43 insertions(+), 53 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 8ac41e2a8cda..82e97a5bac7d 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -9,22 +9,14 @@ on: COMMENT_TRIGGER_PHRASES: description: Flag to comment the Jenkins Trigger Phrases in the new PR required: true - default: 'false' + default: 'true' env: #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? WORKING_BRANCH: 'verify-release-pr-test' jobs: - check_environment: - runs-on: ubuntu-latest - steps: - - name: Install hub cli tool - run: | - sudo apt-get update && sudo apt-get install -y hub - hub version - # Run Gradle release build and all PostCommit Tests against Release Branch on Jenkins. + # Run Gradle release build and all Tests against Release Branch on Jenkins. # This create_release_pr task will create a PR against apache/beam. update_create_release_pr: - needs: check_environment runs-on: ubuntu-latest permissions: pull-requests: write @@ -42,46 +34,44 @@ jobs: ref: ${{ env.RELEASE_BRANCH }} - name: Set git config run: | - echo $GITHUB_ACTOR - echo $RUNNER_NAME - echo $RUNNER_OS - git config user.name - git config user.email -#git config user.name $GITHUB_ACTOR -# - name: Set Dataflow Python Batch Tests Release Version -# run: | -# git checkout -b ${{env.WORKING_BRANCH}} --quiet -# /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add -# # In case the version string was not changed, append a newline to CHANGES.md -# echo "" >> CHANGES.md -# git add CHANGES.md -# git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet -# git push origin $WORKING_BRANCH --quiet -# - name: Create new Release PR -# id: create_pr -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# run: | -# GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. -# You can run many tests automatically using release/src/main/scripts/mass_comment.py.") -# echo "::set-output name=git_pr::${GITHUB_PR_URL}" -# outputs: -# git_pr: ${{ steps.create_pr.outputs.git_pr }} -# comment_trigger_phrases: -# if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} -# needs: update_create_release_pr -# runs-on: ubuntu-latest -# permissions: -# pull-requests: write -# steps: -# - name: Check out code -# uses: actions/checkout@v3 -# with: -# ref: ${{ env.RELEASE_BRANCH }} -# - name: Comment Trigger Phrases in PR -# run: | -# GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} -# sh ./comment_pr_trigger_phrases.sh $GITHUB_PR -# env: -# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# working-directory: 'scripts/ci/release' + git config user.name $GITHUB_ACTOR + - name: Set Release Version + run: | + git checkout -b ${{env.WORKING_BRANCH}} --quiet + /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add + # In case the version string was not changed, append a newline to CHANGES.md + echo "" >> CHANGES.md + - name: Push Changes in ${{ env.WORKING_BRANCH }} Branch + run: | + git add CHANGES.md + git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet + git push origin $WORKING_BRANCH --quiet + - name: Create new Release PR + id: create_pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + + You can run many tests automatically using release/src/main/scripts/mass_comment.py.") + echo "::set-output name=git_pr::${GITHUB_PR_URL}" + outputs: + git_pr: ${{ steps.create_pr.outputs.git_pr }} + comment_trigger_phrases: + if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} + needs: update_create_release_pr + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + ref: ${{ env.RELEASE_BRANCH }} + - name: Comment Trigger Phrases in PR + run: | + GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} + sh ./comment_pr_trigger_phrases.sh $GITHUB_PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: 'scripts/ci/release' From 645617e4b8a92ecef1829f519cd528f9f62eab8d Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 20 Sep 2022 17:23:37 -0500 Subject: [PATCH 50/66] Testing pr creation in apache repo. Commented code. --- .github/workflows/verify_release_build.yml | 38 +++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 82e97a5bac7d..900fe7ad301b 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -51,27 +51,27 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - GITHUB_PR_URL=$(hub pull-request -b $GITHUB_ACTOR:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + GITHUB_PR_URL=$(hub pull-request -b apache:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") echo "::set-output name=git_pr::${GITHUB_PR_URL}" outputs: git_pr: ${{ steps.create_pr.outputs.git_pr }} - comment_trigger_phrases: - if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} - needs: update_create_release_pr - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - ref: ${{ env.RELEASE_BRANCH }} - - name: Comment Trigger Phrases in PR - run: | - GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} - sh ./comment_pr_trigger_phrases.sh $GITHUB_PR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: 'scripts/ci/release' +# comment_trigger_phrases: +# if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} +# needs: update_create_release_pr +# runs-on: ubuntu-latest +# permissions: +# pull-requests: write +# steps: +# - name: Check out code +# uses: actions/checkout@v3 +# with: +# ref: ${{ env.RELEASE_BRANCH }} +# - name: Comment Trigger Phrases in PR +# run: | +# GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} +# sh ./comment_pr_trigger_phrases.sh $GITHUB_PR +# env: +# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# working-directory: 'scripts/ci/release' From 47d2b136c7f23f99ea1fd7aa52e6bd7852662123 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 20 Sep 2022 17:33:46 -0500 Subject: [PATCH 51/66] Create the PR in testing org. --- .github/workflows/verify_release_build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 900fe7ad301b..ef081f066787 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -10,6 +10,10 @@ on: description: Flag to comment the Jenkins Trigger Phrases in the new PR required: true default: 'true' + ORG: + description: Org name + required: true + env: #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? WORKING_BRANCH: 'verify-release-pr-test' @@ -50,8 +54,9 @@ jobs: id: create_pr env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ORG: ${{ github.event.inputs.ORG }} run: | - GITHUB_PR_URL=$(hub pull-request -b apache:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + GITHUB_PR_URL=$(hub pull-request -b $ORG:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") echo "::set-output name=git_pr::${GITHUB_PR_URL}" From cd5f50f6533ff88b1e2376469f75acfa2994d6ea Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 22 Sep 2022 16:01:11 -0500 Subject: [PATCH 52/66] Removing workflow permissions --- .github/workflows/verify_release_build.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index ef081f066787..a01ac4bf9316 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -22,9 +22,9 @@ jobs: # This create_release_pr task will create a PR against apache/beam. update_create_release_pr: runs-on: ubuntu-latest - permissions: - pull-requests: write - contents: write + # permissions: + # pull-requests: write + # contents: write env: RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} steps: @@ -39,6 +39,7 @@ jobs: - name: Set git config run: | git config user.name $GITHUB_ACTOR + #git config user.email actions@"$RUNNER_NAME".local - name: Set Release Version run: | git checkout -b ${{env.WORKING_BRANCH}} --quiet @@ -56,7 +57,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ORG: ${{ github.event.inputs.ORG }} run: | - GITHUB_PR_URL=$(hub pull-request -b $ORG:${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + BASE_BRANCH=$ORG:${{env.RELEASE_BRANCH}} + echo $BASE_BRANCH + GITHUB_PR_URL=$(hub pull-request -b $BASE_BRANCH -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") echo "::set-output name=git_pr::${GITHUB_PR_URL}" From a6056ec7304156d2f56f6c1bc78be52ef6f34091 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Thu, 22 Sep 2022 16:35:32 -0500 Subject: [PATCH 53/66] Removed owner from head branch. --- .github/workflows/verify_release_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index a01ac4bf9316..0702d69e49a7 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -59,7 +59,7 @@ jobs: run: | BASE_BRANCH=$ORG:${{env.RELEASE_BRANCH}} echo $BASE_BRANCH - GITHUB_PR_URL=$(hub pull-request -b $BASE_BRANCH -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. + GITHUB_PR_URL=$(hub pull-request -b ${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. You can run many tests automatically using release/src/main/scripts/mass_comment.py.") echo "::set-output name=git_pr::${GITHUB_PR_URL}" From d788c059262d7eabdf0d24485e1ac9bc7be13b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniela=20Mart=C3=ADn?= Date: Fri, 23 Sep 2022 13:24:32 -0500 Subject: [PATCH 54/66] Adding setup-gh-cli as an composite action. --- .github/actions/setup-gh-cli-linux/action.yml | 31 ++++++++++ .github/workflows/verify_release_build.yml | 58 +++++++++---------- 2 files changed, 60 insertions(+), 29 deletions(-) create mode 100644 .github/actions/setup-gh-cli-linux/action.yml diff --git a/.github/actions/setup-gh-cli-linux/action.yml b/.github/actions/setup-gh-cli-linux/action.yml new file mode 100644 index 000000000000..41d38b7a56c5 --- /dev/null +++ b/.github/actions/setup-gh-cli-linux/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup GH CLI - Linux' +description: 'Setup GitHub CLI tool in a Linux Operating System.' +runs: + using: "composite" + steps: + - id: setup-gh-cli-linux + shell: bash + run: | + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ + && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh -y + gh --version diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 0702d69e49a7..bb65f7e8e575 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -21,13 +21,15 @@ jobs: # Run Gradle release build and all Tests against Release Branch on Jenkins. # This create_release_pr task will create a PR against apache/beam. update_create_release_pr: - runs-on: ubuntu-latest - # permissions: - # pull-requests: write - # contents: write + runs-on: [self-hosted, ubuntu-20.04] + permissions: + pull-requests: write + contents: write env: RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} steps: + - name: Install gh cli + uses: ./.github/actions/setup-gh-cli-linux - name: Set RELEASE_BRANCH env variable run: | RELEASE_BRANCH=release-${{env.RELEASE_VER}} @@ -40,31 +42,29 @@ jobs: run: | git config user.name $GITHUB_ACTOR #git config user.email actions@"$RUNNER_NAME".local - - name: Set Release Version - run: | - git checkout -b ${{env.WORKING_BRANCH}} --quiet - /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add - # In case the version string was not changed, append a newline to CHANGES.md - echo "" >> CHANGES.md - - name: Push Changes in ${{ env.WORKING_BRANCH }} Branch - run: | - git add CHANGES.md - git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet - git push origin $WORKING_BRANCH --quiet - - name: Create new Release PR - id: create_pr - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ORG: ${{ github.event.inputs.ORG }} - run: | - BASE_BRANCH=$ORG:${{env.RELEASE_BRANCH}} - echo $BASE_BRANCH - GITHUB_PR_URL=$(hub pull-request -b ${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. - - You can run many tests automatically using release/src/main/scripts/mass_comment.py.") - echo "::set-output name=git_pr::${GITHUB_PR_URL}" - outputs: - git_pr: ${{ steps.create_pr.outputs.git_pr }} + # - name: Set Release Version + # run: | + # git checkout -b ${{env.WORKING_BRANCH}} --quiet + # /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add + # # In case the version string was not changed, append a newline to CHANGES.md + # echo "" >> CHANGES.md + # - name: Push Changes in ${{ env.WORKING_BRANCH }} Branch + # run: | + # git add CHANGES.md + # git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet + # git push origin $WORKING_BRANCH --quiet + # - name: Create new Release PR + # id: create_pr + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # ORG: ${{ github.event.inputs.ORG }} + # run: | + # BASE_BRANCH=$ORG:${{env.RELEASE_BRANCH}} + # echo $BASE_BRANCH + # GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b You can run many tests automatically using release/src/main/scripts/mass_comment.py.") + # echo "::set-output name=git_pr::${GITHUB_PR_URL}" + # outputs: + # git_pr: ${{ steps.create_pr.outputs.git_pr }} # comment_trigger_phrases: # if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} # needs: update_create_release_pr From 0d03675b1c8867b9dc7ede8e02e028079804c776 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 23 Sep 2022 17:40:48 -0500 Subject: [PATCH 55/66] Created a setup-gh-cli-linux composite action. Updated verify_release workflow ti runs on slef-hosted an using gh cli. --- .github/actions/setup-gh-cli-linux/action.yml | 31 +++++++++++ .github/workflows/verify_release_build.yml | 52 ++++++------------- 2 files changed, 46 insertions(+), 37 deletions(-) create mode 100644 .github/actions/setup-gh-cli-linux/action.yml diff --git a/.github/actions/setup-gh-cli-linux/action.yml b/.github/actions/setup-gh-cli-linux/action.yml new file mode 100644 index 000000000000..41d38b7a56c5 --- /dev/null +++ b/.github/actions/setup-gh-cli-linux/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup GH CLI - Linux' +description: 'Setup GitHub CLI tool in a Linux Operating System.' +runs: + using: "composite" + steps: + - id: setup-gh-cli-linux + shell: bash + run: | + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ + && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh -y + gh --version diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 0702d69e49a7..36984e9129d2 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -10,10 +10,6 @@ on: description: Flag to comment the Jenkins Trigger Phrases in the new PR required: true default: 'true' - ORG: - description: Org name - required: true - env: #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? WORKING_BRANCH: 'verify-release-pr-test' @@ -21,10 +17,10 @@ jobs: # Run Gradle release build and all Tests against Release Branch on Jenkins. # This create_release_pr task will create a PR against apache/beam. update_create_release_pr: - runs-on: ubuntu-latest - # permissions: - # pull-requests: write - # contents: write + runs-on: [self-hosted, ubuntu-20.04] + permissions: + pull-requests: write + contents: write env: RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} steps: @@ -36,10 +32,12 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ env.RELEASE_BRANCH }} + - name: Install gh cli + uses: ./.github/actions/setup-gh-cli-linux - name: Set git config run: | git config user.name $GITHUB_ACTOR - #git config user.email actions@"$RUNNER_NAME".local + git config user.email actions@"$RUNNER_NAME".local - name: Set Release Version run: | git checkout -b ${{env.WORKING_BRANCH}} --quiet @@ -52,34 +50,14 @@ jobs: git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet git push origin $WORKING_BRANCH --quiet - name: Create new Release PR - id: create_pr env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ORG: ${{ github.event.inputs.ORG }} run: | - BASE_BRANCH=$ORG:${{env.RELEASE_BRANCH}} - echo $BASE_BRANCH - GITHUB_PR_URL=$(hub pull-request -b ${{env.RELEASE_BRANCH}} -h $GITHUB_ACTOR:${{env.WORKING_BRANCH}} -l ${{env.RELEASE_BRANCH}} -F - <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch. - - You can run many tests automatically using release/src/main/scripts/mass_comment.py.") - echo "::set-output name=git_pr::${GITHUB_PR_URL}" - outputs: - git_pr: ${{ steps.create_pr.outputs.git_pr }} -# comment_trigger_phrases: -# if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} -# needs: update_create_release_pr -# runs-on: ubuntu-latest -# permissions: -# pull-requests: write -# steps: -# - name: Check out code -# uses: actions/checkout@v3 -# with: -# ref: ${{ env.RELEASE_BRANCH }} -# - name: Comment Trigger Phrases in PR -# run: | -# GITHUB_PR=${{ needs.update_create_release_pr.outputs.git_pr }} -# sh ./comment_pr_trigger_phrases.sh $GITHUB_PR -# env: -# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# working-directory: 'scripts/ci/release' + GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "You can run many tests automatically using release/src/main/scripts/mass_comment.py.") + - name: Comment Jenkins Trigger Phrases in PR + if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} + run: | + sh ./comment_pr_trigger_phrases.sh $GITHUB_PR_URL + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: 'scripts/ci/release' From a1f74f79076ad33600b80e9e973e6f868fca7c23 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Mon, 26 Sep 2022 10:48:00 -0500 Subject: [PATCH 56/66] Updated sh file to comment the real trigger phrases insted of the example file. Updated PR body text. --- .github/workflows/verify_release_build.yml | 11 ++++------- scripts/ci/release/comment_pr_trigger_phrases.sh | 6 ++---- scripts/ci/release/example.txt | 4 ---- 3 files changed, 6 insertions(+), 15 deletions(-) delete mode 100644 scripts/ci/release/example.txt diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 36984e9129d2..0797d890ece5 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -9,14 +9,11 @@ on: COMMENT_TRIGGER_PHRASES: description: Flag to comment the Jenkins Trigger Phrases in the new PR required: true - default: 'true' + default: 'false' env: - #WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? - WORKING_BRANCH: 'verify-release-pr-test' + WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? jobs: - # Run Gradle release build and all Tests against Release Branch on Jenkins. - # This create_release_pr task will create a PR against apache/beam. - update_create_release_pr: + verify_release: runs-on: [self-hosted, ubuntu-20.04] permissions: pull-requests: write @@ -53,7 +50,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "You can run many tests automatically using release/src/main/scripts/mass_comment.py.") + GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "You are going to run many tests automatically in this PR (refer to scripts/ci/release/jenkins_trigger_phrases.txt), however, there are some PostCommits jobs that have been migrated to GitHub Actions workflows and for the moment they don't have Trigger Phrases. Please run the rest of PostCommit tests manually.") - name: Comment Jenkins Trigger Phrases in PR if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} run: | diff --git a/scripts/ci/release/comment_pr_trigger_phrases.sh b/scripts/ci/release/comment_pr_trigger_phrases.sh index c7db76a783d6..711be5bf1598 100755 --- a/scripts/ci/release/comment_pr_trigger_phrases.sh +++ b/scripts/ci/release/comment_pr_trigger_phrases.sh @@ -1,9 +1,7 @@ #!/bin/bash -#file="./jenkins_trigger_phrases.txt" -file="./example.txt" +file="./jenkins_trigger_phrases.txt" GITHUB_PR=$1 while IFS= read -r trigger_phrase do - #printf '%s\n' "$trigger_phrase" gh pr comment "$GITHUB_PR" --body "$trigger_phrase" -done <"$file" \ No newline at end of file +done <"$file" diff --git a/scripts/ci/release/example.txt b/scripts/ci/release/example.txt deleted file mode 100644 index f378a59a93b3..000000000000 --- a/scripts/ci/release/example.txt +++ /dev/null @@ -1,4 +0,0 @@ -Run PythonFormatter PreCommit -Run PythonLint PreCommit -Run RAT PreCommit -Run Whitespace PreCommit \ No newline at end of file From dce26080b34bca6b2484cbac6b36c2f09d11970c Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 27 Sep 2022 12:33:18 -0500 Subject: [PATCH 57/66] Updated jenkins_trigger_phrases.txt file using the Jobs that have not been migrated by the team. Also, included the mass_comment Trigger Phrases file if needed. --- .github/workflows/verify_release_build.yml | 3 +- .../ci/release/jenkins_trigger_phrases.txt | 70 +----------- scripts/ci/release/mass_comment.txt | 103 ++++++++++++++++++ 3 files changed, 111 insertions(+), 65 deletions(-) create mode 100644 scripts/ci/release/mass_comment.txt diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 0797d890ece5..ec57b8d826e5 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -50,7 +50,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "You are going to run many tests automatically in this PR (refer to scripts/ci/release/jenkins_trigger_phrases.txt), however, there are some PostCommits jobs that have been migrated to GitHub Actions workflows and for the moment they don't have Trigger Phrases. Please run the rest of PostCommit tests manually.") + GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "Run Gradle Release Build and many PreCommit and PostCommits tests automatically in this PR (refer to scripts/ci/release/jenkins_trigger_phrases.txt), however, there are some PreCommit and PostCommit jobs that have been migrated to GitHub Actions workflows and for the moment they don't have Trigger Phrases enabled. Please run the missing tests manually.") + # Run Gradle Release Build and PostCommit Tests against Release Branch on Jenkins. - name: Comment Jenkins Trigger Phrases in PR if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} run: | diff --git a/scripts/ci/release/jenkins_trigger_phrases.txt b/scripts/ci/release/jenkins_trigger_phrases.txt index e4884c76ded5..05e8325f560b 100644 --- a/scripts/ci/release/jenkins_trigger_phrases.txt +++ b/scripts/ci/release/jenkins_trigger_phrases.txt @@ -1,77 +1,19 @@ +Run Release Gradle Build Run CommunityMetrics PreCommit -Run Dataflow Runner Nexmark Tests -Run Dataflow Runner V2 Java 11 Nexmark Tests -Run Dataflow Runner V2 Java 17 Nexmark Tests -Run Dataflow Runner V2 Nexmark Tests -Run Dataflow Streaming ValidatesRunner -Run Dataflow ValidatesRunner Java 11 -Run Dataflow ValidatesRunner Java 17 -Run Dataflow ValidatesRunner -Run Direct Runner Nexmark Tests Run Direct ValidatesRunner Java 11 Run Direct ValidatesRunner Java 17 -Run Direct ValidatesRunner in Java 11 -Run Direct ValidatesRunner -Run Flink Runner Nexmark Tests -Run Flink ValidatesRunner Java 11 -Run Flink ValidatesRunner -Run Go Flink ValidatesRunner -Run Go PreCommit -Run Go Samza ValidatesRunner -Run Go Spark ValidatesRunner -Run GoPortable PreCommit Run Java 11 Examples on Dataflow Runner V2 Run Java 17 Examples on Dataflow Runner V2 -Run Java Dataflow V2 ValidatesRunner Streaming -Run Java Dataflow V2 ValidatesRunner -Run Java Examples on Dataflow Runner V2 -Run Java Examples_Direct -Run Java Examples_Flink -Run Java Examples_Spark -Run Java Flink PortableValidatesRunner Streaming -Run Java Portability examples on Dataflow with Java 11 Run Java PreCommit -Run Java Samza PortableValidatesRunner -Run Java Spark PortableValidatesRunner Batch -Run Java Spark v2 PortableValidatesRunner Streaming -Run Java Spark v3 PortableValidatesRunner Streaming Run Java examples on Dataflow Java 11 Run Java examples on Dataflow Java 17 -Run Java examples on Dataflow with Java 11 -Run Java_Examples_Dataflow PreCommit -Run Java_Examples_Dataflow_Java11 PreCommit -Run Java_Examples_Dataflow_Java17 PreCommit -Run Java_PVR_Flink_Batch PreCommit -Run Java_PVR_Flink_Docker PreCommit -Run Portable_Python PreCommit -Run Python Dataflow V2 ValidatesRunner -Run Python Dataflow ValidatesContainer -Run Python Dataflow ValidatesRunner -Run Python Examples_Dataflow -Run Python Examples_Direct -Run Python Examples_Flink -Run Python Examples_Spark +Run Javadoc PostCommit +Run Jpms Dataflow Java 17 PostCommit +Run Jpms Direct Java 17 PostCommit Run Python Flink ValidatesRunner Run Python PreCommit Run Python Samza ValidatesRunner Run Python Spark ValidatesRunner -Run PythonDocker PreCommit -Run PythonDocs PreCommit -Run PythonFormatter PreCommit -Run PythonLint PreCommit -Run Python_PVR_Flink PreCommit -Run RAT PreCommit -Run Release Gradle Build -Run SQL PreCommit -Run SQL_Java11 PreCommit Run SQL_Java17 PreCommit -Run Samza ValidatesRunner -Run Spark Runner Nexmark Tests -Run Spark StructuredStreaming ValidatesRunner -Run Spark ValidatesRunner Java 11 -Run Spark ValidatesRunner -Run Spotless PreCommit -Run Twister2 ValidatesRunner -Run Typescript PreCommit -Run ULR Loopback ValidatesRunner -Run Whitespace PreCommit \ No newline at end of file +Run XVR_Flink PostCommit +Run XVR_GoUsingJava_Dataflow PostCommit \ No newline at end of file diff --git a/scripts/ci/release/mass_comment.txt b/scripts/ci/release/mass_comment.txt new file mode 100644 index 000000000000..328ec7eede62 --- /dev/null +++ b/scripts/ci/release/mass_comment.txt @@ -0,0 +1,103 @@ +Run Release Gradle Build +Run CommunityMetrics PreCommit +Run Dataflow Runner Nexmark Tests +Run Dataflow Runner V2 Java 11 Nexmark Tests +Run Dataflow Runner V2 Java 17 Nexmark Tests +Run Dataflow Runner V2 Nexmark Tests +Run Dataflow Streaming ValidatesRunner +Run Dataflow ValidatesRunner Java 11 +Run Dataflow ValidatesRunner Java 17 +Run Dataflow ValidatesRunner +Run Direct Runner Nexmark Tests +Run Direct ValidatesRunner Java 11 +Run Direct ValidatesRunner Java 17 +Run Direct ValidatesRunner in Java 11 +Run Direct ValidatesRunner +Run Flink Runner Nexmark Tests +Run Flink ValidatesRunner Java 11 +Run Flink ValidatesRunner +Run Go Flink ValidatesRunner +Run Go PostCommit +Run Go PreCommit +Run Go Samza ValidatesRunner +Run Go Spark ValidatesRunner +Run GoPortable PreCommit +Run Java 11 Examples on Dataflow Runner V2 +Run Java 17 Examples on Dataflow Runner V2 +Run Java Dataflow V2 ValidatesRunner Streaming +Run Java Dataflow V2 ValidatesRunner +Run Java Examples on Dataflow Runner V2 +Run Java Examples_Direct +Run Java Examples_Flink +Run Java Examples_Spark +Run Java Flink PortableValidatesRunner Streaming +Run Java Portability examples on Dataflow with Java 11 +Run Java PostCommit +Run Java PreCommit +Run Java Samza PortableValidatesRunner +Run Java Spark PortableValidatesRunner Batch +Run Java Spark v2 PortableValidatesRunner Streaming +Run Java Spark v3 PortableValidatesRunner Streaming +Run Java examples on Dataflow Java 11 +Run Java examples on Dataflow Java 17 +Run Java examples on Dataflow with Java 11 +Run Java_Examples_Dataflow PreCommit +Run Java_Examples_Dataflow_Java11 PreCommit +Run Java_Examples_Dataflow_Java17 PreCommit +Run Java_PVR_Flink_Batch PreCommit +Run Java_PVR_Flink_Docker PreCommit +Run Javadoc PostCommit +Run Jpms Dataflow Java 11 PostCommit +Run Jpms Dataflow Java 17 PostCommit +Run Jpms Direct Java 11 PostCommit +Run Jpms Direct Java 17 PostCommit +Run Jpms Flink Java 11 PostCommit +Run Jpms Spark Java 11 PostCommit +Run PortableJar_Flink PostCommit +Run PortableJar_Spark PostCommit +Run Portable_Python PreCommit +Run PostCommit_Java_Dataflow +Run PostCommit_Java_DataflowV2 +Run PostCommit_Java_Hadoop_Versions +Run Python 3.7 PostCommit +Run Python 3.8 PostCommit +Run Python 3.9 PostCommit +Run Python Dataflow V2 ValidatesRunner +Run Python Dataflow ValidatesContainer +Run Python Dataflow ValidatesRunner +Run Python Examples_Dataflow +Run Python Examples_Direct +Run Python Examples_Flink +Run Python Examples_Spark +Run Python Flink ValidatesRunner +Run Python PreCommit +Run Python Samza ValidatesRunner +Run Python Spark ValidatesRunner +Run PythonDocker PreCommit +Run PythonDocs PreCommit +Run PythonFormatter PreCommit +Run PythonLint PreCommit +Run Python_PVR_Flink PreCommit +Run RAT PreCommit +Run SQL PostCommit +Run SQL PreCommit +Run SQL_Java11 PreCommit +Run SQL_Java17 PreCommit +Run Samza ValidatesRunner +Run Spark Runner Nexmark Tests +Run Spark StructuredStreaming ValidatesRunner +Run Spark ValidatesRunner Java 11 +Run Spark ValidatesRunner +Run Spotless PreCommit +Run Twister2 ValidatesRunner +Run Typescript PreCommit +Run ULR Loopback ValidatesRunner +Run Whitespace PreCommit +Run XVR_Direct PostCommit +Run XVR_Flink PostCommit +Run XVR_JavaUsingPython_Dataflow PostCommit +Run XVR_PythonUsingJavaSQL_Dataflow PostCommit +Run XVR_PythonUsingJava_Dataflow PostCommit +Run XVR_Samza PostCommit +Run XVR_Spark PostCommit +Run XVR_Spark3 PostCommit \ No newline at end of file From 3b0106b8baf3b8427a967f09cbe05d75cf65f78a Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 27 Sep 2022 15:32:03 -0500 Subject: [PATCH 58/66] Added Verify Release Build workflow in CI.md doc. --- CI.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CI.md b/CI.md index ee911584f7f0..25f038ca65fc 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,13 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### Release Preparation and Validation Workflows +#### Verify Release Build - [verify_release_build.yml](.github/workflows/verify_release_build.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Verify Release Build | Verifies the release branch, creates a PR and comments Trigger Phrases. | No | No | No | No | + + ### All migrated workflows run based on the following triggers | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Workflow Dispatch | From 76d8fbf29f38d5e5b4941076a7a51996d3a27d20 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 27 Sep 2022 15:43:48 -0500 Subject: [PATCH 59/66] Updated comment_pr_trigger_phrases file to point to mass_comment file. --- .github/workflows/verify_release_build.yml | 2 +- scripts/ci/release/comment_pr_trigger_phrases.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index ec57b8d826e5..7ef7ce63976c 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -50,7 +50,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "Run Gradle Release Build and many PreCommit and PostCommits tests automatically in this PR (refer to scripts/ci/release/jenkins_trigger_phrases.txt), however, there are some PreCommit and PostCommit jobs that have been migrated to GitHub Actions workflows and for the moment they don't have Trigger Phrases enabled. Please run the missing tests manually.") + GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "Run Gradle Release Build and Jenkins PreCommit and PostCommits tests automatically in this PR (refer to scripts/ci/release/mass_comment.txt). Please run the missing tests manually.") # Run Gradle Release Build and PostCommit Tests against Release Branch on Jenkins. - name: Comment Jenkins Trigger Phrases in PR if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} diff --git a/scripts/ci/release/comment_pr_trigger_phrases.sh b/scripts/ci/release/comment_pr_trigger_phrases.sh index 711be5bf1598..48ee05dac27e 100755 --- a/scripts/ci/release/comment_pr_trigger_phrases.sh +++ b/scripts/ci/release/comment_pr_trigger_phrases.sh @@ -1,5 +1,5 @@ #!/bin/bash -file="./jenkins_trigger_phrases.txt" +file="./mass_comment.txt" GITHUB_PR=$1 while IFS= read -r trigger_phrase do From 78da9863ca0c29332efb9105126ff390626ab665 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 27 Sep 2022 15:53:51 -0500 Subject: [PATCH 60/66] Commented with Apache License in verify_release_build.yml and comment_pr_trigger_phrases.sh files. --- .github/workflows/verify_release_build.yml | 21 ++++++++++++++++++- CI.md | 2 +- .../ci/release/comment_pr_trigger_phrases.sh | 19 +++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index 7ef7ce63976c..dfb0abc25426 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -1,3 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + name: Verify Release Build on: workflow_dispatch: @@ -51,7 +70,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "Run Gradle Release Build and Jenkins PreCommit and PostCommits tests automatically in this PR (refer to scripts/ci/release/mass_comment.txt). Please run the missing tests manually.") - # Run Gradle Release Build and PostCommit Tests against Release Branch on Jenkins. + # Run Gradle Release Build and PostCommit/PreCommit Tests against Release Branch on Jenkins. - name: Comment Jenkins Trigger Phrases in PR if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} run: | diff --git a/CI.md b/CI.md index 25f038ca65fc..588f2a424639 100644 --- a/CI.md +++ b/CI.md @@ -129,7 +129,7 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog #### Verify Release Build - [verify_release_build.yml](.github/workflows/verify_release_build.yml) | Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | |------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| -| Verify Release Build | Verifies the release branch, creates a PR and comments Trigger Phrases. | No | No | No | No | +| Verify Release Build | Verifies full life cycle of Gradle Build and all PostCommit/PreCommit tests against Release Branch on Jenkins. | No | No | No | No | ### All migrated workflows run based on the following triggers diff --git a/scripts/ci/release/comment_pr_trigger_phrases.sh b/scripts/ci/release/comment_pr_trigger_phrases.sh index 48ee05dac27e..4a086fb50fa8 100755 --- a/scripts/ci/release/comment_pr_trigger_phrases.sh +++ b/scripts/ci/release/comment_pr_trigger_phrases.sh @@ -1,4 +1,23 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script adds comments from a txt file, in the given GitHub Pull Request. +# mass_comment.txt file has the Trigger Phrases for running Gradle Build and PostCommit/PreCommit tests. + file="./mass_comment.txt" GITHUB_PR=$1 while IFS= read -r trigger_phrase From 1245874fcde0f0f2e049f2adba4a957eb290beb2 Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 27 Sep 2022 15:56:13 -0500 Subject: [PATCH 61/66] Removed unused code in inputs vars in verify_release_build.yml --- .github/workflows/verify_release_build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index dfb0abc25426..e3d6dca812cf 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -24,11 +24,9 @@ on: RELEASE_VER: description: Beam version of current release required: true - default: vtest # remove default before merging COMMENT_TRIGGER_PHRASES: description: Flag to comment the Jenkins Trigger Phrases in the new PR required: true - default: 'false' env: WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? jobs: From 2764d43bc06dc2d57b7d382aec5dbd34ece1681a Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Tue, 27 Sep 2022 16:14:54 -0500 Subject: [PATCH 62/66] Added blank line at the end of txt files to remove RAT issue. --- scripts/ci/release/jenkins_trigger_phrases.txt | 2 +- scripts/ci/release/mass_comment.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/release/jenkins_trigger_phrases.txt b/scripts/ci/release/jenkins_trigger_phrases.txt index 05e8325f560b..4f59885de8b4 100644 --- a/scripts/ci/release/jenkins_trigger_phrases.txt +++ b/scripts/ci/release/jenkins_trigger_phrases.txt @@ -16,4 +16,4 @@ Run Python Samza ValidatesRunner Run Python Spark ValidatesRunner Run SQL_Java17 PreCommit Run XVR_Flink PostCommit -Run XVR_GoUsingJava_Dataflow PostCommit \ No newline at end of file +Run XVR_GoUsingJava_Dataflow PostCommit diff --git a/scripts/ci/release/mass_comment.txt b/scripts/ci/release/mass_comment.txt index 328ec7eede62..b3a307c9e698 100644 --- a/scripts/ci/release/mass_comment.txt +++ b/scripts/ci/release/mass_comment.txt @@ -100,4 +100,4 @@ Run XVR_PythonUsingJavaSQL_Dataflow PostCommit Run XVR_PythonUsingJava_Dataflow PostCommit Run XVR_Samza PostCommit Run XVR_Spark PostCommit -Run XVR_Spark3 PostCommit \ No newline at end of file +Run XVR_Spark3 PostCommit From 170d3a8a7979a90a796f89b70607018eebd4d86a Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Wed, 5 Oct 2022 10:22:27 -0500 Subject: [PATCH 63/66] Moved trigger_phrases txt files into /test/resources folder. --- scripts/ci/release/comment_pr_trigger_phrases.sh | 4 ++-- .../release/{ => test/resources}/jenkins_trigger_phrases.txt | 0 scripts/ci/release/{ => test/resources}/mass_comment.txt | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename scripts/ci/release/{ => test/resources}/jenkins_trigger_phrases.txt (100%) rename scripts/ci/release/{ => test/resources}/mass_comment.txt (100%) diff --git a/scripts/ci/release/comment_pr_trigger_phrases.sh b/scripts/ci/release/comment_pr_trigger_phrases.sh index 4a086fb50fa8..10479dd94dd8 100755 --- a/scripts/ci/release/comment_pr_trigger_phrases.sh +++ b/scripts/ci/release/comment_pr_trigger_phrases.sh @@ -16,9 +16,9 @@ # limitations under the License. # # This script adds comments from a txt file, in the given GitHub Pull Request. -# mass_comment.txt file has the Trigger Phrases for running Gradle Build and PostCommit/PreCommit tests. +# test/resources/mass_comment.txt file has the Trigger Phrases for running Gradle Build and PostCommit/PreCommit tests. -file="./mass_comment.txt" +file="./test/resources/mass_comment.txt" GITHUB_PR=$1 while IFS= read -r trigger_phrase do diff --git a/scripts/ci/release/jenkins_trigger_phrases.txt b/scripts/ci/release/test/resources/jenkins_trigger_phrases.txt similarity index 100% rename from scripts/ci/release/jenkins_trigger_phrases.txt rename to scripts/ci/release/test/resources/jenkins_trigger_phrases.txt diff --git a/scripts/ci/release/mass_comment.txt b/scripts/ci/release/test/resources/mass_comment.txt similarity index 100% rename from scripts/ci/release/mass_comment.txt rename to scripts/ci/release/test/resources/mass_comment.txt From de6e130139249392319d539240c299fef7cc1937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniela=20Mart=C3=ADn?= Date: Wed, 5 Oct 2022 13:41:28 -0500 Subject: [PATCH 64/66] Update CI.md, removing Jenkins word to make it generic. Co-authored-by: Danny McCormick --- CI.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI.md b/CI.md index 588f2a424639..0fcc87da9521 100644 --- a/CI.md +++ b/CI.md @@ -129,7 +129,7 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog #### Verify Release Build - [verify_release_build.yml](.github/workflows/verify_release_build.yml) | Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | |------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| -| Verify Release Build | Verifies full life cycle of Gradle Build and all PostCommit/PreCommit tests against Release Branch on Jenkins. | No | No | No | No | +| Verify Release Build | Verifies full life cycle of Gradle Build and all PostCommit/PreCommit tests against Release Branch on CI. | No | No | No | No | ### All migrated workflows run based on the following triggers From 0d8424ea86d2c57ee9b7f7d4b3803d6f3f4a5d2b Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 14 Oct 2022 15:23:10 -0500 Subject: [PATCH 65/66] Adding script for branch validation. --- .github/workflows/verify_release_build.yml | 6 +++- scripts/ci/ci_check_git_branch.sh | 34 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 scripts/ci/ci_check_git_branch.sh diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml index e3d6dca812cf..5a128a4eec61 100644 --- a/.github/workflows/verify_release_build.yml +++ b/.github/workflows/verify_release_build.yml @@ -28,7 +28,7 @@ on: description: Flag to comment the Jenkins Trigger Phrases in the new PR required: true env: - WORKING_BRANCH: postcommit_validation_pr # Is it always the same branch name? or should we need to add it as input? + WORKING_BRANCH: postcommit_validation_pr jobs: verify_release: runs-on: [self-hosted, ubuntu-20.04] @@ -38,6 +38,10 @@ jobs: env: RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} steps: + - name: Verify branch name + run: + - sh ./ci_check_git_branch.sh $WORKING_BRANCH + working-directory: 'scripts/ci' - name: Set RELEASE_BRANCH env variable run: | RELEASE_BRANCH=release-${{env.RELEASE_VER}} diff --git a/scripts/ci/ci_check_git_branch.sh b/scripts/ci/ci_check_git_branch.sh new file mode 100644 index 000000000000..9a9f12850463 --- /dev/null +++ b/scripts/ci/ci_check_git_branch.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function is_branch() { + #if nothing matches show-ref will return an error code of 1 + if git show-ref --quiet --verify -- "refs/heads/$1" ; then + return 1 + else + return 0 + fi + $ret +} + +if is_branch "$1"; then + echo "Branch [$1] doesn't exist." +else + echo "Branch [$1] already exists!" + echo >&2 "Please make sure your branch doesn't exist." +fi From cc9064a79bc8813fa277af2976ecb51a490e1f1d Mon Sep 17 00:00:00 2001 From: dannymartinm Date: Fri, 14 Oct 2022 15:24:38 -0500 Subject: [PATCH 66/66] Removing unused word. --- scripts/ci/ci_check_git_branch.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ci/ci_check_git_branch.sh b/scripts/ci/ci_check_git_branch.sh index 9a9f12850463..e7bff3e77998 100644 --- a/scripts/ci/ci_check_git_branch.sh +++ b/scripts/ci/ci_check_git_branch.sh @@ -23,7 +23,6 @@ function is_branch() { else return 0 fi - $ret } if is_branch "$1"; then