apache · damccorm · Oct 16, 2022 · Sep 1, 2022 · Sep 1, 2022 · Sep 2, 2022
diff --git a/.github/actions/setup-gh-cli-linux/action.yml b/.github/actions/setup-gh-cli-linux/action.yml
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: 'Setup GH CLI - Linux'
+description: 'Setup GitHub CLI tool in a Linux Operating System.'
+runs:
+  using: "composite"
+  steps:
+    - id: setup-gh-cli-linux
+      shell: bash
+      run: |
+        curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+        && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+        && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+        && sudo apt update \
+        && sudo apt install gh -y
+        gh --version
diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# To learn more about GitHub Actions in Apache Beam check the CI.md
+
+name: Verify Release Build
+on:
+  workflow_dispatch:
+    inputs:
+      RELEASE_VER:
+        description: Beam version of current release
+        required: true
+      COMMENT_TRIGGER_PHRASES:
+        description: Flag to comment the Jenkins Trigger Phrases in the new PR
+        required: true
+env:
+  WORKING_BRANCH: postcommit_validation_pr
+jobs:
+  verify_release:
+    runs-on: [self-hosted, ubuntu-20.04]
+    permissions:
+      pull-requests: write
+      contents: write
+    env:
+      RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }}
+    steps:
+      - name: Verify branch name
+        run:
+          - sh ./ci_check_git_branch.sh $WORKING_BRANCH
+        working-directory: 'scripts/ci'
+      - name: Set RELEASE_BRANCH env variable
+        run: |
+          RELEASE_BRANCH=release-${{env.RELEASE_VER}}
+          echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.RELEASE_BRANCH }}
+      - name: Install gh cli
+        uses: ./.github/actions/setup-gh-cli-linux
+      - name: Set git config
+        run: |
+          git config user.name $GITHUB_ACTOR
+          git config user.email actions@"$RUNNER_NAME".local
+      - name: Set Release Version
+        run: |
+          git checkout -b ${{env.WORKING_BRANCH}} --quiet
+          /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add
+          # In case the version string was not changed, append a newline to CHANGES.md
+          echo "" >> CHANGES.md
+      - name: Push Changes in ${{ env.WORKING_BRANCH }} Branch
+        run: |
+          git add CHANGES.md
+          git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet
+          git push origin $WORKING_BRANCH --quiet
+      - name: Create new Release PR
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "Run Gradle Release Build and Jenkins PreCommit and PostCommits tests automatically in this PR (refer to scripts/ci/release/mass_comment.txt). Please run the missing tests manually.")
+      # Run Gradle Release Build and PostCommit/PreCommit Tests against Release Branch on Jenkins.
+      - name: Comment Jenkins Trigger Phrases in PR
+        if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }}
+        run: |
+          sh ./comment_pr_trigger_phrases.sh $GITHUB_PR_URL
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        working-directory: 'scripts/ci/release'
diff --git a/CI.md b/CI.md
@@ -125,6 +125,13 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog
 | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner.                                               | Yes              | Yes                   | Yes           | -                        |
 | Java Wordcount Dataflow      | Runs Java WordCount example with DataFlow Runner.                                             | -                | Yes                   | Yes           | Yes                      |
 
+### Release Preparation and Validation Workflows
+#### Verify Release Build - [verify_release_build.yml](.github/workflows/verify_release_build.yml)
+| Job                          | Description                                                                                   | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials |
+|------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------|
+| Verify Release Build         | Verifies full life cycle of Gradle Build and all PostCommit/PreCommit tests against Release Branch on CI.                   | No               | No                    | No            | No                       |
+
+
 ### All migrated workflows run based on the following triggers
 
 | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Workflow Dispatch |

diff --git a/scripts/ci/ci_check_git_branch.sh b/scripts/ci/ci_check_git_branch.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function is_branch() {
+    #if nothing matches show-ref will return an error code of 1
+    if git show-ref --quiet --verify -- "refs/heads/$1" ; then
+      return 1
+    else
+      return 0
+    fi
+}
+
+if is_branch "$1"; then
+  echo "Branch [$1] doesn't exist."
+else
+  echo "Branch [$1] already exists!"
+  echo >&2 "Please make sure your branch doesn't exist."
+fi
diff --git a/scripts/ci/release/comment_pr_trigger_phrases.sh b/scripts/ci/release/comment_pr_trigger_phrases.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# This script adds comments from a txt file, in the given GitHub Pull Request.
+# test/resources/mass_comment.txt file has the Trigger Phrases for running Gradle Build and PostCommit/PreCommit tests.
+
+file="./test/resources/mass_comment.txt"
+GITHUB_PR=$1
+while IFS= read -r trigger_phrase
+do
+    gh pr comment "$GITHUB_PR" --body "$trigger_phrase"
+done <"$file"
diff --git a/scripts/ci/release/test/resources/jenkins_trigger_phrases.txt b/scripts/ci/release/test/resources/jenkins_trigger_phrases.txt
@@ -0,0 +1,19 @@
+Run Release Gradle Build
+Run CommunityMetrics PreCommit
+Run Direct ValidatesRunner Java 11
+Run Direct ValidatesRunner Java 17
+Run Java 11 Examples on Dataflow Runner V2
+Run Java 17 Examples on Dataflow Runner V2
+Run Java PreCommit
+Run Java examples on Dataflow Java 11
+Run Java examples on Dataflow Java 17
+Run Javadoc PostCommit
+Run Jpms Dataflow Java 17 PostCommit
+Run Jpms Direct Java 17 PostCommit
+Run Python Flink ValidatesRunner
+Run Python PreCommit
+Run Python Samza ValidatesRunner
+Run Python Spark ValidatesRunner
+Run SQL_Java17 PreCommit
+Run XVR_Flink PostCommit
+Run XVR_GoUsingJava_Dataflow PostCommit
diff --git a/scripts/ci/release/test/resources/mass_comment.txt b/scripts/ci/release/test/resources/mass_comment.txt
@@ -0,0 +1,103 @@
+Run Release Gradle Build
+Run CommunityMetrics PreCommit
+Run Dataflow Runner Nexmark Tests
+Run Dataflow Runner V2 Java 11 Nexmark Tests
+Run Dataflow Runner V2 Java 17 Nexmark Tests
+Run Dataflow Runner V2 Nexmark Tests
+Run Dataflow Streaming ValidatesRunner
+Run Dataflow ValidatesRunner Java 11
+Run Dataflow ValidatesRunner Java 17
+Run Dataflow ValidatesRunner
+Run Direct Runner Nexmark Tests
+Run Direct ValidatesRunner Java 11
+Run Direct ValidatesRunner Java 17
+Run Direct ValidatesRunner in Java 11
+Run Direct ValidatesRunner
+Run Flink Runner Nexmark Tests
+Run Flink ValidatesRunner Java 11
+Run Flink ValidatesRunner
+Run Go Flink ValidatesRunner
+Run Go PostCommit
+Run Go PreCommit
+Run Go Samza ValidatesRunner
+Run Go Spark ValidatesRunner
+Run GoPortable PreCommit
+Run Java 11 Examples on Dataflow Runner V2
+Run Java 17 Examples on Dataflow Runner V2
+Run Java Dataflow V2 ValidatesRunner Streaming
+Run Java Dataflow V2 ValidatesRunner
+Run Java Examples on Dataflow Runner V2
+Run Java Examples_Direct
+Run Java Examples_Flink
+Run Java Examples_Spark
+Run Java Flink PortableValidatesRunner Streaming
+Run Java Portability examples on Dataflow with Java 11
+Run Java PostCommit
+Run Java PreCommit
+Run Java Samza PortableValidatesRunner
+Run Java Spark PortableValidatesRunner Batch
+Run Java Spark v2 PortableValidatesRunner Streaming
+Run Java Spark v3 PortableValidatesRunner Streaming
+Run Java examples on Dataflow Java 11
+Run Java examples on Dataflow Java 17
+Run Java examples on Dataflow with Java 11
+Run Java_Examples_Dataflow PreCommit
+Run Java_Examples_Dataflow_Java11 PreCommit
+Run Java_Examples_Dataflow_Java17 PreCommit
+Run Java_PVR_Flink_Batch PreCommit
+Run Java_PVR_Flink_Docker PreCommit
+Run Javadoc PostCommit
+Run Jpms Dataflow Java 11 PostCommit
+Run Jpms Dataflow Java 17 PostCommit
+Run Jpms Direct Java 11 PostCommit
+Run Jpms Direct Java 17 PostCommit
+Run Jpms Flink Java 11 PostCommit
+Run Jpms Spark Java 11 PostCommit
+Run PortableJar_Flink PostCommit
+Run PortableJar_Spark PostCommit
+Run Portable_Python PreCommit
+Run PostCommit_Java_Dataflow
+Run PostCommit_Java_DataflowV2
+Run PostCommit_Java_Hadoop_Versions
+Run Python 3.7 PostCommit
+Run Python 3.8 PostCommit
+Run Python 3.9 PostCommit
+Run Python Dataflow V2 ValidatesRunner
+Run Python Dataflow ValidatesContainer
+Run Python Dataflow ValidatesRunner
+Run Python Examples_Dataflow
+Run Python Examples_Direct
+Run Python Examples_Flink
+Run Python Examples_Spark
+Run Python Flink ValidatesRunner
+Run Python PreCommit
+Run Python Samza ValidatesRunner
+Run Python Spark ValidatesRunner
+Run PythonDocker PreCommit
+Run PythonDocs PreCommit
+Run PythonFormatter PreCommit
+Run PythonLint PreCommit
+Run Python_PVR_Flink PreCommit
+Run RAT PreCommit
+Run SQL PostCommit
+Run SQL PreCommit
+Run SQL_Java11 PreCommit
+Run SQL_Java17 PreCommit
+Run Samza ValidatesRunner
+Run Spark Runner Nexmark Tests
+Run Spark StructuredStreaming ValidatesRunner
+Run Spark ValidatesRunner Java 11
+Run Spark ValidatesRunner
+Run Spotless PreCommit
+Run Twister2 ValidatesRunner
+Run Typescript PreCommit
+Run ULR Loopback ValidatesRunner
+Run Whitespace PreCommit
+Run XVR_Direct PostCommit
+Run XVR_Flink PostCommit
+Run XVR_JavaUsingPython_Dataflow PostCommit
+Run XVR_PythonUsingJavaSQL_Dataflow PostCommit
+Run XVR_PythonUsingJava_Dataflow PostCommit
+Run XVR_Samza PostCommit
+Run XVR_Spark PostCommit
+Run XVR_Spark3 PostCommit