diff --git a/.github/actions/setup-gh-cli-linux/action.yml b/.github/actions/setup-gh-cli-linux/action.yml new file mode 100644 index 000000000000..41d38b7a56c5 --- /dev/null +++ b/.github/actions/setup-gh-cli-linux/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup GH CLI - Linux' +description: 'Setup GitHub CLI tool in a Linux Operating System.' +runs: + using: "composite" + steps: + - id: setup-gh-cli-linux + shell: bash + run: | + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ + && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh -y + gh --version diff --git a/.github/workflows/verify_release_build.yml b/.github/workflows/verify_release_build.yml new file mode 100644 index 000000000000..5a128a4eec61 --- /dev/null +++ b/.github/workflows/verify_release_build.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: Verify Release Build +on: + workflow_dispatch: + inputs: + RELEASE_VER: + description: Beam version of current release + required: true + COMMENT_TRIGGER_PHRASES: + description: Flag to comment the Jenkins Trigger Phrases in the new PR + required: true +env: + WORKING_BRANCH: postcommit_validation_pr +jobs: + verify_release: + runs-on: [self-hosted, ubuntu-20.04] + permissions: + pull-requests: write + contents: write + env: + RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }} + steps: + - name: Verify branch name + run: + - sh ./ci_check_git_branch.sh $WORKING_BRANCH + working-directory: 'scripts/ci' + - name: Set RELEASE_BRANCH env variable + run: | + RELEASE_BRANCH=release-${{env.RELEASE_VER}} + echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV + - name: Check out code + uses: actions/checkout@v3 + with: + ref: ${{ env.RELEASE_BRANCH }} + - name: Install gh cli + uses: ./.github/actions/setup-gh-cli-linux + - name: Set git config + run: | + git config user.name $GITHUB_ACTOR + git config user.email actions@"$RUNNER_NAME".local + - name: Set Release Version + run: | + git checkout -b ${{env.WORKING_BRANCH}} --quiet + /bin/bash release/src/main/scripts/set_version.sh "$RELEASE_VER" --git-add + # In case the version string was not changed, append a newline to CHANGES.md + echo "" >> CHANGES.md + - name: Push Changes in ${{ env.WORKING_BRANCH }} Branch + run: | + git add CHANGES.md + git commit -m "Changed version.py and gradle.properties to python dev version to create a test PR" --quiet + git push origin $WORKING_BRANCH --quiet + - name: Create new Release PR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t "[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch" -b "Run Gradle Release Build and Jenkins PreCommit and PostCommits tests automatically in this PR (refer to scripts/ci/release/mass_comment.txt). Please run the missing tests manually.") + # Run Gradle Release Build and PostCommit/PreCommit Tests against Release Branch on Jenkins. + - name: Comment Jenkins Trigger Phrases in PR + if: ${{ github.event.inputs.COMMENT_TRIGGER_PHRASES == 'true' }} + run: | + sh ./comment_pr_trigger_phrases.sh $GITHUB_PR_URL + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: 'scripts/ci/release' diff --git a/CI.md b/CI.md index ee911584f7f0..0fcc87da9521 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,13 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### Release Preparation and Validation Workflows +#### Verify Release Build - [verify_release_build.yml](.github/workflows/verify_release_build.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Verify Release Build | Verifies full life cycle of Gradle Build and all PostCommit/PreCommit tests against Release Branch on CI. | No | No | No | No | + + ### All migrated workflows run based on the following triggers | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Workflow Dispatch | diff --git a/scripts/ci/ci_check_git_branch.sh b/scripts/ci/ci_check_git_branch.sh new file mode 100644 index 000000000000..e7bff3e77998 --- /dev/null +++ b/scripts/ci/ci_check_git_branch.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function is_branch() { + #if nothing matches show-ref will return an error code of 1 + if git show-ref --quiet --verify -- "refs/heads/$1" ; then + return 1 + else + return 0 + fi +} + +if is_branch "$1"; then + echo "Branch [$1] doesn't exist." +else + echo "Branch [$1] already exists!" + echo >&2 "Please make sure your branch doesn't exist." +fi diff --git a/scripts/ci/release/comment_pr_trigger_phrases.sh b/scripts/ci/release/comment_pr_trigger_phrases.sh new file mode 100755 index 000000000000..10479dd94dd8 --- /dev/null +++ b/scripts/ci/release/comment_pr_trigger_phrases.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script adds comments from a txt file, in the given GitHub Pull Request. +# test/resources/mass_comment.txt file has the Trigger Phrases for running Gradle Build and PostCommit/PreCommit tests. + +file="./test/resources/mass_comment.txt" +GITHUB_PR=$1 +while IFS= read -r trigger_phrase +do + gh pr comment "$GITHUB_PR" --body "$trigger_phrase" +done <"$file" diff --git a/scripts/ci/release/test/resources/jenkins_trigger_phrases.txt b/scripts/ci/release/test/resources/jenkins_trigger_phrases.txt new file mode 100644 index 000000000000..4f59885de8b4 --- /dev/null +++ b/scripts/ci/release/test/resources/jenkins_trigger_phrases.txt @@ -0,0 +1,19 @@ +Run Release Gradle Build +Run CommunityMetrics PreCommit +Run Direct ValidatesRunner Java 11 +Run Direct ValidatesRunner Java 17 +Run Java 11 Examples on Dataflow Runner V2 +Run Java 17 Examples on Dataflow Runner V2 +Run Java PreCommit +Run Java examples on Dataflow Java 11 +Run Java examples on Dataflow Java 17 +Run Javadoc PostCommit +Run Jpms Dataflow Java 17 PostCommit +Run Jpms Direct Java 17 PostCommit +Run Python Flink ValidatesRunner +Run Python PreCommit +Run Python Samza ValidatesRunner +Run Python Spark ValidatesRunner +Run SQL_Java17 PreCommit +Run XVR_Flink PostCommit +Run XVR_GoUsingJava_Dataflow PostCommit diff --git a/scripts/ci/release/test/resources/mass_comment.txt b/scripts/ci/release/test/resources/mass_comment.txt new file mode 100644 index 000000000000..b3a307c9e698 --- /dev/null +++ b/scripts/ci/release/test/resources/mass_comment.txt @@ -0,0 +1,103 @@ +Run Release Gradle Build +Run CommunityMetrics PreCommit +Run Dataflow Runner Nexmark Tests +Run Dataflow Runner V2 Java 11 Nexmark Tests +Run Dataflow Runner V2 Java 17 Nexmark Tests +Run Dataflow Runner V2 Nexmark Tests +Run Dataflow Streaming ValidatesRunner +Run Dataflow ValidatesRunner Java 11 +Run Dataflow ValidatesRunner Java 17 +Run Dataflow ValidatesRunner +Run Direct Runner Nexmark Tests +Run Direct ValidatesRunner Java 11 +Run Direct ValidatesRunner Java 17 +Run Direct ValidatesRunner in Java 11 +Run Direct ValidatesRunner +Run Flink Runner Nexmark Tests +Run Flink ValidatesRunner Java 11 +Run Flink ValidatesRunner +Run Go Flink ValidatesRunner +Run Go PostCommit +Run Go PreCommit +Run Go Samza ValidatesRunner +Run Go Spark ValidatesRunner +Run GoPortable PreCommit +Run Java 11 Examples on Dataflow Runner V2 +Run Java 17 Examples on Dataflow Runner V2 +Run Java Dataflow V2 ValidatesRunner Streaming +Run Java Dataflow V2 ValidatesRunner +Run Java Examples on Dataflow Runner V2 +Run Java Examples_Direct +Run Java Examples_Flink +Run Java Examples_Spark +Run Java Flink PortableValidatesRunner Streaming +Run Java Portability examples on Dataflow with Java 11 +Run Java PostCommit +Run Java PreCommit +Run Java Samza PortableValidatesRunner +Run Java Spark PortableValidatesRunner Batch +Run Java Spark v2 PortableValidatesRunner Streaming +Run Java Spark v3 PortableValidatesRunner Streaming +Run Java examples on Dataflow Java 11 +Run Java examples on Dataflow Java 17 +Run Java examples on Dataflow with Java 11 +Run Java_Examples_Dataflow PreCommit +Run Java_Examples_Dataflow_Java11 PreCommit +Run Java_Examples_Dataflow_Java17 PreCommit +Run Java_PVR_Flink_Batch PreCommit +Run Java_PVR_Flink_Docker PreCommit +Run Javadoc PostCommit +Run Jpms Dataflow Java 11 PostCommit +Run Jpms Dataflow Java 17 PostCommit +Run Jpms Direct Java 11 PostCommit +Run Jpms Direct Java 17 PostCommit +Run Jpms Flink Java 11 PostCommit +Run Jpms Spark Java 11 PostCommit +Run PortableJar_Flink PostCommit +Run PortableJar_Spark PostCommit +Run Portable_Python PreCommit +Run PostCommit_Java_Dataflow +Run PostCommit_Java_DataflowV2 +Run PostCommit_Java_Hadoop_Versions +Run Python 3.7 PostCommit +Run Python 3.8 PostCommit +Run Python 3.9 PostCommit +Run Python Dataflow V2 ValidatesRunner +Run Python Dataflow ValidatesContainer +Run Python Dataflow ValidatesRunner +Run Python Examples_Dataflow +Run Python Examples_Direct +Run Python Examples_Flink +Run Python Examples_Spark +Run Python Flink ValidatesRunner +Run Python PreCommit +Run Python Samza ValidatesRunner +Run Python Spark ValidatesRunner +Run PythonDocker PreCommit +Run PythonDocs PreCommit +Run PythonFormatter PreCommit +Run PythonLint PreCommit +Run Python_PVR_Flink PreCommit +Run RAT PreCommit +Run SQL PostCommit +Run SQL PreCommit +Run SQL_Java11 PreCommit +Run SQL_Java17 PreCommit +Run Samza ValidatesRunner +Run Spark Runner Nexmark Tests +Run Spark StructuredStreaming ValidatesRunner +Run Spark ValidatesRunner Java 11 +Run Spark ValidatesRunner +Run Spotless PreCommit +Run Twister2 ValidatesRunner +Run Typescript PreCommit +Run ULR Loopback ValidatesRunner +Run Whitespace PreCommit +Run XVR_Direct PostCommit +Run XVR_Flink PostCommit +Run XVR_JavaUsingPython_Dataflow PostCommit +Run XVR_PythonUsingJavaSQL_Dataflow PostCommit +Run XVR_PythonUsingJava_Dataflow PostCommit +Run XVR_Samza PostCommit +Run XVR_Spark PostCommit +Run XVR_Spark3 PostCommit