From 4f94a1b415be83afa6647d3ac644104006c5b713 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Tue, 23 Aug 2022 13:09:21 -0600 Subject: [PATCH 01/24] precommit rat, spotless and whitespace --- .github/workflows/job-precommit-rat.yml | 45 +++++++++++++++++ .github/workflows/job-precommit-spotless.yml | 50 +++++++++++++++++++ .../workflows/job-precommit-whitespace.yml | 46 +++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 .github/workflows/job-precommit-rat.yml create mode 100644 .github/workflows/job-precommit-spotless.yml create mode 100644 .github/workflows/job-precommit-whitespace.yml diff --git a/.github/workflows/job-precommit-rat.yml b/.github/workflows/job-precommit-rat.yml new file mode 100644 index 000000000000..800735546ad3 --- /dev/null +++ b/.github/workflows/job-precommit-rat.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: Precommit RAT + +on: + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + precommit-rat: + name: Run Precommit RAT + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run RAT + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: rat diff --git a/.github/workflows/job-precommit-spotless.yml b/.github/workflows/job-precommit-spotless.yml new file mode 100644 index 000000000000..79e8a10503f3 --- /dev/null +++ b/.github/workflows/job-precommit-spotless.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: Precommit Spotless + +on: + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['buildSrc/**', + 'sdks/java/**', + 'runners/**', + 'examples/java/**', + 'examples/kotlin/**'] + +jobs: + spotless: + name: Run Spotless + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run spotlessCheck checkStyleMain checkStyleTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: spotlessCheck checkStyleMain checkStyleTest diff --git a/.github/workflows/job-precommit-whitespace.yml b/.github/workflows/job-precommit-whitespace.yml new file mode 100644 index 000000000000..c00f0e4f819f --- /dev/null +++ b/.github/workflows/job-precommit-whitespace.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: Precommit Whitespace + +on: + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['.*\\.md$', '.*build\\.gradle$'] + +jobs: + precommit-whitespace: + name: Run Precommit Whitespace + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run Whitespace + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: whitespacePreCommit \ No newline at end of file From 83e069808ad9d7215e35523d7278c931a21edbb8 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Tue, 23 Aug 2022 13:40:31 -0600 Subject: [PATCH 02/24] removing files --- .test-infra/jenkins/README.md | 3 -- .test-infra/jenkins/job_PreCommit_RAT.groovy | 26 ----------- .../jenkins/job_PreCommit_Spotless.groovy | 45 ------------------- .../jenkins/job_PreCommit_Whitespace.groovy | 30 ------------- 4 files changed, 104 deletions(-) delete mode 100644 .test-infra/jenkins/job_PreCommit_RAT.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Spotless.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Whitespace.groovy diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 2c252a4d55aa..6a953c33d16e 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -48,13 +48,10 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PreCommit_PythonDocs| [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Phrase/) | `Run PythonDocs PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/) | | beam_PreCommit_Python_PVR_Flink | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Phrase/) | `Run Python_PVR_Flink PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron) | | beam_PreCommit_PythonFormatter | [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Phrase/) | `Run PythonFormatter PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/) | -| beam_PreCommit_RAT | [commit](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Phrase/) | `Run RAT PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron) | -| beam_PreCommit_Spotless | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Phrase/) | `Run Spotless PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron) | | beam_PreCommit_SQL | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Phrase/) | `Run SQL PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/) | | beam_PreCommit_SQL_Java11 | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Phrase/) | `Run SQL_Java11 PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/) | | beam_PreCommit_Website | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Website_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Website_Phrase/) | `Run Website PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron) | | beam_PreCommit_Website_Stage_GCS | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Phrase/) | `Run Website_Stage_GCS PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron) | -| beam_PreCommit_Whitespace | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Phrase/) | `Run Whitespace PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron) | | beam_PreCommit_Typescript | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Phrase/) | `Run Typescript PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron) | ### PostCommit Jobs diff --git a/.test-infra/jenkins/job_PreCommit_RAT.groovy b/.test-infra/jenkins/job_PreCommit_RAT.groovy deleted file mode 100644 index 613caa9af0de..000000000000 --- a/.test-infra/jenkins/job_PreCommit_RAT.groovy +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'RAT', - gradleTask: ':rat' - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Spotless.groovy b/.test-infra/jenkins/job_PreCommit_Spotless.groovy deleted file mode 100644 index a9da1ad5491a..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Spotless.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Spotless', - gradleTask: 'spotlessCheck checkStyleMain checkStyleTest', - triggerPathPatterns: [ - '^buildSrc/.*$', - '^sdks/java/.*$', - '^runners/.*$', - '^examples/java/.*$', - '^examples/kotlin/.*$', - '^.test-infra/jenkins/.*$', - ] - ) -builder.build { - publishers { - recordIssues { - tools { - checkStyle { - pattern('**/build/reports/checkstyle/*.xml') - } - } - enabledForFailure(true) - } - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Whitespace.groovy b/.test-infra/jenkins/job_PreCommit_Whitespace.groovy deleted file mode 100644 index 1b8341395a68..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Whitespace.groovy +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Whitespace', - gradleTask: ':whitespacePreCommit', - triggerPathPatterns: [ - '.*\\.md$', - '.*build\\.gradle$', - ] - ) -builder.build() From 7d8cef864deb38ce9316634c1c5112b65ed1b1a1 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Thu, 8 Sep 2022 00:35:31 -0600 Subject: [PATCH 03/24] Update job-precommit-rat.yml --- .github/workflows/job-precommit-rat.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/job-precommit-rat.yml b/.github/workflows/job-precommit-rat.yml index 800735546ad3..98f8cab68707 100644 --- a/.github/workflows/job-precommit-rat.yml +++ b/.github/workflows/job-precommit-rat.yml @@ -17,7 +17,7 @@ # To learn more about GitHub Actions in Apache Beam check the CI.md -name: Precommit RAT +name: PreCommit RAT on: push: @@ -28,8 +28,8 @@ on: tags: 'v*' jobs: - precommit-rat: - name: Run Precommit RAT + rat: + name: Run PreCommit RAT runs-on: self-hosted steps: - name: Checkout code @@ -39,7 +39,7 @@ jobs: submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - - name: Run RAT + - name: Run PreCommit RAT uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: rat From 326e1afb55709a51cd1592c80cf45f9842d085de Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Thu, 8 Sep 2022 00:36:32 -0600 Subject: [PATCH 04/24] Update job-precommit-spotless.yml --- .github/workflows/job-precommit-spotless.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/job-precommit-spotless.yml b/.github/workflows/job-precommit-spotless.yml index 79e8a10503f3..0622d00da3cc 100644 --- a/.github/workflows/job-precommit-spotless.yml +++ b/.github/workflows/job-precommit-spotless.yml @@ -17,7 +17,7 @@ # To learn more about GitHub Actions in Apache Beam check the CI.md -name: Precommit Spotless +name: PreCommit Spotless on: push: @@ -34,7 +34,7 @@ on: jobs: spotless: - name: Run Spotless + name: Run PreCommit Spotless runs-on: self-hosted steps: - name: Checkout code @@ -48,3 +48,4 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: spotlessCheck checkStyleMain checkStyleTest + From af6fdc1db803694fab401b7c9db371f93fb087a2 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Thu, 8 Sep 2022 00:37:34 -0600 Subject: [PATCH 05/24] Update job-precommit-whitespace.yml --- .github/workflows/job-precommit-whitespace.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/job-precommit-whitespace.yml b/.github/workflows/job-precommit-whitespace.yml index c00f0e4f819f..7605181149b1 100644 --- a/.github/workflows/job-precommit-whitespace.yml +++ b/.github/workflows/job-precommit-whitespace.yml @@ -17,7 +17,7 @@ # To learn more about GitHub Actions in Apache Beam check the CI.md -name: Precommit Whitespace +name: PreCommit Whitespace on: push: @@ -29,8 +29,8 @@ on: paths: ['.*\\.md$', '.*build\\.gradle$'] jobs: - precommit-whitespace: - name: Run Precommit Whitespace + whitespace: + name: Run PreCommit Whitespace runs-on: self-hosted steps: - name: Checkout code @@ -40,7 +40,7 @@ jobs: submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - - name: Run Whitespace + - name: Run PreCommit Whitespace uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: whitespacePreCommit \ No newline at end of file + gradle-command: whitespacePreCommit From 9b648d5b2265286387d9a0c629799feb3cd35a2f Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 8 Sep 2022 14:31:50 -0500 Subject: [PATCH 06/24] Base actions to use for precommit and postcommit workflows --- .../action.yml | 45 +++++++++++++++++ .../setup-default-test-properties/action.yml | 31 ++++++++++++ .../test-properties.json | 17 +++++++ .../setup-self-hosted-action/action.yml | 50 +++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 .github/actions/gradle-command-self-hosted-action/action.yml create mode 100644 .github/actions/setup-default-test-properties/action.yml create mode 100644 .github/actions/setup-default-test-properties/test-properties.json create mode 100644 .github/actions/setup-self-hosted-action/action.yml diff --git a/.github/actions/gradle-command-self-hosted-action/action.yml b/.github/actions/gradle-command-self-hosted-action/action.yml new file mode 100644 index 000000000000..7dd835e1032e --- /dev/null +++ b/.github/actions/gradle-command-self-hosted-action/action.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Gradle command' +description: 'Configuration actions to run a gradle command in self-hosted runner' +inputs: + gradle-command: + required: true + arguments: + required: false + description: 'Gradle options' + default: '' + default-arguments: + required: false + description: 'Default gradle switches' # Copied from CommonJobProperties.groovy' + default: | + --continue -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g \ + -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses + max-workers: + required: false + description: 'Max number of workers' + default: 8 + +runs: + using: "composite" + steps: + - name: Run Gradle Command + shell: bash + run: | + ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \ + ${{ inputs.default-arguments }} \ No newline at end of file diff --git a/.github/actions/setup-default-test-properties/action.yml b/.github/actions/setup-default-test-properties/action.yml new file mode 100644 index 000000000000..78144098e285 --- /dev/null +++ b/.github/actions/setup-default-test-properties/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup TestProperties' +description: 'Setup default test properties from a Json config file' +outputs: + properties: + description: "Test properties" + value: ${{ steps.test-properties-json.outputs.test-properties }} +runs: + using: "composite" + steps: + - id: test-properties-json + shell: bash + run: | + JSON=$(cat ./.github/actions/setup-default-test-properties/test-properties.json) + echo "::set-output name=test-properties::$(echo $JSON)" \ No newline at end of file diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json new file mode 100644 index 000000000000..9cae4b5ec8c6 --- /dev/null +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -0,0 +1,17 @@ +{ + "PythonTestProperties": { + "ALL_SUPPORTED_VERSIONS": ["3.7", "3.8", "3.9"], + "LOWEST_SUPPORTED": ["3.7"], + "HIGHEST_SUPPORTED": ["3.9"], + "ESSENTIAL_VERSIONS": ["3.7", "3.9"], + "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.7", "3.9"], + "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.9"], + "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.7", "3.8", "3.9"], + "LOAD_TEST_PYTHON_VERSION": "3.7", + "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.7", + "DEFAULT_INTERPRETER": "python3.8" + }, + "JavaTestProperties": { + "SUPPORTED_CONTAINER_TASKS": ["java8", "java11", "java17"] + } +} \ No newline at end of file diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml new file mode 100644 index 000000000000..6b190629859b --- /dev/null +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup environment for self-hosted runners' +description: 'Setup action to run jobs in a self-hosted runner' + +runs: + using: "composite" + steps: + - name: Install python 3.7 + uses: actions/setup-python@v4 + with: + python-version: "3.7" + - name: Install python 3.8 + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install python 3.9 + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Set Java Version + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 8 + - name: Set Go Version + uses: actions/setup-go@v3 + with: + go-version: '>=1.16.0' + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue of Beam's + # gradle-command-action plugin and github's provided maven + # settings.xml file + shell: bash + run: rm ~/.m2/settings.xml \ No newline at end of file From 774f5f895f006f530edbc8ed643ba1a42ac01448 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Thu, 8 Sep 2022 14:44:51 -0600 Subject: [PATCH 07/24] Adding Java, Flink, Spark and Golang supported versions to json file --- .../setup-default-test-properties/test-properties.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index 9cae4b5ec8c6..324329035b71 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -12,6 +12,11 @@ "DEFAULT_INTERPRETER": "python3.8" }, "JavaTestProperties": { - "SUPPORTED_CONTAINER_TASKS": ["java8", "java11", "java17"] + "SUPPORTED_VERSIONS": ["8", "11", "17"], + "FLINK_VERSIONS": ["1.13", "1.14", "1.15"], + "SPARK_VERSIONS": ["2", "3"] + }, + "GoTestProperties": { + "SUPPORTED_VERSIONS": ["1.18"] } } \ No newline at end of file From dff0b3752ddecdf4c037d7293f833087845e66fb Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Thu, 8 Sep 2022 15:51:25 -0600 Subject: [PATCH 08/24] updating CI.md and yaml files --- .github/workflows/job-precommit-rat.yml | 2 ++ .github/workflows/job-precommit-spotless.yml | 2 ++ .../workflows/job-precommit-whitespace.yml | 2 ++ CI.md | 20 +++++++++++++++++++ 4 files changed, 26 insertions(+) diff --git a/.github/workflows/job-precommit-rat.yml b/.github/workflows/job-precommit-rat.yml index 98f8cab68707..0d8c5fe3dcbf 100644 --- a/.github/workflows/job-precommit-rat.yml +++ b/.github/workflows/job-precommit-rat.yml @@ -17,6 +17,8 @@ # To learn more about GitHub Actions in Apache Beam check the CI.md +# Runs RAT scheduled on cron, on push to master and on pull request + name: PreCommit RAT on: diff --git a/.github/workflows/job-precommit-spotless.yml b/.github/workflows/job-precommit-spotless.yml index 0622d00da3cc..d588b55cce56 100644 --- a/.github/workflows/job-precommit-spotless.yml +++ b/.github/workflows/job-precommit-spotless.yml @@ -17,6 +17,8 @@ # To learn more about GitHub Actions in Apache Beam check the CI.md +# Runs Spotless scheduled on cron, on push to master and on pull request + name: PreCommit Spotless on: diff --git a/.github/workflows/job-precommit-whitespace.yml b/.github/workflows/job-precommit-whitespace.yml index 7605181149b1..10d596ec1e89 100644 --- a/.github/workflows/job-precommit-whitespace.yml +++ b/.github/workflows/job-precommit-whitespace.yml @@ -17,6 +17,8 @@ # To learn more about GitHub Actions in Apache Beam check the CI.md +# Runs Whitespace scheduled on cron, on push to master and on pull request + name: PreCommit Whitespace on: diff --git a/CI.md b/CI.md index ed75a4dd4433..1bda3bce7670 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,26 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PreCommit Workflows + +#### RAT PreCommit - [job-precommit-rat.yml](.github/workflows/job-precommit-rat.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|----------------------|---------------------|------------------|-----------------------|---------------|--------------------------| +| Run RAT PreCommit | Runs RAT PreCommit | Yes | Yes | Yes | No | + +#### Spotless PreCommit - [job-precommit-spotless.yml](.github/workflows/job-precommit-spotless.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|---------------------------|-------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Spotless PreCommit | Runs Spotless PreCommit | Yes | Yes | Yes | No | + +#### Whitespace PreCommit - [job-precommit-spotless.yml](.github/workflows/job-precommit-spotless.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|--------------------------|---------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Whitespace PreCommit | Runs Whitespace PreCommit | Yes | Yes | Yes | No | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From d8689ee9092cc2209682087410c37d97961c989d Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Fri, 9 Sep 2022 01:16:57 -0600 Subject: [PATCH 09/24] switching to temurin due to Apache License compatibility --- .github/actions/setup-self-hosted-action/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml index 6b190629859b..825251d44085 100644 --- a/.github/actions/setup-self-hosted-action/action.yml +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -36,7 +36,7 @@ runs: - name: Set Java Version uses: actions/setup-java@v3 with: - distribution: 'zulu' + distribution: 'temurin' java-version: 8 - name: Set Go Version uses: actions/setup-go@v3 From e1240a9c77774dd7f37abcbbe96a3c314e8ea8dc Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Mon, 12 Sep 2022 13:11:36 -0600 Subject: [PATCH 10/24] adding postcommit java dataflow v1 and v2 --- .../job-postcommit-java-dataflow-v1.yml | 72 +++++++++++++++++++ .../job-postcommit-java-dataflow-v2.yml | 72 +++++++++++++++++++ .test-infra/jenkins/README.md | 2 - .../job_PostCommit_Java_Dataflow.groovy | 48 ------------- .../job_PostCommit_Java_DataflowV2.groovy | 48 ------------- CI.md | 14 ++++ 6 files changed, 158 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/job-postcommit-java-dataflow-v1.yml create mode 100644 .github/workflows/job-postcommit-java-dataflow-v2.yml delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy diff --git a/.github/workflows/job-postcommit-java-dataflow-v1.yml b/.github/workflows/job-postcommit-java-dataflow-v1.yml new file mode 100644 index 000000000000..76e28b530831 --- /dev/null +++ b/.github/workflows/job-postcommit-java-dataflow-v1.yml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +# Runs Java PostCommit on Dataflow runner v1 via cron and on push to master + +name: PostCommit Java Dataflow v1 + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master'] + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + postcommit-java-dataflow-v1: + needs: set-properties + name: Run PostCommit Java Dataflow v1 + runs-on: self-hosted + strategy: + fail-fast: true + matrix: #JDK11 and 17 compilation will fail due to changes in ByteBuffer class since JDK9 + #TODO: Retest once issue https://github.com/apache/beam/issues/23068 gets resolved. + java: ${{fromJson(needs.set-properties.outputs.properties).JavaTestProperties.SUPPORTED_VERSIONS}} + exclude: + - java: "11" + - java: "17" + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + - name: Run :runners:google-cloud-dataflow-java:postCommit in Java-${{ matrix.java }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:postCommit diff --git a/.github/workflows/job-postcommit-java-dataflow-v2.yml b/.github/workflows/job-postcommit-java-dataflow-v2.yml new file mode 100644 index 000000000000..ee6980eb402e --- /dev/null +++ b/.github/workflows/job-postcommit-java-dataflow-v2.yml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +# Runs Java PostCommit on Dataflow runner v2 via cron and on push to master + +name: PostCommit Java Dataflow v2 + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master'] + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + postcommit-java-dataflow-v2: + needs: set-properties + name: Run PostCommit Java Dataflow v2 + runs-on: self-hosted + strategy: + fail-fast: true + matrix: #JDK11 and 17 compilation will fail due to changes in ByteBuffer class since JDK9 + #TODO: Retest once issue https://github.com/apache/beam/issues/23068 gets resolved. + java: ${{fromJson(needs.set-properties.outputs.properties).JavaTestProperties.SUPPORTED_VERSIONS}} + exclude: + - java: "11" + - java: "17" + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + - name: Run :runners:google-cloud-dataflow-java:postCommitRunnerV2 in Java-${{ matrix.java }} + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:postCommitRunnerV2 diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 2c252a4d55aa..25d7ce448e86 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -74,8 +74,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Go_VR_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark_PR/) | `Run Go Spark ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/) | | beam_PostCommit_Java_Hadoop_Versions | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Hadoop_Versions/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Hadoop_Versions_PR/) | `Run PostCommit_Java_Hadoop_Versions` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Hadoop_Versions/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Hadoop_Versions/) | | beam_PostCommit_Java | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_PR/) | `Run Java PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Java/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Java) | -| beam_PostCommit_Java_DataflowV1 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1_PR/) | `Run PostCommit_Java_Dataflow` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1/) | -| beam_PostCommit_Java_DataflowV2 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2_PR/) | `Run PostCommit_Java_DataflowV2` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2/) | | beam_PostCommit_Java_Nexmark_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow_PR/) | `Dataflow Runner Nexmark Tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow) | | beam_PostCommit_Java_Nexmark_Dataflow_V2 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_DataflowV2_PR/) | `Run Dataflow Runner V2 Nexmark Tests` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2/) | | beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/badge/icon) | `Dataflow Runner V2 Java 11 Nexmark Tests` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/) | diff --git a/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy deleted file mode 100644 index 2ab27da0f1b0..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - - -// This job runs the Java postcommit tests, including the suite of integration -// tests. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_DataflowV1', 'Run PostCommit_Java_Dataflow', - 'Dataflow Java Post Commit Tests', this) { - - description('Dataflow Java Post Commit Tests') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":runners:google-cloud-dataflow-java:postCommit") - commonJobProperties.setGradleSwitches(delegate) - // Specify maven home on Jenkins, needed by Maven archetype integration tests. - switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy b/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy deleted file mode 100644 index 03d82d48974e..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - - -// This job runs the Java postcommit tests, including the suite of integration -// tests. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_DataflowV2', 'Run PostCommit_Java_DataflowV2', - 'Dataflow V2 Java Post Commit Tests', this) { - - description('Dataflow V2 Java Post Commit Tests') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":runners:google-cloud-dataflow-java:postCommitRunnerV2") - commonJobProperties.setGradleSwitches(delegate) - // Specify maven home on Jenkins, needed by Maven archetype integration tests. - switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') - } - } - } diff --git a/CI.md b/CI.md index ed75a4dd4433..c70ed6068a73 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,20 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PostCommit Workflows + +#### Java PostCommit DataFlow v1- [job-postcommit-java-dataflow-v1.yml](.github/workflows/job-postcommit-java-dataflow-v1.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|---------------------------------|----------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java PostCommit DataFlow v1 | Runs Java PostCommit DataFlow v1 | No | Yes | Yes | Yes | + +#### Java PostCommit DataFlow v2- [job-postcommit-java-dataflow-v2.yml](.github/workflows/job-postcommit-java-dataflow-v2.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-----------------------------------|------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java PostCommit DataFlow v2 | Runs Java PostCommit DataFlow v2 | No | Yes | Yes | Yes | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From 5f407a4dd7900ad00793f0f2fd01296d0ff97c1f Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Tue, 13 Sep 2022 08:32:51 -0600 Subject: [PATCH 11/24] Update job-postcommit-java-dataflow-v1.yml --- .github/workflows/job-postcommit-java-dataflow-v1.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/job-postcommit-java-dataflow-v1.yml b/.github/workflows/job-postcommit-java-dataflow-v1.yml index 76e28b530831..06a915df473f 100644 --- a/.github/workflows/job-postcommit-java-dataflow-v1.yml +++ b/.github/workflows/job-postcommit-java-dataflow-v1.yml @@ -53,6 +53,7 @@ jobs: exclude: - java: "11" - java: "17" + timeout-minutes: 1200 steps: - name: Checkout code uses: actions/checkout@v2 From 2d74377cd283c07092fbd67c69ead448d82ecf42 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Tue, 13 Sep 2022 10:08:51 -0600 Subject: [PATCH 12/24] excluding github and website dirs --- sdks/python/test-suites/tox/py38/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 20dfbfe0fd5a..b8895d400ce0 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -104,6 +104,8 @@ task archiveFilesToLint(type: Zip) { include "**/*.md" include "**/build.gradle" exclude "**/node_modules/*" + exclude ".github/*" + exclude "website/*" } } From d82c23441b95c66d6d697e4d2d580f04cde32713 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Tue, 13 Sep 2022 12:00:29 -0600 Subject: [PATCH 13/24] setting go version to 1.18 --- .github/actions/setup-self-hosted-action/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml index 825251d44085..565c0500989e 100644 --- a/.github/actions/setup-self-hosted-action/action.yml +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -41,10 +41,10 @@ runs: - name: Set Go Version uses: actions/setup-go@v3 with: - go-version: '>=1.16.0' + go-version: '1.18' - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # gradle-command-action plugin and github's provided maven # settings.xml file shell: bash - run: rm ~/.m2/settings.xml \ No newline at end of file + run: rm ~/.m2/settings.xml From ab7c627c4d0f8789ed7aacc07230f2e7ab7bb134 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Wed, 14 Sep 2022 12:22:40 -0600 Subject: [PATCH 14/24] Update DockerEnvironmentFactory.java --- .../fnexecution/environment/DockerEnvironmentFactory.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index ee816a944e5a..edead73227d7 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,8 +182,7 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "--mount", - String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); + "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); } From 3a229f0e1762d5c2bad580f12e515c18b28373ea Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Wed, 14 Sep 2022 12:50:12 -0600 Subject: [PATCH 15/24] Update job-postcommit-java-dataflow-v1.yml --- .github/workflows/job-postcommit-java-dataflow-v1.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/job-postcommit-java-dataflow-v1.yml b/.github/workflows/job-postcommit-java-dataflow-v1.yml index 06a915df473f..ddff406ef92a 100644 --- a/.github/workflows/job-postcommit-java-dataflow-v1.yml +++ b/.github/workflows/job-postcommit-java-dataflow-v1.yml @@ -22,6 +22,7 @@ name: PostCommit Java Dataflow v1 on: + workflow_dispatch: schedule: - cron: '0 */6 * * *' push: From 53384e06e4daef2061b3e9acfa249e48503971d4 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 14 Sep 2022 16:48:30 -0500 Subject: [PATCH 16/24] Add environment as optionals in setup-base-action --- .../test-properties.json | 3 +- .../setup-self-hosted-action/action.yml | 29 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index 324329035b71..6d30acafbb04 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -9,7 +9,8 @@ "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.7", "3.8", "3.9"], "LOAD_TEST_PYTHON_VERSION": "3.7", "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.7", - "DEFAULT_INTERPRETER": "python3.8" + "DEFAULT_INTERPRETER": "python3.8", + "TOX_ENV": ["Cloud", "Cython"] }, "JavaTestProperties": { "SUPPORTED_VERSIONS": ["8", "11", "17"], diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml index 565c0500989e..b268357c08e6 100644 --- a/.github/actions/setup-self-hosted-action/action.yml +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -17,32 +17,59 @@ name: 'Setup environment for self-hosted runners' description: 'Setup action to run jobs in a self-hosted runner' +inputs: + requires-py-37: + required: false + description: 'Set as false if does not require py37 setup' + default: 'true' + requires-py-38: + required: false + description: 'Set as false if does not require py38 setup' + default: 'true' + requires-py-39: + required: false + description: 'Set as false if does not require py39 setup' + default: 'true' + requires-java-8: + required: false + description: 'Set as false if does not require java-8 setup' + default: 'true' + requires-go-18: + required: false + description: 'Set as false if does not require go-18 setup' + default: 'true' runs: using: "composite" steps: - name: Install python 3.7 + if: ${{ inputs.requires-py-37 == 'true' }} uses: actions/setup-python@v4 with: python-version: "3.7" - name: Install python 3.8 + if: ${{ inputs.requires-py-38 == 'true' }} uses: actions/setup-python@v4 with: python-version: "3.8" - name: Install python 3.9 + if: ${{ inputs.requires-py-39 == 'true' }} uses: actions/setup-python@v4 with: python-version: "3.9" - name: Set Java Version + if: ${{ inputs.requires-java-8 == 'true' }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: 8 - name: Set Go Version + if: ${{ inputs.requires-go-18 == 'true' }} uses: actions/setup-go@v3 with: - go-version: '1.18' + go-version: '1.18.0' - name: Remove default github maven configuration + if: ${{ inputs.requires-java-8 == 'true' }} # This step is a workaround to avoid a decryption issue of Beam's # gradle-command-action plugin and github's provided maven # settings.xml file From 9377bc69d50d32dfe41c3348a5166a20c885a8b6 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Thu, 15 Sep 2022 08:48:56 -0600 Subject: [PATCH 17/24] Update DockerEnvironmentFactory.java --- .../fnexecution/environment/DockerEnvironmentFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index edead73227d7..1d6fe0633428 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,7 +182,8 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); + "-v", + String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); } From b3c38aba523c5663b7f3e9a969c2eb2d86076e99 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Thu, 15 Sep 2022 08:50:10 -0600 Subject: [PATCH 18/24] Update job-precommit-spotless.yml --- .github/workflows/job-precommit-spotless.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/job-precommit-spotless.yml b/.github/workflows/job-precommit-spotless.yml index d588b55cce56..de5554d3df2d 100644 --- a/.github/workflows/job-precommit-spotless.yml +++ b/.github/workflows/job-precommit-spotless.yml @@ -22,6 +22,7 @@ name: PreCommit Spotless on: + workflow_dispatch: push: branches: ['master', 'release-*'] tags: 'v*' From c2365d46f2097c846b4484ad9f5d6d9c432c201a Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Thu, 15 Sep 2022 09:16:06 -0600 Subject: [PATCH 19/24] Update DockerEnvironmentFactory.java --- .../fnexecution/environment/DockerEnvironmentFactory.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index 1d6fe0633428..edead73227d7 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,8 +182,7 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "-v", - String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); + "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); } From 9a82d13ef1687b6aa026fe5d271a367d4094488c Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Mon, 19 Sep 2022 12:53:38 -0600 Subject: [PATCH 20/24] python postcommit examples --- .../job-postcommit-python-examples.yml | 70 +++++++++++++++++++ .../python/test-suites/dataflow/common.gradle | 25 +++++++ 2 files changed, 95 insertions(+) create mode 100644 .github/workflows/.github/workflows/job-postcommit-python-examples.yml diff --git a/.github/workflows/.github/workflows/job-postcommit-python-examples.yml b/.github/workflows/.github/workflows/job-postcommit-python-examples.yml new file mode 100644 index 000000000000..9c501f751fda --- /dev/null +++ b/.github/workflows/.github/workflows/job-postcommit-python-examples.yml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the tests for python Examples with DataflowRunner. + +name: PostCommit Python Examples + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_examples: + needs: set-properties + name: Python Examples + runs-on: self-hosted + strategy: + fail-fast: true + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run :sdks:python:test-suites:dataflow:py${{env.PYTHON_VERSION}}:postCommitITByTestDirectory + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{env.PYTHON_VERSION}}:postCommitITByTestDirectory + arguments: "-PtestDir=apache_beam/examples" \ No newline at end of file diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index be278f6d3e3c..908773f5ce3b 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -101,6 +101,31 @@ task preCommitIT_V2{ dependsOn preCommitIT_streaming_V2 } +task postCommitITByTestDirectory { + def testDir = project.hasProperty('testDir') ? project.testDir : '' + dependsOn 'installGcpTest' + dependsOn ':sdks:python:sdist' + dependsOn ':runners:google-cloud-dataflow-java:worker:shadowJar' + + def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + + doLast { + def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"] + def argMap = [ + "test_opts": testOpts + [testDir], + "sdk_location": files(configurations.distTarBall.files).singleFile, + "worker_jar": dataflowWorkerJar, + "suite": "postCommitIT-df${pythonVersionSuffix}", + "collect": "it_postcommit" + ] + def cmdArgs = mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + task postCommitIT { dependsOn 'installGcpTest' dependsOn ':sdks:python:sdist' From c1bb678560b1229553e96b5805caddf2f7fc3421 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Mon, 19 Sep 2022 13:00:50 -0600 Subject: [PATCH 21/24] Update CI.md --- CI.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CI.md b/CI.md index ed75a4dd4433..079b89398fe5 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,14 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PostCommit Workflows + +#### Python PostCommit Examples DataFlow - [job-postcommit-python-examples.yml](.github/workflows/job-postcommit-python-examples.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|--------------------------------------------|--------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Python PostCommit Examples on DataFlow | Run Python PostCommit Examples on DataFlow | No | Yes | Yes | No | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From 3c03a6938e7a072314d4d396e9226c5105ee1d95 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Mon, 19 Sep 2022 15:24:03 -0600 Subject: [PATCH 22/24] Update job-postcommit-python-examples.yml --- .../{.github/workflows => }/job-postcommit-python-examples.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{.github/workflows => }/job-postcommit-python-examples.yml (100%) diff --git a/.github/workflows/.github/workflows/job-postcommit-python-examples.yml b/.github/workflows/job-postcommit-python-examples.yml similarity index 100% rename from .github/workflows/.github/workflows/job-postcommit-python-examples.yml rename to .github/workflows/job-postcommit-python-examples.yml From ec204b6462f98c10be2af73905fb758cbb6242e1 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Mon, 19 Sep 2022 15:39:48 -0600 Subject: [PATCH 23/24] reverting changes from other merge --- .../job-postcommit-java-dataflow-v1.yml | 74 ------------------- .../job-postcommit-java-dataflow-v2.yml | 72 ------------------ .github/workflows/job-precommit-rat.yml | 47 ------------ .github/workflows/job-precommit-spotless.yml | 54 -------------- .../workflows/job-precommit-whitespace.yml | 48 ------------ .test-infra/jenkins/README.md | 7 +- .../job_PostCommit_Java_Dataflow.groovy | 48 ++++++++++++ .../job_PostCommit_Java_DataflowV2.groovy | 48 ++++++++++++ .test-infra/jenkins/job_PreCommit_RAT.groovy | 26 +++++++ .../jenkins/job_PreCommit_Spotless.groovy | 45 +++++++++++ .../jenkins/job_PreCommit_Whitespace.groovy | 30 ++++++++ CI.md | 34 --------- .../environment/DockerEnvironmentFactory.java | 3 +- sdks/python/test-suites/tox/py38/build.gradle | 2 - 14 files changed, 205 insertions(+), 333 deletions(-) delete mode 100644 .github/workflows/job-postcommit-java-dataflow-v1.yml delete mode 100644 .github/workflows/job-postcommit-java-dataflow-v2.yml delete mode 100644 .github/workflows/job-precommit-rat.yml delete mode 100644 .github/workflows/job-precommit-spotless.yml delete mode 100644 .github/workflows/job-precommit-whitespace.yml create mode 100644 .test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy create mode 100644 .test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_RAT.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Spotless.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Whitespace.groovy diff --git a/.github/workflows/job-postcommit-java-dataflow-v1.yml b/.github/workflows/job-postcommit-java-dataflow-v1.yml deleted file mode 100644 index ddff406ef92a..000000000000 --- a/.github/workflows/job-postcommit-java-dataflow-v1.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# To learn more about GitHub Actions in Apache Beam check the CI.md - -# Runs Java PostCommit on Dataflow runner v1 via cron and on push to master - -name: PostCommit Java Dataflow v1 - -on: - workflow_dispatch: - schedule: - - cron: '0 */6 * * *' - push: - branches: ['master'] - -jobs: - set-properties: - runs-on: self-hosted - outputs: - properties: ${{ steps.test-properties.outputs.properties }} - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - id: test-properties - uses: ./.github/actions/setup-default-test-properties - - postcommit-java-dataflow-v1: - needs: set-properties - name: Run PostCommit Java Dataflow v1 - runs-on: self-hosted - strategy: - fail-fast: true - matrix: #JDK11 and 17 compilation will fail due to changes in ByteBuffer class since JDK9 - #TODO: Retest once issue https://github.com/apache/beam/issues/23068 gets resolved. - java: ${{fromJson(needs.set-properties.outputs.properties).JavaTestProperties.SUPPORTED_VERSIONS}} - exclude: - - java: "11" - - java: "17" - timeout-minutes: 1200 - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Setup environment - uses: ./.github/actions/setup-self-hosted-action - - name: Setup Java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - name: Run :runners:google-cloud-dataflow-java:postCommit in Java-${{ matrix.java }} - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :runners:google-cloud-dataflow-java:postCommit diff --git a/.github/workflows/job-postcommit-java-dataflow-v2.yml b/.github/workflows/job-postcommit-java-dataflow-v2.yml deleted file mode 100644 index ee6980eb402e..000000000000 --- a/.github/workflows/job-postcommit-java-dataflow-v2.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# To learn more about GitHub Actions in Apache Beam check the CI.md - -# Runs Java PostCommit on Dataflow runner v2 via cron and on push to master - -name: PostCommit Java Dataflow v2 - -on: - schedule: - - cron: '0 */6 * * *' - push: - branches: ['master'] - -jobs: - set-properties: - runs-on: self-hosted - outputs: - properties: ${{ steps.test-properties.outputs.properties }} - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - id: test-properties - uses: ./.github/actions/setup-default-test-properties - - postcommit-java-dataflow-v2: - needs: set-properties - name: Run PostCommit Java Dataflow v2 - runs-on: self-hosted - strategy: - fail-fast: true - matrix: #JDK11 and 17 compilation will fail due to changes in ByteBuffer class since JDK9 - #TODO: Retest once issue https://github.com/apache/beam/issues/23068 gets resolved. - java: ${{fromJson(needs.set-properties.outputs.properties).JavaTestProperties.SUPPORTED_VERSIONS}} - exclude: - - java: "11" - - java: "17" - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Setup environment - uses: ./.github/actions/setup-self-hosted-action - - name: Setup Java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - name: Run :runners:google-cloud-dataflow-java:postCommitRunnerV2 in Java-${{ matrix.java }} - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :runners:google-cloud-dataflow-java:postCommitRunnerV2 diff --git a/.github/workflows/job-precommit-rat.yml b/.github/workflows/job-precommit-rat.yml deleted file mode 100644 index 0d8c5fe3dcbf..000000000000 --- a/.github/workflows/job-precommit-rat.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# To learn more about GitHub Actions in Apache Beam check the CI.md - -# Runs RAT scheduled on cron, on push to master and on pull request - -name: PreCommit RAT - -on: - push: - branches: ['master', 'release-*'] - tags: 'v*' - pull_request: - branches: ['master', 'release-*'] - tags: 'v*' - -jobs: - rat: - name: Run PreCommit RAT - runs-on: self-hosted - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Setup environment - uses: ./.github/actions/setup-self-hosted-action - - name: Run PreCommit RAT - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: rat diff --git a/.github/workflows/job-precommit-spotless.yml b/.github/workflows/job-precommit-spotless.yml deleted file mode 100644 index de5554d3df2d..000000000000 --- a/.github/workflows/job-precommit-spotless.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# To learn more about GitHub Actions in Apache Beam check the CI.md - -# Runs Spotless scheduled on cron, on push to master and on pull request - -name: PreCommit Spotless - -on: - workflow_dispatch: - push: - branches: ['master', 'release-*'] - tags: 'v*' - pull_request: - branches: ['master', 'release-*'] - tags: 'v*' - paths: ['buildSrc/**', - 'sdks/java/**', - 'runners/**', - 'examples/java/**', - 'examples/kotlin/**'] - -jobs: - spotless: - name: Run PreCommit Spotless - runs-on: self-hosted - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Setup environment - uses: ./.github/actions/setup-self-hosted-action - - name: Run spotlessCheck checkStyleMain checkStyleTest - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: spotlessCheck checkStyleMain checkStyleTest - diff --git a/.github/workflows/job-precommit-whitespace.yml b/.github/workflows/job-precommit-whitespace.yml deleted file mode 100644 index 10d596ec1e89..000000000000 --- a/.github/workflows/job-precommit-whitespace.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# To learn more about GitHub Actions in Apache Beam check the CI.md - -# Runs Whitespace scheduled on cron, on push to master and on pull request - -name: PreCommit Whitespace - -on: - push: - branches: ['master', 'release-*'] - tags: 'v*' - pull_request: - branches: ['master', 'release-*'] - tags: 'v*' - paths: ['.*\\.md$', '.*build\\.gradle$'] - -jobs: - whitespace: - name: Run PreCommit Whitespace - runs-on: self-hosted - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Setup environment - uses: ./.github/actions/setup-self-hosted-action - - name: Run PreCommit Whitespace - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: whitespacePreCommit diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index c6b513bb8866..c70bd013562e 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -48,10 +48,13 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PreCommit_PythonDocs| [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Phrase/) | `Run PythonDocs PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/) | | beam_PreCommit_Python_PVR_Flink | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Phrase/) | `Run Python_PVR_Flink PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron) | | beam_PreCommit_PythonFormatter | [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Phrase/) | `Run PythonFormatter PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/) | +| beam_PreCommit_RAT (checks for valid Apache license headers) | [commit](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Phrase/) | `Run RAT PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron) | +| beam_PreCommit_Spotless | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Phrase/) | `Run Spotless PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron) | | beam_PreCommit_SQL | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Phrase/) | `Run SQL PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/) | | beam_PreCommit_SQL_Java11 | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Phrase/) | `Run SQL_Java11 PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/) | | beam_PreCommit_Website | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Website_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Website_Phrase/) | `Run Website PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron) | | beam_PreCommit_Website_Stage_GCS | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Phrase/) | `Run Website_Stage_GCS PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron) | +| beam_PreCommit_Whitespace | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Phrase/) | `Run Whitespace PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron) | | beam_PreCommit_Typescript | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Phrase/) | `Run Typescript PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron) | ### PostCommit Jobs @@ -71,6 +74,8 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Go_VR_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark_PR/) | `Run Go Spark ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/) | | beam_PostCommit_Java_Hadoop_Versions | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Hadoop_Versions/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Hadoop_Versions_PR/) | `Run PostCommit_Java_Hadoop_Versions` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Hadoop_Versions/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Hadoop_Versions/) | | beam_PostCommit_Java | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_PR/) | `Run Java PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Java/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Java) | +| beam_PostCommit_Java_DataflowV1 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1_PR/) | `Run PostCommit_Java_Dataflow` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1/) | +| beam_PostCommit_Java_DataflowV2 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2_PR/) | `Run PostCommit_Java_DataflowV2` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2/) | | beam_PostCommit_Java_Nexmark_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow_PR/) | `Dataflow Runner Nexmark Tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow) | | beam_PostCommit_Java_Nexmark_Dataflow_V2 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_DataflowV2_PR/) | `Run Dataflow Runner V2 Nexmark Tests` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2/) | | beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/badge/icon) | `Dataflow Runner V2 Java 11 Nexmark Tests` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/) | @@ -257,4 +262,4 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) retest this please ``` -* Last update (mm/dd/yyyy): 04/04/2022 +* Last update (mm/dd/yyyy): 04/04/2022 \ No newline at end of file diff --git a/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy new file mode 100644 index 000000000000..8988ae29f85b --- /dev/null +++ b/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import CommonJobProperties as commonJobProperties +import PostcommitJobBuilder + + +// This job runs the Java postcommit tests, including the suite of integration +// tests. +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_DataflowV1', 'Run PostCommit_Java_Dataflow', + 'Dataflow Java Post Commit Tests', this) { + + description('Dataflow Java Post Commit Tests') + + // Set common parameters. + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240) + + // Publish all test results to Jenkins + publishers { + archiveJunit('**/build/test-results/**/*.xml') + } + + // Gradle goals for this job. + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(":runners:google-cloud-dataflow-java:postCommit") + commonJobProperties.setGradleSwitches(delegate) + // Specify maven home on Jenkins, needed by Maven archetype integration tests. + switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') + } + } + } \ No newline at end of file diff --git a/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy b/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy new file mode 100644 index 000000000000..99d927f3d7c1 --- /dev/null +++ b/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import CommonJobProperties as commonJobProperties +import PostcommitJobBuilder + + +// This job runs the Java postcommit tests, including the suite of integration +// tests. +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_DataflowV2', 'Run PostCommit_Java_DataflowV2', + 'Dataflow V2 Java Post Commit Tests', this) { + + description('Dataflow V2 Java Post Commit Tests') + + // Set common parameters. + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240) + + // Publish all test results to Jenkins + publishers { + archiveJunit('**/build/test-results/**/*.xml') + } + + // Gradle goals for this job. + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(":runners:google-cloud-dataflow-java:postCommitRunnerV2") + commonJobProperties.setGradleSwitches(delegate) + // Specify maven home on Jenkins, needed by Maven archetype integration tests. + switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') + } + } + } \ No newline at end of file diff --git a/.test-infra/jenkins/job_PreCommit_RAT.groovy b/.test-infra/jenkins/job_PreCommit_RAT.groovy new file mode 100644 index 000000000000..c1c5ac0149ae --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_RAT.groovy @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'RAT', + gradleTask: ':rat' + ) +builder.build() \ No newline at end of file diff --git a/.test-infra/jenkins/job_PreCommit_Spotless.groovy b/.test-infra/jenkins/job_PreCommit_Spotless.groovy new file mode 100644 index 000000000000..211d168a3ee5 --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Spotless.groovy @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Spotless', + gradleTask: 'spotlessCheck checkStyleMain checkStyleTest', + triggerPathPatterns: [ + '^buildSrc/.*$', + '^sdks/java/.*$', + '^runners/.*$', + '^examples/java/.*$', + '^examples/kotlin/.*$', + '^.test-infra/jenkins/.*$', + ] + ) +builder.build { + publishers { + recordIssues { + tools { + checkStyle { + pattern('**/build/reports/checkstyle/*.xml') + } + } + enabledForFailure(true) + } + } +} \ No newline at end of file diff --git a/.test-infra/jenkins/job_PreCommit_Whitespace.groovy b/.test-infra/jenkins/job_PreCommit_Whitespace.groovy new file mode 100644 index 000000000000..66667b4336bd --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Whitespace.groovy @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Whitespace', + gradleTask: ':whitespacePreCommit', + triggerPathPatterns: [ + '.*\\.md$', + '.*build\\.gradle$', + ] + ) +builder.build() \ No newline at end of file diff --git a/CI.md b/CI.md index 14494c8ed51a..079b89398fe5 100644 --- a/CI.md +++ b/CI.md @@ -133,40 +133,6 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog |--------------------------------------------|--------------------------------------------|------------------|-----------------------|---------------|--------------------------| | Run Python PostCommit Examples on DataFlow | Run Python PostCommit Examples on DataFlow | No | Yes | Yes | No | -### PreCommit Workflows - -#### RAT PreCommit - [job-precommit-rat.yml](.github/workflows/job-precommit-rat.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | -|----------------------|---------------------|------------------|-----------------------|---------------|--------------------------| -| Run RAT PreCommit | Runs RAT PreCommit | Yes | Yes | Yes | No | - -#### Spotless PreCommit - [job-precommit-spotless.yml](.github/workflows/job-precommit-spotless.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | -|---------------------------|-------------------------|------------------|-----------------------|---------------|--------------------------| -| Run Spotless PreCommit | Runs Spotless PreCommit | Yes | Yes | Yes | No | - -#### Whitespace PreCommit - [job-precommit-spotless.yml](.github/workflows/job-precommit-spotless.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | -|--------------------------|---------------------------|------------------|-----------------------|---------------|--------------------------| -| Run Whitespace PreCommit | Runs Whitespace PreCommit | Yes | Yes | Yes | No | - -### PostCommit Workflows - -#### Java PostCommit DataFlow v1- [job-postcommit-java-dataflow-v1.yml](.github/workflows/job-postcommit-java-dataflow-v1.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | -|---------------------------------|----------------------------------|------------------|-----------------------|---------------|--------------------------| -| Run Java PostCommit DataFlow v1 | Runs Java PostCommit DataFlow v1 | No | Yes | Yes | Yes | - -#### Java PostCommit DataFlow v2- [job-postcommit-java-dataflow-v2.yml](.github/workflows/job-postcommit-java-dataflow-v2.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | -|-----------------------------------|------------------------------------|------------------|-----------------------|---------------|--------------------------| -| Run Java PostCommit DataFlow v2 | Runs Java PostCommit DataFlow v2 | No | Yes | Yes | Yes | - ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index edead73227d7..ee816a944e5a 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,7 +182,8 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); + "--mount", + String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); } diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 4287c62559c0..6c2cdf7f72b9 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -112,8 +112,6 @@ task archiveFilesToLint(type: Zip) { include "**/*.md" include "**/build.gradle" exclude "**/node_modules/*" - exclude ".github/*" - exclude "website/*" } } From d8a35c9d3a93b38259a9bd910335a37a45210190 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Mon, 19 Sep 2022 15:43:33 -0600 Subject: [PATCH 24/24] tweaking --- .test-infra/jenkins/README.md | 2 +- .test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy | 2 +- .test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy | 2 +- .test-infra/jenkins/job_PreCommit_RAT.groovy | 2 +- .test-infra/jenkins/job_PreCommit_Spotless.groovy | 2 +- .test-infra/jenkins/job_PreCommit_Whitespace.groovy | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index c70bd013562e..431e2c495b15 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -48,7 +48,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PreCommit_PythonDocs| [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Phrase/) | `Run PythonDocs PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/) | | beam_PreCommit_Python_PVR_Flink | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Phrase/) | `Run Python_PVR_Flink PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron) | | beam_PreCommit_PythonFormatter | [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Phrase/) | `Run PythonFormatter PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonFormatter_Cron/) | -| beam_PreCommit_RAT (checks for valid Apache license headers) | [commit](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Phrase/) | `Run RAT PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron) | +| beam_PreCommit_RAT | [commit](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Phrase/) | `Run RAT PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_RAT_Cron) | | beam_PreCommit_Spotless | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Phrase/) | `Run Spotless PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron) | | beam_PreCommit_SQL | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Phrase/) | `Run SQL PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/) | | beam_PreCommit_SQL_Java11 | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Phrase/) | `Run SQL_Java11 PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/) | diff --git a/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy index 8988ae29f85b..2ab27da0f1b0 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy @@ -45,4 +45,4 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_DataflowV1', 'Run PostC switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') } } - } \ No newline at end of file + } diff --git a/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy b/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy index 99d927f3d7c1..03d82d48974e 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_DataflowV2.groovy @@ -45,4 +45,4 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_DataflowV2', 'Run PostC switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') } } - } \ No newline at end of file + } diff --git a/.test-infra/jenkins/job_PreCommit_RAT.groovy b/.test-infra/jenkins/job_PreCommit_RAT.groovy index c1c5ac0149ae..613caa9af0de 100644 --- a/.test-infra/jenkins/job_PreCommit_RAT.groovy +++ b/.test-infra/jenkins/job_PreCommit_RAT.groovy @@ -23,4 +23,4 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'RAT', gradleTask: ':rat' ) -builder.build() \ No newline at end of file +builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Spotless.groovy b/.test-infra/jenkins/job_PreCommit_Spotless.groovy index 211d168a3ee5..a9da1ad5491a 100644 --- a/.test-infra/jenkins/job_PreCommit_Spotless.groovy +++ b/.test-infra/jenkins/job_PreCommit_Spotless.groovy @@ -42,4 +42,4 @@ builder.build { enabledForFailure(true) } } -} \ No newline at end of file +} diff --git a/.test-infra/jenkins/job_PreCommit_Whitespace.groovy b/.test-infra/jenkins/job_PreCommit_Whitespace.groovy index 66667b4336bd..1b8341395a68 100644 --- a/.test-infra/jenkins/job_PreCommit_Whitespace.groovy +++ b/.test-infra/jenkins/job_PreCommit_Whitespace.groovy @@ -27,4 +27,4 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( '.*build\\.gradle$', ] ) -builder.build() \ No newline at end of file +builder.build()