From 9b648d5b2265286387d9a0c629799feb3cd35a2f Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 8 Sep 2022 14:31:50 -0500 Subject: [PATCH 1/5] Base actions to use for precommit and postcommit workflows --- .../action.yml | 45 +++++++++++++++++ .../setup-default-test-properties/action.yml | 31 ++++++++++++ .../test-properties.json | 17 +++++++ .../setup-self-hosted-action/action.yml | 50 +++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 .github/actions/gradle-command-self-hosted-action/action.yml create mode 100644 .github/actions/setup-default-test-properties/action.yml create mode 100644 .github/actions/setup-default-test-properties/test-properties.json create mode 100644 .github/actions/setup-self-hosted-action/action.yml diff --git a/.github/actions/gradle-command-self-hosted-action/action.yml b/.github/actions/gradle-command-self-hosted-action/action.yml new file mode 100644 index 000000000000..7dd835e1032e --- /dev/null +++ b/.github/actions/gradle-command-self-hosted-action/action.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Gradle command' +description: 'Configuration actions to run a gradle command in self-hosted runner' +inputs: + gradle-command: + required: true + arguments: + required: false + description: 'Gradle options' + default: '' + default-arguments: + required: false + description: 'Default gradle switches' # Copied from CommonJobProperties.groovy' + default: | + --continue -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g \ + -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses + max-workers: + required: false + description: 'Max number of workers' + default: 8 + +runs: + using: "composite" + steps: + - name: Run Gradle Command + shell: bash + run: | + ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \ + ${{ inputs.default-arguments }} \ No newline at end of file diff --git a/.github/actions/setup-default-test-properties/action.yml b/.github/actions/setup-default-test-properties/action.yml new file mode 100644 index 000000000000..78144098e285 --- /dev/null +++ b/.github/actions/setup-default-test-properties/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup TestProperties' +description: 'Setup default test properties from a Json config file' +outputs: + properties: + description: "Test properties" + value: ${{ steps.test-properties-json.outputs.test-properties }} +runs: + using: "composite" + steps: + - id: test-properties-json + shell: bash + run: | + JSON=$(cat ./.github/actions/setup-default-test-properties/test-properties.json) + echo "::set-output name=test-properties::$(echo $JSON)" \ No newline at end of file diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json new file mode 100644 index 000000000000..9cae4b5ec8c6 --- /dev/null +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -0,0 +1,17 @@ +{ + "PythonTestProperties": { + "ALL_SUPPORTED_VERSIONS": ["3.7", "3.8", "3.9"], + "LOWEST_SUPPORTED": ["3.7"], + "HIGHEST_SUPPORTED": ["3.9"], + "ESSENTIAL_VERSIONS": ["3.7", "3.9"], + "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.7", "3.9"], + "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.9"], + "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.7", "3.8", "3.9"], + "LOAD_TEST_PYTHON_VERSION": "3.7", + "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.7", + "DEFAULT_INTERPRETER": "python3.8" + }, + "JavaTestProperties": { + "SUPPORTED_CONTAINER_TASKS": ["java8", "java11", "java17"] + } +} \ No newline at end of file diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml new file mode 100644 index 000000000000..6b190629859b --- /dev/null +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup environment for self-hosted runners' +description: 'Setup action to run jobs in a self-hosted runner' + +runs: + using: "composite" + steps: + - name: Install python 3.7 + uses: actions/setup-python@v4 + with: + python-version: "3.7" + - name: Install python 3.8 + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install python 3.9 + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Set Java Version + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 8 + - name: Set Go Version + uses: actions/setup-go@v3 + with: + go-version: '>=1.16.0' + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue of Beam's + # gradle-command-action plugin and github's provided maven + # settings.xml file + shell: bash + run: rm ~/.m2/settings.xml \ No newline at end of file From 774f5f895f006f530edbc8ed643ba1a42ac01448 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Thu, 8 Sep 2022 14:44:51 -0600 Subject: [PATCH 2/5] Adding Java, Flink, Spark and Golang supported versions to json file --- .../setup-default-test-properties/test-properties.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index 9cae4b5ec8c6..324329035b71 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -12,6 +12,11 @@ "DEFAULT_INTERPRETER": "python3.8" }, "JavaTestProperties": { - "SUPPORTED_CONTAINER_TASKS": ["java8", "java11", "java17"] + "SUPPORTED_VERSIONS": ["8", "11", "17"], + "FLINK_VERSIONS": ["1.13", "1.14", "1.15"], + "SPARK_VERSIONS": ["2", "3"] + }, + "GoTestProperties": { + "SUPPORTED_VERSIONS": ["1.18"] } } \ No newline at end of file From d8689ee9092cc2209682087410c37d97961c989d Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Fri, 9 Sep 2022 01:16:57 -0600 Subject: [PATCH 3/5] switching to temurin due to Apache License compatibility --- .github/actions/setup-self-hosted-action/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml index 6b190629859b..825251d44085 100644 --- a/.github/actions/setup-self-hosted-action/action.yml +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -36,7 +36,7 @@ runs: - name: Set Java Version uses: actions/setup-java@v3 with: - distribution: 'zulu' + distribution: 'temurin' java-version: 8 - name: Set Go Version uses: actions/setup-go@v3 From d82c23441b95c66d6d697e4d2d580f04cde32713 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Tue, 13 Sep 2022 12:00:29 -0600 Subject: [PATCH 4/5] setting go version to 1.18 --- .github/actions/setup-self-hosted-action/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml index 825251d44085..565c0500989e 100644 --- a/.github/actions/setup-self-hosted-action/action.yml +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -41,10 +41,10 @@ runs: - name: Set Go Version uses: actions/setup-go@v3 with: - go-version: '>=1.16.0' + go-version: '1.18' - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # gradle-command-action plugin and github's provided maven # settings.xml file shell: bash - run: rm ~/.m2/settings.xml \ No newline at end of file + run: rm ~/.m2/settings.xml From 53384e06e4daef2061b3e9acfa249e48503971d4 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 14 Sep 2022 16:48:30 -0500 Subject: [PATCH 5/5] Add environment as optionals in setup-base-action --- .../test-properties.json | 3 +- .../setup-self-hosted-action/action.yml | 29 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index 324329035b71..6d30acafbb04 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -9,7 +9,8 @@ "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.7", "3.8", "3.9"], "LOAD_TEST_PYTHON_VERSION": "3.7", "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.7", - "DEFAULT_INTERPRETER": "python3.8" + "DEFAULT_INTERPRETER": "python3.8", + "TOX_ENV": ["Cloud", "Cython"] }, "JavaTestProperties": { "SUPPORTED_VERSIONS": ["8", "11", "17"], diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml index 565c0500989e..b268357c08e6 100644 --- a/.github/actions/setup-self-hosted-action/action.yml +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -17,32 +17,59 @@ name: 'Setup environment for self-hosted runners' description: 'Setup action to run jobs in a self-hosted runner' +inputs: + requires-py-37: + required: false + description: 'Set as false if does not require py37 setup' + default: 'true' + requires-py-38: + required: false + description: 'Set as false if does not require py38 setup' + default: 'true' + requires-py-39: + required: false + description: 'Set as false if does not require py39 setup' + default: 'true' + requires-java-8: + required: false + description: 'Set as false if does not require java-8 setup' + default: 'true' + requires-go-18: + required: false + description: 'Set as false if does not require go-18 setup' + default: 'true' runs: using: "composite" steps: - name: Install python 3.7 + if: ${{ inputs.requires-py-37 == 'true' }} uses: actions/setup-python@v4 with: python-version: "3.7" - name: Install python 3.8 + if: ${{ inputs.requires-py-38 == 'true' }} uses: actions/setup-python@v4 with: python-version: "3.8" - name: Install python 3.9 + if: ${{ inputs.requires-py-39 == 'true' }} uses: actions/setup-python@v4 with: python-version: "3.9" - name: Set Java Version + if: ${{ inputs.requires-java-8 == 'true' }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: 8 - name: Set Go Version + if: ${{ inputs.requires-go-18 == 'true' }} uses: actions/setup-go@v3 with: - go-version: '1.18' + go-version: '1.18.0' - name: Remove default github maven configuration + if: ${{ inputs.requires-java-8 == 'true' }} # This step is a workaround to avoid a decryption issue of Beam's # gradle-command-action plugin and github's provided maven # settings.xml file