diff --git a/.github/workflows/job-postcommit-python-examples.yml b/.github/workflows/job-postcommit-python-examples.yml new file mode 100644 index 000000000000..9c501f751fda --- /dev/null +++ b/.github/workflows/job-postcommit-python-examples.yml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the tests for python Examples with DataflowRunner. + +name: PostCommit Python Examples + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_examples: + needs: set-properties + name: Python Examples + runs-on: self-hosted + strategy: + fail-fast: true + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run :sdks:python:test-suites:dataflow:py${{env.PYTHON_VERSION}}:postCommitITByTestDirectory + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:py${{env.PYTHON_VERSION}}:postCommitITByTestDirectory + arguments: "-PtestDir=apache_beam/examples" \ No newline at end of file diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 2c252a4d55aa..431e2c495b15 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -262,4 +262,4 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) retest this please ``` -* Last update (mm/dd/yyyy): 04/04/2022 +* Last update (mm/dd/yyyy): 04/04/2022 \ No newline at end of file diff --git a/CI.md b/CI.md index ed75a4dd4433..079b89398fe5 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,14 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PostCommit Workflows + +#### Python PostCommit Examples DataFlow - [job-postcommit-python-examples.yml](.github/workflows/job-postcommit-python-examples.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|--------------------------------------------|--------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Python PostCommit Examples on DataFlow | Run Python PostCommit Examples on DataFlow | No | Yes | Yes | No | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index be278f6d3e3c..908773f5ce3b 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -101,6 +101,31 @@ task preCommitIT_V2{ dependsOn preCommitIT_streaming_V2 } +task postCommitITByTestDirectory { + def testDir = project.hasProperty('testDir') ? project.testDir : '' + dependsOn 'installGcpTest' + dependsOn ':sdks:python:sdist' + dependsOn ':runners:google-cloud-dataflow-java:worker:shadowJar' + + def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + + doLast { + def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"] + def argMap = [ + "test_opts": testOpts + [testDir], + "sdk_location": files(configurations.distTarBall.files).singleFile, + "worker_jar": dataflowWorkerJar, + "suite": "postCommitIT-df${pythonVersionSuffix}", + "collect": "it_postcommit" + ] + def cmdArgs = mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + task postCommitIT { dependsOn 'installGcpTest' dependsOn ':sdks:python:sdist'