GA Migrations Python PostCommit Sharding Examples

.github/workflows/job-postcommit-python-examples.yml

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This workflow will run the tests for python Examples with DataflowRunner.
+
+name: PostCommit Python Examples
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: ['v*']
+
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: self-hosted
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  python_examples:
+    needs: set-properties
+    name: Python Examples
+    runs-on: self-hosted
+    strategy:
+      fail-fast: true
+      matrix:
+        version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - name: Set python version
+        run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+        with:
+          requires-go-18: false
+      - name: Run :sdks:python:test-suites:dataflow:py${{env.PYTHON_VERSION}}:postCommitITByTestDirectory
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:dataflow:py${{env.PYTHON_VERSION}}:postCommitITByTestDirectory
+          arguments: "-PtestDir=apache_beam/examples"

.test-infra/jenkins/README.md

@@ -262,4 +262,4 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/)
 retest this please
 ```
 
-* Last update (mm/dd/yyyy): 04/04/2022
+* Last update (mm/dd/yyyy): 04/04/2022

CI.md

@@ -125,6 +125,14 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog
 | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner.                                               | Yes              | Yes                   | Yes           | -                        |
 | Java Wordcount Dataflow      | Runs Java WordCount example with DataFlow Runner.                                             | -                | Yes                   | Yes           | Yes                      |
 
+### PostCommit Workflows
+
+#### Python PostCommit Examples DataFlow - [job-postcommit-python-examples.yml](.github/workflows/job-postcommit-python-examples.yml)
+
+| Job                                        | Description                                | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials |
+|--------------------------------------------|--------------------------------------------|------------------|-----------------------|---------------|--------------------------|
+| Run Python PostCommit Examples on DataFlow | Run Python PostCommit Examples on DataFlow | No               | Yes                   | Yes           | No                       |
+
 ### GitHub Action Tips
 
 * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request.

sdks/python/test-suites/dataflow/common.gradle

@@ -101,6 +101,31 @@ task preCommitIT_V2{
   dependsOn preCommitIT_streaming_V2
 }
 
+task postCommitITByTestDirectory {
+  def testDir =  project.hasProperty('testDir') ? project.testDir : ''
+  dependsOn 'installGcpTest'
+  dependsOn ':sdks:python:sdist'
+  dependsOn ':runners:google-cloud-dataflow-java:worker:shadowJar'
+
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+  doLast {
+    def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"]
+    def argMap = [
+            "test_opts": testOpts + [testDir],
+            "sdk_location": files(configurations.distTarBall.files).singleFile,
+            "worker_jar": dataflowWorkerJar,
+            "suite": "postCommitIT-df${pythonVersionSuffix}",
+            "collect": "it_postcommit"
+    ]
+    def cmdArgs = mapToArgString(argMap)
+    exec {
+      executable 'sh'
+      args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
+    }
+  }
+}
+
 task postCommitIT {
   dependsOn 'installGcpTest'
   dependsOn ':sdks:python:sdist'