From fc659ced1851659c995ec2cecb4585190526df55 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Fri, 10 Jun 2022 12:36:52 -0600 Subject: [PATCH 1/3] sharding GCP IO tests from the javaPostCommit task --- .../job_PreCommit_Java_GCP_IO_Direct.groovy | 56 +++++++++++++++++++ build.gradle.kts | 5 +- 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 .test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy diff --git a/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy new file mode 100644 index 000000000000..c4652068e82d --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Java', + gradleTask: ':javaGCPIODirectRunnerPreCommit', + gradleSwitches: [ + '-PdisableSpotlessCheck=true' + ], // spotless checked in separate pre-commit + timeoutMins: 180, + triggerPathPatterns: [ + '^sdks/java/io/google-cloud-platform/.*$', + '^runners/direct-java/.*$', + ] + ) +builder.build { + publishers { + archiveJunit('**/build/test-results/**/*.xml') + recordIssues { + tools { + errorProne() + java() + checkStyle { + pattern('**/build/reports/checkstyle/*.xml') + } + spotBugs { + pattern('**/build/reports/spotbugs/*.xml') + } + } + enabledForFailure(true) + } + jacocoCodeCoverage { + execPattern('**/build/jacoco/*.exec') + exclusionPattern('**/org/apache/beam/gradle/**,**/org/apache/beam/model/**,' + + '**/org/apache/beam/runners/dataflow/worker/windmill/**,**/AutoValue_*') + } + } +} diff --git a/build.gradle.kts b/build.gradle.kts index c6c0059f62ae..505063428e11 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -182,6 +182,10 @@ tasks.register("javaPreCommit") { dependsOn(":sdks:java:container:java8:docker") } +tasks.register("javaGCPIODirectRunnerPreCommit") { + dependsOn(":sdks:java:io:google-cloud-platform:postCommit") +} + tasks.register("sqlPreCommit") { dependsOn(":sdks:java:extensions:sql:runBasicExample") dependsOn(":sdks:java:extensions:sql:runPojoExample") @@ -198,7 +202,6 @@ tasks.register("javaPostCommit") { dependsOn(":sdks:java:extensions:zetasketch:postCommit") dependsOn(":sdks:java:io:debezium:integrationTest") dependsOn(":sdks:java:io:jdbc:integrationTest") - dependsOn(":sdks:java:io:google-cloud-platform:postCommit") dependsOn(":sdks:java:io:kinesis:integrationTest") dependsOn(":sdks:java:io:amazon-web-services:integrationTest") dependsOn(":sdks:java:io:amazon-web-services2:integrationTest") From 3ac14f046850e2236610c75454f92846dc30ffed Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Fri, 10 Jun 2022 12:57:56 -0600 Subject: [PATCH 2/3] tweaking configurations --- .../jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy | 9 ++------- build.gradle.kts | 4 ---- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy index c4652068e82d..5ca1c591f06f 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy @@ -21,11 +21,11 @@ import PrecommitJobBuilder PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, nameBase: 'Java', - gradleTask: ':javaGCPIODirectRunnerPreCommit', + gradleTask: ':sdks:java:io:google-cloud-platform:postCommit', gradleSwitches: [ '-PdisableSpotlessCheck=true' ], // spotless checked in separate pre-commit - timeoutMins: 180, + timeoutMins: 120, triggerPathPatterns: [ '^sdks/java/io/google-cloud-platform/.*$', '^runners/direct-java/.*$', @@ -47,10 +47,5 @@ builder.build { } enabledForFailure(true) } - jacocoCodeCoverage { - execPattern('**/build/jacoco/*.exec') - exclusionPattern('**/org/apache/beam/gradle/**,**/org/apache/beam/model/**,' + - '**/org/apache/beam/runners/dataflow/worker/windmill/**,**/AutoValue_*') - } } } diff --git a/build.gradle.kts b/build.gradle.kts index 505063428e11..dad92d476f2d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -182,10 +182,6 @@ tasks.register("javaPreCommit") { dependsOn(":sdks:java:container:java8:docker") } -tasks.register("javaGCPIODirectRunnerPreCommit") { - dependsOn(":sdks:java:io:google-cloud-platform:postCommit") -} - tasks.register("sqlPreCommit") { dependsOn(":sdks:java:extensions:sql:runBasicExample") dependsOn(":sdks:java:extensions:sql:runPojoExample") From d210633689be6c2f195813e5b06f2cf7d70530d5 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Tue, 14 Jun 2022 11:32:58 -0600 Subject: [PATCH 3/3] Removing direct runner trigger and recordIssues --- .../job_PreCommit_Java_GCP_IO_Direct.groovy | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy index 5ca1c591f06f..90a8aa4a76f5 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy @@ -28,24 +28,10 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( timeoutMins: 120, triggerPathPatterns: [ '^sdks/java/io/google-cloud-platform/.*$', - '^runners/direct-java/.*$', ] ) builder.build { publishers { archiveJunit('**/build/test-results/**/*.xml') - recordIssues { - tools { - errorProne() - java() - checkStyle { - pattern('**/build/reports/checkstyle/*.xml') - } - spotBugs { - pattern('**/build/reports/spotbugs/*.xml') - } - } - enabledForFailure(true) - } } }