From 33d52a9ebacfbf8080f7f9b460780391b0c3ac73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Sat, 5 Feb 2022 01:20:00 +0100 Subject: [PATCH 1/8] [tests] remove backward compatibility test against yahoo-version (apache Pulsar 1.21) --- settings.gradle | 1 - .../TestCompatUpgradeYahooCustom.groovy | 220 ------------------ .../src/test/resources/arquillian.xml | 28 --- .../scripts/install-pulsar-tarball.sh | 41 ---- 4 files changed, 290 deletions(-) delete mode 100644 tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy delete mode 100644 tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml delete mode 100644 tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh diff --git a/settings.gradle b/settings.gradle index 3d192cfd755..61f3296260b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -90,7 +90,6 @@ include(':bookkeeper-benchmark', 'tests:backward-compat:recovery-no-password', 'tests:backward-compat:upgrade', 'tests:backward-compat:upgrade-direct', - 'tests:backward-compat:yahoo-custom-version', 'tests:integration-tests-base', 'tests:integration-tests-topologies', 'tests:integration-tests-utils', diff --git a/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy b/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy deleted file mode 100644 index f1d024a61dc..00000000000 --- a/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy +++ /dev/null @@ -1,220 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.bookkeeper.tests.backwardcompat - -import static java.nio.charset.StandardCharsets.UTF_8 - -import com.github.dockerjava.api.DockerClient - -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutionException -import java.util.concurrent.TimeUnit -import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils -import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader - -import org.jboss.arquillian.junit.Arquillian -import org.jboss.arquillian.test.api.ArquillianResource - -import org.junit.Assert -import org.junit.Test -import org.junit.runner.RunWith - -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -@RunWith(Arquillian.class) -class TestCompatUpgradeYahooCustom { - private static final Logger LOG = LoggerFactory.getLogger(TestCompatUpgradeYahooCustom.class) - private static byte[] PASSWD = "foobar".getBytes() - - def yahooRepo = "https://raw.githubusercontent.com/yahoo/bookkeeper/mvn-repo" - def yahooArtifact = "org.apache.bookkeeper:bookkeeper-server:4.3.1.85-yahoo" - - @ArquillianResource - DockerClient docker - - int numEntries = 1 - - def yahooConfiguredBookKeeper(def classLoader, def zookeeper) { - // bookkeeper client configured in same way as pulsar configures it - def bkConf = classLoader.newInstance("org.apache.bookkeeper.conf.ClientConfiguration") - - bkConf.setThrottleValue(0) - bkConf.setAddEntryTimeout(30) - bkConf.setReadEntryTimeout(30) - bkConf.setSpeculativeReadTimeout(0) - bkConf.setNumChannelsPerBookie(16) - bkConf.setUseV2WireProtocol(true) - // we can't specify the class name, because it would try to use the thread - // context classloader to load, which doesn't have the required classes loaded. - bkConf.setLedgerManagerType("hierarchical") - - bkConf.enableBookieHealthCheck() - bkConf.setBookieHealthCheckInterval(60, TimeUnit.SECONDS) - bkConf.setBookieErrorThresholdPerInterval(5) - bkConf.setBookieQuarantineTime(1800, TimeUnit.SECONDS) - - bkConf.setZkServers(zookeeper) - return classLoader.newInstance("org.apache.bookkeeper.client.BookKeeper", bkConf) - } - - def addEntry(def classLoader, def ledger, def entryData) { - def promise = new CompletableFuture() - def buffer = classLoader.callStaticMethod("io.netty.buffer.Unpooled", "copiedBuffer", - [entryData.getBytes(UTF_8)]) - def callback = classLoader.createCallback( - "org.apache.bookkeeper.client.AsyncCallback\$AddCallback", - { rc, _ledger, entryId, ctx -> - if (rc != 0) { - promise.completeExceptionally( - classLoader.callStaticMethod("org.apache.bookkeeper.client.BKException", - "create", [rc])) - } else { - promise.complete(entryId) - } - }) - ledger.asyncAddEntry(buffer, callback, null) - return promise - } - - def assertCantWrite(def cl, def ledger) throws Exception { - try { - addEntry(cl, ledger, "Shouldn't write").get() - fail("Shouldn't be able to write to ledger") - } catch (ExecutionException e) { - // correct behaviour - // TODO add more - } - } - def createAndWrite(def cl, def bk) throws Exception { - def ledger = bk.createLedger(3, 2, cl.digestType("CRC32"), PASSWD) - LOG.info("Created ledger ${ledger.getId()}") - for (int i = 0; i < numEntries; i++) { - addEntry(cl, ledger, "foobar" + i).get() - } - return ledger - } - - def openAndVerifyEntries(def cl, def bk, def ledgerId) throws Exception { - LOG.info("Opening ledger $ledgerId") - def ledger = bk.openLedger(ledgerId, cl.digestType("CRC32"), PASSWD) - def entries = ledger.readEntries(0, ledger.getLastAddConfirmed()) - int j = 0 - while (entries.hasMoreElements()) { - def e = entries.nextElement() - Assert.assertEquals(new String(e.getEntry()), "foobar"+ j) - j++ - } - ledger.close() - } - - def List exerciseClients(List toVerify) { - String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker) - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - - def ledgers = [] - def yahooCL = MavenClassLoader.forArtifact(yahooRepo, yahooArtifact) - def yahooBK = yahooConfiguredBookKeeper(yahooCL, zookeeper) - def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion) - def currentBK = yahooConfiguredBookKeeper(currentCL, zookeeper) - try { - // verify we can read ledgers from previous run - for (Long id : toVerify) { - LOG.info("Verifying $id with yahoo client") - openAndVerifyEntries(yahooCL, yahooBK, id) - LOG.info("Verifying $id with current client") - openAndVerifyEntries(currentCL, currentBK, id) - } - // yahoo client and create open and read - def ledger0 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger0.getId()) - ledger0.close() - openAndVerifyEntries(yahooCL, yahooBK, ledger0.getId()) - - // yahoo client can fence on yahoo bookies - def ledger1 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger1.getId()) - openAndVerifyEntries(yahooCL, yahooBK, ledger1.getId()) - assertCantWrite(yahooCL, ledger1) - - // current client can create open and read - def ledger2 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger2.getId()) - ledger2.close() - openAndVerifyEntries(currentCL, currentBK, ledger2.getId()) - - // current client can fence on yahoo bookies - def ledger3 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger3.getId()) - openAndVerifyEntries(currentCL, currentBK, ledger3.getId()) - assertCantWrite(currentCL, ledger3) - - // current client can fence a bookie created by yahoo client - def ledger4 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger4.getId()) - openAndVerifyEntries(currentCL, currentBK, ledger4.getId()) - assertCantWrite(yahooCL, ledger4) - - // Since METADATA_VERSION is upgraded and it is using binary format, the older - // clients which are expecting text format would fail to read ledger metadata. - def ledger5 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger5.getId()) - try { - openAndVerifyEntries(yahooCL, yahooBK, ledger5.getId()) - } catch (Exception exc) { - Assert.assertEquals(exc.getClass().getName(), - "org.apache.bookkeeper.client.BKException\$ZKException") - } - } finally { - currentBK.close() - currentCL.close() - yahooBK.close() - yahooCL.close() - } - return ledgers - } - - @Test - public void testUpgradeYahooCustom() throws Exception { - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - String yahooVersion = "4.3-yahoo" - BookKeeperClusterUtils.metadataFormatIfNeeded(docker, yahooVersion) - - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, yahooVersion)) - def preUpgradeLedgers = exerciseClients([]) - Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)) - BookKeeperClusterUtils.runOnAllBookies( - docker, "cp", "/opt/bookkeeper/${yahooVersion}/conf/bookkeeper.conf", - "/opt/bookkeeper/${currentVersion}/conf/bk_server.conf") - BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, - "logSizeLimit", "1073741824") - BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, - "statsProviderClass", - "org.apache.bookkeeper.stats.NullStatsProvider") - - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion)) - // Since METADATA_VERSION is upgraded and it is using binary format, the older - // clients which are expecting text format would fail to read ledger metadata. - try { - exerciseClients(preUpgradeLedgers) - } catch (Exception exc) { - Assert.assertEquals(exc.getClass().getName(), - "org.apache.bookkeeper.client.BKException\$ZKException") - } - } -} diff --git a/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml b/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml deleted file mode 100644 index f914ff2c258..00000000000 --- a/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - CUBE - cube-definitions/3-node-all-version-unstarted.yaml - - - diff --git a/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh b/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh deleted file mode 100644 index 68fdd0437aa..00000000000 --- a/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -set -e - -gpg --import KEYS.pulsar - -TARBALL=$(ls apache-pulsar-*-incubating-bin.tar.gz | tail -1) -gpg --verify $TARBALL.asc - -tar -zxf $TARBALL -rm $TARBALL -VERSION=4.3-yahoo - -mv apache-pulsar-*-incubating /opt/bookkeeper/$VERSION - -cat > /etc/supervisord/conf.d/bookkeeper-$VERSION.conf < Date: Mon, 7 Feb 2022 14:42:58 +0100 Subject: [PATCH 2/8] fix dockerfile --- tests/docker-images/all-released-versions-image/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index 9c0301fa205..c9d4fb3a78b 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -40,17 +40,14 @@ RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookke RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.3/bookkeeper-server-4.14.3-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/pulsar-1.21.0-incubating/apache-pulsar-1.21.0-incubating-bin.tar.gz{,.asc} RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS RUN wget -nv http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552 -O KEYS.old -RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/KEYS -O KEYS.pulsar RUN mkdir -p /etc/supervisord/conf.d && mkdir -p /var/run/supervisor && mkdir -p /var/log/bookkeeper ADD conf/supervisord.conf /etc/supervisord.conf ADD scripts/install-all-tarballs.sh /install-all-tarballs.sh ADD scripts/install-tarball.sh /install-tarball.sh -ADD scripts/install-pulsar-tarball.sh /install-pulsar-tarball.sh -RUN bash /install-all-tarballs.sh && bash /install-pulsar-tarball.sh && rm -rf /tarballs +RUN bash /install-all-tarballs.sh && rm -rf /tarballs WORKDIR / ADD scripts/update-conf-and-boot.sh /update-conf-and-boot.sh From 4e3d70595e5cc49a16536d5ae9e2fda40b3390ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 7 Feb 2022 16:53:40 +0100 Subject: [PATCH 3/8] upgrade testLogger to avoid java heap space during bookies test --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b8be5831081..4d17e940227 100644 --- a/gradle.properties +++ b/gradle.properties @@ -25,5 +25,5 @@ shadowPluginVersion=6.1.0 licenseGradlePluginVersion=0.15.0 checkStyleVersion=6.19 spotbugsPlugin=4.7.0 -testLogger=2.0.0 +testLogger=3.1.0 testRetry=1.0.0 From 55fbb286686c48abf49b8b685e0f8cd0de299d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 7 Feb 2022 17:41:05 +0100 Subject: [PATCH 4/8] reduce CI logs --- .github/workflows/bookie-tests.yml | 2 +- .github/workflows/client-tests.yml | 2 +- .github/workflows/stream-tests.yml | 2 +- build.gradle | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bookie-tests.yml b/.github/workflows/bookie-tests.yml index 34a70bfa85b..b6e82c3ea34 100644 --- a/.github/workflows/bookie-tests.yml +++ b/.github/workflows/bookie-tests.yml @@ -47,4 +47,4 @@ jobs: java-version: 1.8 - name: Run bookie test - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" -Dtestlogger.theme=plain \ No newline at end of file + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" -Dtestlogger.theme=plain -DtestHideStandardOut=true \ No newline at end of file diff --git a/.github/workflows/client-tests.yml b/.github/workflows/client-tests.yml index f8c2ada72a7..4620da6d364 100644 --- a/.github/workflows/client-tests.yml +++ b/.github/workflows/client-tests.yml @@ -46,4 +46,4 @@ jobs: with: java-version: 1.8 - name: Run client tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" -Dtestlogger.theme=plain -DtestHideStandardOut=true diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml index 1cdb6461766..3be2ad3cf4d 100644 --- a/.github/workflows/stream-tests.yml +++ b/.github/workflows/stream-tests.yml @@ -55,7 +55,7 @@ jobs: - name: Run stream:serve tests run: ./gradlew stream:server:test -Dtestlogger.theme=plain - name: Run stream:statelib tests - run: ./gradlew stream:statelib:test -Dtestlogger.theme=plain + run: ./gradlew stream:statelib:test -Dtestlogger.theme=plain -DtestHideStandardOut=true - name: Run stream:storage:api:test tests run: ./gradlew stream:storage:api:test -Dtestlogger.theme=plain - name: Run stream:storage:impl tests diff --git a/build.gradle b/build.gradle index b92a70cbf41..3d78770927c 100644 --- a/build.gradle +++ b/build.gradle @@ -233,7 +233,7 @@ allprojects { systemProperty "currentVersion", project.rootProject.version testLogging { outputs.upToDateWhen {false} - showStandardStreams = true + showStandardStreams = Boolean.getBoolean("testHideStandardOut") } } tasks.register('writeClasspath') { From 0e8bb60e6bfbd0ba1e9edfe169b0a69ade430193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 7 Feb 2022 17:59:12 +0100 Subject: [PATCH 5/8] reduce CI logs --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 3d78770927c..4585ab030b9 100644 --- a/build.gradle +++ b/build.gradle @@ -233,7 +233,7 @@ allprojects { systemProperty "currentVersion", project.rootProject.version testLogging { outputs.upToDateWhen {false} - showStandardStreams = Boolean.getBoolean("testHideStandardOut") + showStandardStreams = !Boolean.getBoolean("testHideStandardOut") } } tasks.register('writeClasspath') { From 9b4d6af33862352be45c4a6c62ccaee93ccb1e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 7 Feb 2022 20:09:41 +0100 Subject: [PATCH 6/8] reduce CI logs --- .github/workflows/compatibility-check-java11.yml | 2 +- .github/workflows/remaining-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index e83ca17cbf5..60e5369de2f 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -47,4 +47,4 @@ jobs: java-version: 1.11 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" -DtestHideStandardOut=true diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml index 835d39ae3d9..a9838fd0e65 100644 --- a/.github/workflows/remaining-tests.yml +++ b/.github/workflows/remaining-tests.yml @@ -46,4 +46,4 @@ jobs: with: java-version: 1.8 - name: Run remaining tests - run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain \ No newline at end of file + run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain -DtestHideStandardOut=true \ No newline at end of file From cd6682f6842fac56914bb3c9ccaae98aa5a8aa3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 7 Feb 2022 22:56:12 +0100 Subject: [PATCH 7/8] reduce CI logs --- .github/workflows/backward-compat-tests.yml | 2 +- .github/workflows/bookie-tests.yml | 5 ++++- .github/workflows/client-tests.yml | 5 ++++- .../workflows/compatibility-check-java11.yml | 5 ++++- .../workflows/compatibility-check-java8.yml | 5 ++++- .github/workflows/integration-tests.yml | 8 ++++---- .github/workflows/pr-validation.yml | 3 +++ .github/workflows/remaining-tests.yml | 6 +++++- .github/workflows/replication-tests.yml | 4 ++-- .github/workflows/stream-tests.yml | 19 +++++++++++-------- .github/workflows/tls-tests.yml | 5 ++++- 11 files changed, 46 insertions(+), 21 deletions(-) diff --git a/.github/workflows/backward-compat-tests.yml b/.github/workflows/backward-compat-tests.yml index 03f4e66e2da..0c79290b610 100644 --- a/.github/workflows/backward-compat-tests.yml +++ b/.github/workflows/backward-compat-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: diff --git a/.github/workflows/bookie-tests.yml b/.github/workflows/bookie-tests.yml index b6e82c3ea34..cbba2993696 100644 --- a/.github/workflows/bookie-tests.yml +++ b/.github/workflows/bookie-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -47,4 +50,4 @@ jobs: java-version: 1.8 - name: Run bookie test - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" -Dtestlogger.theme=plain -DtestHideStandardOut=true \ No newline at end of file + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" \ No newline at end of file diff --git a/.github/workflows/client-tests.yml b/.github/workflows/client-tests.yml index 4620da6d364..515002de7e1 100644 --- a/.github/workflows/client-tests.yml +++ b/.github/workflows/client-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -46,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run client tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" -Dtestlogger.theme=plain -DtestHideStandardOut=true + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index 60e5369de2f..7c20cfbe841 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: @@ -47,4 +50,4 @@ jobs: java-version: 1.11 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" -DtestHideStandardOut=true + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml index 82094fddf95..aa4b0486dc7 100644 --- a/.github/workflows/compatibility-check-java8.yml +++ b/.github/workflows/compatibility-check-java8.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: @@ -47,4 +50,4 @@ jobs: java-version: 1.8 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 229baeb9762..69f60ea3813 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -52,8 +52,8 @@ jobs: - name: Build tar run: ./gradlew stream:server:build -x test - name: run cluster integration test - run: ./gradlew :tests:integration:cluster:test -Dtestlogger.theme=plain + run: ./gradlew :tests:integration:cluster:test - name: run smoke test - run: ./gradlew tests:integration:smoke:test -Dtestlogger.theme=plain + run: ./gradlew tests:integration:smoke:test - name: run standalone test - run: ./gradlew tests:integration:standalone:test -Dtestlogger.theme=plain + run: ./gradlew tests:integration:standalone:test diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index f96a401dca3..dabbcceab7d 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml index a9838fd0e65..012f2784bca 100644 --- a/.github/workflows/remaining-tests.yml +++ b/.github/workflows/remaining-tests.yml @@ -29,6 +29,10 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + + jobs: test: @@ -46,4 +50,4 @@ jobs: with: java-version: 1.8 - name: Run remaining tests - run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain -DtestHideStandardOut=true \ No newline at end of file + run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" \ No newline at end of file diff --git a/.github/workflows/replication-tests.yml b/.github/workflows/replication-tests.yml index b1c0aeaeb7f..ff053504cf6 100644 --- a/.github/workflows/replication-tests.yml +++ b/.github/workflows/replication-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run replication tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml index 3be2ad3cf4d..1622d7a36c0 100644 --- a/.github/workflows/stream-tests.yml +++ b/.github/workflows/stream-tests.yml @@ -28,6 +28,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -45,19 +48,19 @@ jobs: with: java-version: 1.8 - name: Run stream:distributedlog:core tests - run: ./gradlew stream:distributedlog:core:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:core:test - name: Run stream:distributedlog:common tests - run: ./gradlew stream:distributedlog:common:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:common:test - name: Run stream:distributedlog:protocol tests - run: ./gradlew stream:distributedlog:protocol:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:protocol:test - name: Run stream:proto tests - run: ./gradlew stream:proto:test -Dtestlogger.theme=plain + run: ./gradlew stream:proto:test - name: Run stream:serve tests - run: ./gradlew stream:server:test -Dtestlogger.theme=plain + run: ./gradlew stream:server:test - name: Run stream:statelib tests - run: ./gradlew stream:statelib:test -Dtestlogger.theme=plain -DtestHideStandardOut=true + run: ./gradlew stream:statelib:test - name: Run stream:storage:api:test tests - run: ./gradlew stream:storage:api:test -Dtestlogger.theme=plain + run: ./gradlew stream:storage:api:test - name: Run stream:storage:impl tests - run: ./gradlew stream:storage:impl:test -Dtestlogger.theme=plain + run: ./gradlew stream:storage:impl:test diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml index 3aeab4ac269..546bdf369d4 100644 --- a/.github/workflows/tls-tests.yml +++ b/.github/workflows/tls-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -46,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run tls tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" From 9d727c0759cac32ffdc5a4ce53b80f7ddd3e71dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 8 Feb 2022 00:33:18 +0100 Subject: [PATCH 8/8] reduce CI logs --- .github/workflows/backward-compat-tests.yml | 10 +++++----- .github/workflows/bookie-tests.yml | 4 ++-- .github/workflows/client-tests.yml | 4 ++-- .../workflows/compatibility-check-java11.yml | 4 ++-- .../workflows/compatibility-check-java8.yml | 4 ++-- .github/workflows/integration-tests.yml | 10 +++++----- .github/workflows/pr-validation.yml | 4 ++-- .github/workflows/remaining-tests.yml | 4 ++-- .github/workflows/replication-tests.yml | 4 ++-- .github/workflows/stream-tests.yml | 18 +++++++++--------- .github/workflows/tls-tests.yml | 4 ++-- 11 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/backward-compat-tests.yml b/.github/workflows/backward-compat-tests.yml index 0c79290b610..8efa06f68cc 100644 --- a/.github/workflows/backward-compat-tests.yml +++ b/.github/workflows/backward-compat-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,10 +49,10 @@ jobs: with: java-version: 1.8 - name: Build - run: ./gradlew stream:server:build -x test + run: ./gradlew stream:server:build -x test ${GRADLE_ARGS} - name: Test current server with old clients - run: ./gradlew :tests:backward-compat:current-server-old-clients:test || (tail -n +1 tests/backward-compat/current-server-old-clients/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/current-server-old-clients/build/container-logs/**/* && exit 1) + run: ./gradlew :tests:backward-compat:current-server-old-clients:test ${GRADLE_ARGS} || (tail -n +1 tests/backward-compat/current-server-old-clients/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/current-server-old-clients/build/container-logs/**/* && exit 1) - name: Test progressive upgrade - run: ./gradlew :tests:backward-compat:upgrade:test || (tail -n +1 tests/backward-compat/upgrade/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/upgrade/build/container-logs/**/* && exit 1) + run: ./gradlew :tests:backward-compat:upgrade:test ${GRADLE_ARGS} || (tail -n +1 tests/backward-compat/upgrade/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/upgrade/build/container-logs/**/* && exit 1) - name: Other tests - run: ./gradlew :tests:backward-compat:test -x tests:backward-compat:upgrade:test -x :tests:backward-compat:current-server-old-clients:test + run: ./gradlew :tests:backward-compat:test -x tests:backward-compat:upgrade:test -x :tests:backward-compat:current-server-old-clients:test ${GRADLE_ARGS} diff --git a/.github/workflows/bookie-tests.yml b/.github/workflows/bookie-tests.yml index cbba2993696..8eec299c5a0 100644 --- a/.github/workflows/bookie-tests.yml +++ b/.github/workflows/bookie-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -50,4 +50,4 @@ jobs: java-version: 1.8 - name: Run bookie test - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" \ No newline at end of file + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" ${GRADLE_ARGS} \ No newline at end of file diff --git a/.github/workflows/client-tests.yml b/.github/workflows/client-tests.yml index 515002de7e1..398278fc937 100644 --- a/.github/workflows/client-tests.yml +++ b/.github/workflows/client-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run client tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" ${GRADLE_ARGS} diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index 7c20cfbe841..7d93c50b90b 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: check: @@ -50,4 +50,4 @@ jobs: java-version: 1.11 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS} diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml index aa4b0486dc7..629fed6dac0 100644 --- a/.github/workflows/compatibility-check-java8.yml +++ b/.github/workflows/compatibility-check-java8.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: check: @@ -50,4 +50,4 @@ jobs: java-version: 1.8 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 69f60ea3813..59c5128c5bc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -50,10 +50,10 @@ jobs: java-version: 1.8 - name: Build tar - run: ./gradlew stream:server:build -x test + run: ./gradlew stream:server:build -x test ${GRADLE_ARGS} - name: run cluster integration test - run: ./gradlew :tests:integration:cluster:test + run: ./gradlew :tests:integration:cluster:test ${GRADLE_ARGS} - name: run smoke test - run: ./gradlew tests:integration:smoke:test + run: ./gradlew tests:integration:smoke:test ${GRADLE_ARGS} - name: run standalone test - run: ./gradlew tests:integration:standalone:test + run: ./gradlew tests:integration:standalone:test ${GRADLE_ARGS} diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index dabbcceab7d..480e43ff55b 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: @@ -50,6 +50,6 @@ jobs: with: java-version: 1.8 - name: Validate pull request - run: ./gradlew build -x spotbugsTest -x signDistTar -x test + run: ./gradlew build -x spotbugsTest -x signDistTar -x test ${GRADLE_ARGS} - name: Check license files run: dev/check-all-licenses diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml index 012f2784bca..a3f53f44297 100644 --- a/.github/workflows/remaining-tests.yml +++ b/.github/workflows/remaining-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: @@ -50,4 +50,4 @@ jobs: with: java-version: 1.8 - name: Run remaining tests - run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" \ No newline at end of file + run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" ${GRADLE_ARGS} \ No newline at end of file diff --git a/.github/workflows/replication-tests.yml b/.github/workflows/replication-tests.yml index ff053504cf6..9664d8b822c 100644 --- a/.github/workflows/replication-tests.yml +++ b/.github/workflows/replication-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run replication tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" ${GRADLE_ARGS} diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml index 1622d7a36c0..612b47bf1a8 100644 --- a/.github/workflows/stream-tests.yml +++ b/.github/workflows/stream-tests.yml @@ -29,7 +29,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -48,19 +48,19 @@ jobs: with: java-version: 1.8 - name: Run stream:distributedlog:core tests - run: ./gradlew stream:distributedlog:core:test + run: ./gradlew stream:distributedlog:core:test ${GRADLE_ARGS} - name: Run stream:distributedlog:common tests - run: ./gradlew stream:distributedlog:common:test + run: ./gradlew stream:distributedlog:common:test ${GRADLE_ARGS} - name: Run stream:distributedlog:protocol tests - run: ./gradlew stream:distributedlog:protocol:test + run: ./gradlew stream:distributedlog:protocol:test ${GRADLE_ARGS} - name: Run stream:proto tests - run: ./gradlew stream:proto:test + run: ./gradlew stream:proto:test ${GRADLE_ARGS} - name: Run stream:serve tests - run: ./gradlew stream:server:test + run: ./gradlew stream:server:test ${GRADLE_ARGS} - name: Run stream:statelib tests - run: ./gradlew stream:statelib:test + run: ./gradlew stream:statelib:test ${GRADLE_ARGS} - name: Run stream:storage:api:test tests - run: ./gradlew stream:storage:api:test + run: ./gradlew stream:storage:api:test ${GRADLE_ARGS} - name: Run stream:storage:impl tests - run: ./gradlew stream:storage:impl:test + run: ./gradlew stream:storage:impl:test ${GRADLE_ARGS} diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml index 546bdf369d4..48f7f74d4d2 100644 --- a/.github/workflows/tls-tests.yml +++ b/.github/workflows/tls-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - GRADLE_OPTS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run tls tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" ${GRADLE_ARGS}