diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml index a81d8aa15c500..96222d7f2874e 100644 --- a/.github/workflows/ci-cpp.yaml +++ b/.github/workflows/ci-cpp.yaml @@ -31,19 +31,40 @@ jobs: cpp-tests: name: runs-on: ubuntu-latest - container: - image: apachepulsar/pulsar-build:latest timeout-minutes: 120 steps: + - name: checkout uses: actions/checkout@v2 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: clean disk + run: | + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + docker rmi $(docker images -q) -f + df -h + - name: build package - run: mvn -B -ntp -q package -pl distribution/server -am -DskipTests + run: mvn -B -ntp -q install -Pcore-modules -DskipTests - name: build cpp artifacts run: | diff --git a/.github/workflows/ci-unit-broker-broker-gp1.yaml b/.github/workflows/ci-unit-broker-broker-gp1.yaml index b4b1fe9608b6f..a28a0cf808cef 100644 --- a/.github/workflows/ci-unit-broker-broker-gp1.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp1.yaml @@ -43,13 +43,13 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: build modules - run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests - - - name: run unit tests for pulsar-broker - run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/**/*.java' -Dexclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java,**/AdminApiOffloadTest.java' + run: mvn -B -ntp -q clean install -Pcore-modules -DskipTests - name: run flaky test - run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AdminApiOffloadTest.java' -DtestForkCount=1 -DtestReuseFork=true + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AdminApiOffloadTest.java' -DtestForkCount=1 -DtestReuseFork=true + + - name: run unit tests for pulsar-broker + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/**/*.java' -Dexclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java,**/AdminApiOffloadTest.java' - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker-broker-gp2.yaml b/.github/workflows/ci-unit-broker-broker-gp2.yaml index 1839ec5efd097..dd1297ea1226a 100644 --- a/.github/workflows/ci-unit-broker-broker-gp2.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp2.yaml @@ -43,23 +43,22 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: build modules - run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests + run: mvn -B -ntp -q clean install -Pcore-modules -DskipTests - name: run flaky test - run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/MessagePublishBufferThrottleTest.java' -DtestForkCount=1 -DtestReuseFork=true + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/MessagePublishBufferThrottleTest.java' -DtestForkCount=1 -DtestReuseFork=true - name: run flaky test - run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/ReplicatorTest.java' -DtestForkCount=1 -DtestReuseFork=true + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/ReplicatorTest.java' -DtestForkCount=1 -DtestReuseFork=true - name: run flaky test - run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/TopicOwnerTest.java' -DtestForkCount=1 -DtestReuseFork=true + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/TopicOwnerTest.java' -DtestForkCount=1 -DtestReuseFork=true - name: run flaky test - run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AntiAffinityNamespaceGroupTest.java' -DtestForkCount=1 -DtestReuseFork=true + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AntiAffinityNamespaceGroupTest.java' -DtestForkCount=1 -DtestReuseFork=true - name: run unit tests for pulsar-broker "org/apache/pulsar/broker/" - run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java' -Dexclude='**/ReplicatorTest.java,**/MessagePublishBufferThrottleTest.java,**/TopicOwnerTest.java,**/AntiAffinityNamespaceGroupTest.java' - + run: ./build/retry.sh mvn ${MAVEN_CLI_OPTS} test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java' -Dexclude='**/ReplicatorTest.java,**/MessagePublishBufferThrottleTest.java,**/TopicOwnerTest.java,**/AntiAffinityNamespaceGroupTest.java' - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml index d71ac446127e1..c918147bca636 100644 --- a/.github/workflows/ci-unit-broker-client-api.yaml +++ b/.github/workflows/ci-unit-broker-client-api.yaml @@ -43,12 +43,12 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: build modules - run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests + run: mvn -B -ntp -q install -Pcore-modules -DskipTests - name: run unit tests for pulsar-broker "org/apache/pulsar/client/impl" run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/client/api/**/*.java' -Dexclude='**/DispatcherBlockConsumerTest.java' - - name: run flay test + - name: run flaky test run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/DispatcherBlockConsumerTest.java' -DtestForkCount=1 -DtestReuseFork=true - name: package surefire artifacts diff --git a/.github/workflows/ci-unit-broker-client-impl.yaml b/.github/workflows/ci-unit-broker-client-impl.yaml index 32324e13502f6..56410c62cc5db 100644 --- a/.github/workflows/ci-unit-broker-client-impl.yaml +++ b/.github/workflows/ci-unit-broker-client-impl.yaml @@ -43,7 +43,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: build modules - run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests + run: mvn -B -ntp -q install -Pcore-modules -DskipTests - name: run unit tests for pulsar-broker "org/apache/pulsar/client/impl" run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/client/impl/**/*.java' diff --git a/.github/workflows/ci-unit-broker-other.yaml b/.github/workflows/ci-unit-broker-other.yaml index 7362466548384..6dc6cf8ec9884 100644 --- a/.github/workflows/ci-unit-broker-other.yaml +++ b/.github/workflows/ci-unit-broker-other.yaml @@ -43,7 +43,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: build modules - run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests + run: mvn -B -ntp -q clean install -Pcore-modules -DskipTests - name: run unit tests for pulsar-broker exlcude "org/apache/pulsar/broker", "org/apache/pulsar/client" run: mvn -B -ntp test -pl pulsar-broker -Dexclude='org/apache/pulsar/broker/**/*.java,org/apache/pulsar/client/**/*.java' diff --git a/.github/workflows/ci-unit-proxy.yaml b/.github/workflows/ci-unit-proxy.yaml index ca7f2899ade16..858c755c73c53 100644 --- a/.github/workflows/ci-unit-proxy.yaml +++ b/.github/workflows/ci-unit-proxy.yaml @@ -43,22 +43,22 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: build modules pulsar-proxy - run: mvn -B -q -ntp install -pl pulsar-proxy -am -DskipTests + run: mvn -B -ntp -q install -Pcore-modules -DskipTests - name: run flaky test - run: mvn -B -ntp install -pl pulsar-proxy -Dexclude="**/ProxyRolesEnforcementTest.java,**/ProxyAuthenticationTest.java,**/ProxyTest.java,**/MessagePublishBufferThrottleTest.java" + run: mvn -B -ntp install -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dexclude="**/ProxyRolesEnforcementTest.java,**/ProxyAuthenticationTest.java,**/ProxyTest.java,**/MessagePublishBufferThrottleTest.java" - name: run flaky test - run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyRolesEnforcementTest.java" -DtestForkCount=1 -DtestReuseFork=true + run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyRolesEnforcementTest.java" -DtestForkCount=1 -DtestReuseFork=true - name: run flaky test - run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyAuthenticationTest.java" -DtestForkCount=1 -DtestReuseFork=true + run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyAuthenticationTest.java" -DtestForkCount=1 -DtestReuseFork=true - name: run flaky test - run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyTest.java" -DtestForkCount=1 -DtestReuseFork=true + run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyTest.java" -DtestForkCount=1 -DtestReuseFork=true - name: run flaky test - run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/MessagePublishBufferThrottleTest.java" -DtestForkCount=1 -DtestReuseFork=true + run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/MessagePublishBufferThrottleTest.java" -DtestForkCount=1 -DtestReuseFork=true - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit.yaml b/.github/workflows/ci-unit.yaml index c584d3f5e8962..6f18c98dbaa11 100644 --- a/.github/workflows/ci-unit.yaml +++ b/.github/workflows/ci-unit.yaml @@ -42,20 +42,23 @@ jobs: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: run flaky test - run: mvn -B -ntp install -pl managed-ledger -Dinclude="**/ManagedLedgerTest.java" -DtestForkCount=1 -DtestReuseFork=true + - name: build and run unit tests exclude pulsar-broker and pulsar-proxy + run: ./build/retry.sh mvn -B -ntp install -PbrokerSkipTest -Dexclude="org/apache/pulsar/proxy/**/*.java,**/KafkaProducerSimpleConsumerTest.java,**/ManagedLedgerTest.java,**/TestPulsarKeyValueSchemaHandler.java,**/PrimitiveSchemaTest.java,**/BlobStoreManagedLedgerOffloaderTest.java" - - name: run flaky test - run: mvn -B -ntp install -pl pulsar-client-kafka-compat/pulsar-client-kafka_0_8 -Dinclude="**/KafkaProducerSimpleConsumerTest.java" -DtestForkCount=1 -DtestReuseFork=true + - name: run flaky test "**/ManagedLedgerTest.java" + run: ./build/retry.sh mvn -B -ntp test -pl managed-ledger -Dinclude="**/ManagedLedgerTest.java" -DtestForkCount=1 -DtestReuseFork=true - - name: run flaky test - run: mvn -B -ntp install -pl pulsar-sql/presto-pulsar-plugin -Dinclude="**/TestPulsarKeyValueSchemaHandler.java" -DtestForkCount=1 -DtestReuseFork=true + - name: run flaky test "**/KafkaProducerSimpleConsumerTest.java" + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-client-kafka-compat/pulsar-client-kafka_0_8 -Dinclude="**/KafkaProducerSimpleConsumerTest.java" -DtestForkCount=1 -DtestReuseFork=true - - name: run flaky test - run: mvn -B -ntp install -pl pulsar-client -Dinclude="**/PrimitiveSchemaTest.java" -DtestForkCount=1 -DtestReuseFork=true + - name: run flaky test "**/TestPulsarKeyValueSchemaHandler.java" + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-sql/presto-pulsar-plugin -Dinclude="**/TestPulsarKeyValueSchemaHandler.java" -DtestForkCount=1 -DtestReuseFork=true - - name: run unit tests exclude pulsar-broker and pulsar-proxy - run: mvn -B -ntp install -PbrokerSkipTest -Dexclude="org/apache/pulsar/proxy/**/*.java,**/KafkaProducerSimpleConsumerTest.java,**/ManagedLedgerTest.java,**/TestPulsarKeyValueSchemaHandler.java,**/PrimitiveSchemaTest.java" + - name: run flaky test "**/PrimitiveSchemaTest.java" + run: ./build/retry.sh mvn -B -ntp test -pl pulsar-client -Dinclude="**/PrimitiveSchemaTest.java" -DtestForkCount=1 -DtestReuseFork=true + + - name: run flaky test "**/BlobStoreManagedLedgerOffloaderTest.java" + run: ./build/retry.sh mvn -B -ntp test -pl tiered-storage/jcloud -Dinclude="**/BlobStoreManagedLedgerOffloaderTest.java" -DtestForkCount=1 -DtestReuseFork=true - name: package surefire artifacts if: failure() diff --git a/build/retry.sh b/build/retry.sh new file mode 100755 index 0000000000000..ddb0724c49bd3 --- /dev/null +++ b/build/retry.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +function fail { + echo $1 >&2 + exit 1 +} + +function retry { + local n=1 + local max=3 + local delay=10 + while true; do + "$@" && break || { + if [[ $n -lt $max ]]; then + ((n++)) + echo "Command failed. Attempt $n/$max:" + sleep $delay; + else + fail "The command has failed after $n attempts." + fi + } + done +} + +retry $@ \ No newline at end of file diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 74bd351abdb07..0557a8eb46fb1 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -333,6 +333,13 @@ + + org.bouncycastle + bc-fips + ${bouncycastlefips.version} + test + + diff --git a/pulsar-discovery-service/pom.xml b/pulsar-discovery-service/pom.xml index a5e96a2951613..587c06a4347e8 100644 --- a/pulsar-discovery-service/pom.xml +++ b/pulsar-discovery-service/pom.xml @@ -133,6 +133,13 @@ javax.activation + + org.bouncycastle + bc-fips + ${bouncycastlefips.version} + test + + ${project.groupId} testmocks