diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml index 12850852082d8..681dd63718b02 100644 --- a/.github/changes-filter.yaml +++ b/.github/changes-filter.yaml @@ -22,4 +22,10 @@ need_owasp: - 'pom.xml' - '**/pom.xml' - 'src/owasp-dependency-check-false-positives.xml' - - 'src/owasp-dependency-check-suppressions.xml' \ No newline at end of file + - 'src/owasp-dependency-check-suppressions.xml' +pulsar_dockerfile_wolfi: + - 'docker/pulsar/Dockerfile.wolfi' + - 'tests/docker-images/java-test-image/Dockerfile' +pulsar_all_dockerfile_wolfi: + - 'docker/pulsar-all/Dockerfile.wolfi' + - 'tests/docker-images/latest-version-image/Dockerfile' \ No newline at end of file diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 679bd08009d48..ad234bec6f2ff 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -70,6 +70,14 @@ on: - 'fail_on_leak' - 'off' default: 'report' + docker_image_type: + description: 'Type of docker image to build. When set to "alpine", an Alpine based docker image will be built. When set to "wolfi", a Wolfi based docker image will be built.' + required: true + type: choice + options: + - 'alpine' + - 'wolfi' + default: 'alpine' concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }} @@ -95,6 +103,8 @@ jobs: jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }} java_non_tests: ${{ steps.changes.outputs.java_non_tests }} netty_leak_detection: ${{ steps.netty_leak_detection.outputs.netty_leak_detection }} + java_test_image_base_matrix: ${{ steps.docker_images_matrix.outputs.java_test_image_base_matrix }} + latest_version_image_base_matrix: ${{ steps.docker_images_matrix.outputs.latest_version_image_base_matrix }} steps: - name: Cancel scheduled jobs in forks by default if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }} @@ -137,19 +147,57 @@ jobs: echo docs_only=false >> $GITHUB_OUTPUT fi + - name: Choose docker images to build + id: docker_images_matrix + run: | + # build JSON for java_test_image_base_matrix and latest_version_image_base_matrix + # This JSON is used as input or the build matrixes in the docker image build jobs. + # For workflow_dispatch, the docker_image_type input is used to determine the base image + # For scheduled builds, the default base image is alpine + # For pull requests, the default base image is alpine. If the changed files include Dockerfile.wolfi changes + # the wolfi base image is built in addition to the alpine image. + # create json for the default base image of the build + main_type="alpine" + if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then + main_type="${{ github.event.inputs.docker_image_type }}" + fi + if [[ "$main_type" == "wolfi" ]]; then + main_type_json='{"type": "'$main_type'", "save_artifact": true, "additional_build_args": "-Pdocker-wolfi"}' + else + main_type_json='{"type": "'$main_type'", "save_artifact": true}' + fi + # build JSON for java_test_image_base_matrix + echo "java_test_image_base_matrix<> $GITHUB_OUTPUT + echo "[" >> $GITHUB_OUTPUT + echo "$main_type_json" >> $GITHUB_OUTPUT + if [[ ${{ github.event_name == 'pull_request' && fromJSON(steps.changes.outputs.pulsar_dockerfile_wolfi_count) || 0 }} -gt 0 ]]; then + echo ',{"type": "wolfi", "save_artifact": false, "additional_build_args": "-Pdocker-wolfi"}' >> $GITHUB_OUTPUT + fi + echo "]" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + # build JSON for latest_version_image_base_matrix + echo "latest_version_image_base_matrix<> $GITHUB_OUTPUT + echo "[" >> $GITHUB_OUTPUT + echo "$main_type_json" >> $GITHUB_OUTPUT + if [[ ${{ github.event_name == 'pull_request' && fromJSON(steps.changes.outputs.pulsar_all_dockerfile_wolfi_count) || 0 }} -gt 0 ]]; then + echo ',{"type": "wolfi", "save_artifact": false, "additional_build_args": "-Pdocker-wolfi"}' >> $GITHUB_OUTPUT + fi + echo "]" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Check if coverage should be collected id: check_coverage run: | - echo "collect_coverage=${{ - (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' - && (github.base_ref == 'master' || github.ref_name == 'master')) + echo "collect_coverage=${{ + (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' + && (github.base_ref == 'master' || github.ref_name == 'master')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true') }}" >> $GITHUB_OUTPUT - name: Set Netty leak detection mode id: netty_leak_detection run: | - echo "netty_leak_detection=${{ + echo "netty_leak_detection=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.netty_leak_detection || 'report' }}" >> $GITHUB_OUTPUT @@ -457,7 +505,7 @@ jobs: - name: Restore coverage files from build artifacts and create Jacoco reports run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report cd $GITHUB_WORKSPACE/target zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true @@ -487,7 +535,7 @@ jobs: action: wait pulsar-java-test-image: - name: Build Pulsar java-test-image docker image + name: Build Pulsar java-test-image ${{ matrix.base.type }} docker image for ${{ matrix.platform }} runs-on: ubuntu-22.04 timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] @@ -495,6 +543,7 @@ jobs: strategy: fail-fast: false matrix: + base: ${{ fromJSON(needs.preconditions.outputs.java_test_image_base_matrix) }} platform: - linux/amd64 - linux/arm64 @@ -547,16 +596,16 @@ jobs: with: platforms: arm64 - - name: Build java-test-image docker image - ${{ matrix.platform }} + - name: Build java-test-image ${{ matrix.base.type }} docker image - ${{ matrix.platform }} run: | # build docker image DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ - -Ddocker.platforms=${{ matrix.platform }} \ + -Ddocker.platforms=${{ matrix.platform }} ${{ matrix.base.additional_build_args }} \ -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache - if: ${{ matrix.platform == 'linux/amd64' }} + if: ${{ matrix.base.save_artifact && matrix.platform == 'linux/amd64' }} run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image @@ -814,7 +863,7 @@ jobs: - name: Restore coverage files from build artifacts and create Jacoco reports run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report cd $GITHUB_WORKSPACE/target zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true @@ -869,11 +918,15 @@ jobs: gh-actions-artifact-client.js delete pulsar-java-test-image.zst pulsar-test-latest-version-image: - name: Build Pulsar docker image + name: Build Pulsar ${{ matrix.base.type }} docker image runs-on: ubuntu-22.04 timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + strategy: + fail-fast: false + matrix: + base: ${{ fromJSON(needs.preconditions.outputs.latest_version_image_base_matrix) }} env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} @@ -924,12 +977,12 @@ jobs: cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - name: Build latest-version-image docker image + - name: Build latest-version-image docker ${{ matrix.base.type }} image run: | # build docker image # include building of Connectors, Offloaders and server distros DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ - -Pmain,docker -Dmaven.test.skip=true \ + -Pmain,docker -Dmaven.test.skip=true ${{ matrix.base.additional_build_args }} \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true # check full build artifacts licenses @@ -957,6 +1010,7 @@ jobs: sarif_file: 'trivy-results.sarif' - name: Clean up disk space + if: ${{ matrix.base.save_artifact }} run: | # release disk space since saving docker image consumes local disk space # @@ -981,6 +1035,7 @@ jobs: echo "::endgroup::" - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache + if: ${{ matrix.base.save_artifact }} run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image diff --git a/docker/pulsar-all/Dockerfile.wolfi b/docker/pulsar-all/Dockerfile.wolfi new file mode 100644 index 0000000000000..ce9f423533148 --- /dev/null +++ b/docker/pulsar-all/Dockerfile.wolfi @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Dockerfile for pulsar-all image which uses Wolfi (chainguard/wolfi-base) as it's base image +# More information about Wolfi OS: https://github.com/wolfi-dev/ +# wolfi-base CVEs: https://images.chainguard.dev/directory/image/wolfi-base/vulnerabilities + +# global arguments (only to be used in FROM clauses) +ARG PULSAR_IMAGE + +FROM busybox AS pulsar-extensions +ARG PULSAR_IO_DIR +ADD ${PULSAR_IO_DIR} /connectors +ARG PULSAR_OFFLOADER_TARBALL +ADD ${PULSAR_OFFLOADER_TARBALL} / +RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders + +FROM ${PULSAR_IMAGE} +COPY --from=pulsar-extensions /connectors /pulsar/connectors +COPY --from=pulsar-extensions /offloaders /pulsar/offloaders \ No newline at end of file diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 4f579aa0f0490..bd59bbeac9abf 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -30,6 +30,10 @@ Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All Components) pom + + Dockerfile + + ${project.groupId} @@ -156,6 +160,7 @@ ${docker.organization}/${docker.image}-all:${docker.tag} + ${docker.dockerfile} ${project.basedir} ${docker.skip.tag} @@ -182,6 +187,12 @@ + + docker-wolfi + + Dockerfile.wolfi + + docker-push diff --git a/docker/pulsar/Dockerfile.wolfi b/docker/pulsar/Dockerfile.wolfi new file mode 100644 index 0000000000000..061dead595578 --- /dev/null +++ b/docker/pulsar/Dockerfile.wolfi @@ -0,0 +1,123 @@ +# syntax=docker/dockerfile:1 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Dockerfile for pulsar image which uses Wolfi (chainguard/wolfi-base) as it's base image +# More information about Wolfi OS: https://github.com/wolfi-dev/ +# wolfi-base CVEs: https://images.chainguard.dev/directory/image/wolfi-base/vulnerabilities + +ARG IMAGE_JDK_MAJOR_VERSION=21 + +# First create a stage with just the Pulsar tarball and scripts +FROM chainguard/wolfi-base as pulsar + +RUN apk add zip + +ARG PULSAR_TARBALL + +ADD ${PULSAR_TARBALL} / +RUN mv /apache-pulsar-* /pulsar +RUN rm -rf /pulsar/bin/*.cmd + +COPY build-scripts /build-scripts/ +RUN /build-scripts/remove-unnecessary-native-binaries.sh glibc + +COPY scripts/* /pulsar/bin/ + +# The final image needs to give the root group sufficient permission for Pulsar components +# to write to specific directories within /pulsar +# The ownership is changed to uid 10000 to allow using a different root group. This is necessary when running the +# container when gid=0 is prohibited. In that case, the container must be run with uid 10000 with +# any group id != 0 (for example 10001). +# The file permissions are preserved when copying files from this builder image to the target image. +RUN for SUBDIRECTORY in conf data download logs instances/deps packages-storage; do \ + mkdir -p /pulsar/$SUBDIRECTORY; \ + chmod -R ug+rwx /pulsar/$SUBDIRECTORY; \ + chown -R 10000:0 /pulsar/$SUBDIRECTORY; \ + done + +RUN chmod -R g+rx /pulsar/bin +RUN chmod -R o+rx /pulsar + +### Create one stage to include JVM distribution +FROM amazoncorretto:${IMAGE_JDK_MAJOR_VERSION} AS jvm + +# Install binutils which is required by jlink +RUN yum update -y && yum install -y binutils + +# Use JLink to create a slimmer JDK distribution (see: https://adoptium.net/blog/2021/10/jlink-to-produce-own-runtime/) +# This still includes all JDK modules, though in the future we could compile a list of required modules +# first try with Java 17/21 compatible jlink command and if it fails, fallback to Java 24+ compatible command +RUN /usr/lib/jvm/java/bin/jlink --add-modules ALL-MODULE-PATH --compress=2 --no-man-pages --no-header-files --strip-debug --output /opt/jvm || /usr/lib/jvm/java/bin/jlink --module-path /usr/lib/jvm/java/jmods --add-modules ALL-MODULE-PATH --compress=zip-9 --no-man-pages --no-header-files --strip-debug --output /opt/jvm +RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security +RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.security + +## Create final stage +## and add OpenJDK and Python dependencies (for Pulsar functions) +FROM chainguard/wolfi-base +ENV LANG C.UTF-8 + +ARG PYTHON_VERSION=3.12 +# Install some utilities, some are required by Pulsar scripts +RUN apk add --no-cache \ + bash \ + python-${PYTHON_VERSION} \ + py${PYTHON_VERSION}-pip \ + libgcc \ + libstdc++ \ + libuuid \ + ca-certificates \ + procps \ + curl \ + bind-tools \ + openssl \ + coreutils + +# Upgrade all packages to get latest versions with security fixes +RUN apk upgrade --no-cache + +# Python dependencies +# The pinned grpcio and protobuf versions should be compatible with the generated Protobuf and gRPC stubs used +# in Pulsar Functions Python runtime. You should also update the grpcio version in src/update_python_protobuf_stubs.sh +# and regenerate the Python stubs if you change the grpcio version here. Please see +# pulsar-functions/instance/src/main/python/README.md for more details. +ARG PULSAR_CLIENT_PYTHON_VERSION +RUN pip3 install --break-system-packages --no-cache-dir \ + --only-binary \ + grpcio==1.73.1 \ + protobuf==6.31.1 \ + pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} \ + kazoo \ + pyyaml + +COPY --from=jvm /opt/jvm /opt/jvm +ENV JAVA_HOME=/opt/jvm + +# The default is /pulsat/bin and cannot be written. +ENV PULSAR_PID_DIR=/pulsar/logs + +COPY --from=pulsar /pulsar /pulsar + +WORKDIR /pulsar +ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin + +# The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value. +ARG DEFAULT_USERNAME=pulsar +RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D -H -h /pulsar/data +USER 10000 diff --git a/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh b/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh index fe97b71179d43..d96e04603518a 100755 --- a/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh +++ b/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh @@ -41,9 +41,13 @@ unzip $ROCKSDB_JAR > /dev/null if [ "$ARCH" = "x86_64" ] then - ROCKSDB_TO_KEEP=linux64-musl + ROCKSDB_TO_KEEP=linux64 else - ROCKSDB_TO_KEEP=linux-$ARCH-musl + ROCKSDB_TO_KEEP=linux-$ARCH +fi + +if [ "$1" != "glibc" ]; then + ROCKSDB_TO_KEEP="${ROCKSDB_TO_KEEP}-musl" fi ls librocksdbjni-* | grep -v librocksdbjni-${ROCKSDB_TO_KEEP}.so | xargs rm diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 95f5ed2184441..72ab40852fb0f 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -30,6 +30,10 @@ Apache Pulsar :: Docker Images :: Pulsar Latest Version pom + + Dockerfile + + ${project.groupId} @@ -85,6 +89,7 @@ ${snappy.version} ${IMAGE_JDK_MAJOR_VERSION} + ${docker.dockerfile} ${project.basedir} ${docker.skip.tag} @@ -123,6 +128,12 @@ + + docker-wolfi + + Dockerfile.wolfi + + docker-push diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index 981c892d60e44..e4bfa77355cbf 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -18,9 +18,10 @@ # ARG PULSAR_ALL_IMAGE +ARG GOLANG_IMAGE # build go lang examples first in a separate layer -FROM golang:1.23-alpine as pulsar-function-go +FROM $GOLANG_IMAGE as pulsar-function-go COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./... diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 3e1495e1de467..ba2f94e747485 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -30,6 +30,10 @@ Apache Pulsar :: Tests :: Docker Images :: Latest Version Testing pom + + golang:1.23-alpine + + docker @@ -169,6 +173,7 @@ ${project.basedir} ${docker.organization}/${docker.image}-all:${project.version}-${git.commit.id.abbrev} + ${docker.golang.image} ${docker.install.asyncprofiler} true @@ -187,5 +192,11 @@ + + docker-wolfi + + golang:1.23 + + diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java index 1ca4352615eec..ac312f8cfd572 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/profiling/PulsarProfilingTest.java @@ -100,7 +100,8 @@ public CompletableFuture consume(String topicName) throws Exception { "/pulsar/bin/pulsar-perf", "consume", topicName, "-u", "pulsar://" + brokerHostname + ":6650", "-st", "Shared", - "-m", String.valueOf(numberOfMessages), "-ml", "200M"); + "-aq", + "-m", String.valueOf(numberOfMessages), "-ml", "400M"); } public CompletableFuture produce(String topicName) throws Exception { @@ -110,7 +111,7 @@ public CompletableFuture produce(String topicName) throws Exception { "-au", "http://" + brokerHostname + ":8080", "-r", String.valueOf(Integer.MAX_VALUE), // max-rate "-s", "8192", // 8kB message size - "-m", String.valueOf(numberOfMessages), "-ml", "200M"); + "-m", String.valueOf(numberOfMessages), "-ml", "400M"); } } @@ -136,6 +137,14 @@ public void tearDownCluster() throws Exception { super.tearDownCluster(); } + @Override + protected void beforeStartCluster() throws Exception { + super.beforeStartCluster(); + pulsarCluster.forEachContainer( + // This is effective only when -Pdocker-wolfi has been passed when building java-test-image + c -> c.withEnv("GLIBC_TUNABLES", "glibc.malloc.hugetlb=1:glibc.malloc.mmap_threshold=2097152")); + } + @Override protected PulsarClusterSpec.PulsarClusterSpecBuilder beforeSetupCluster(String clusterName, PulsarClusterSpec.PulsarClusterSpecBuilder specBuilder) { @@ -157,7 +166,8 @@ protected PulsarClusterSpec.PulsarClusterSpecBuilder beforeSetupCluster(String c "managedLedgerMaxSizePerLedgerMbytes", "512", "managedLedgerDefaultEnsembleSize", "1", "managedLedgerDefaultWriteQuorum", "1", - "managedLedgerDefaultAckQuorum", "1" + "managedLedgerDefaultAckQuorum", "1", + "maxPendingPublishRequestsPerConnection", "100000" )); // Increase memory for bookkeepers and make compaction run more often diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index 2ee571dafbbdc..73d9076ad08d0 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Consumer; import java.util.function.Function; import lombok.Cleanup; import lombok.Getter; @@ -310,6 +311,25 @@ public String getPlainTextServiceUrl() { return proxyContainer.getPlainTextServiceUrl(); } + public void forEachContainer(Consumer> consumer) { + if (zkContainer != null) { + consumer.accept(zkContainer); + } + if (csContainer != null) { + consumer.accept(csContainer); + } + if (oxiaContainer != null) { + consumer.accept(oxiaContainer); + } + if (proxyContainer != null) { + consumer.accept(proxyContainer); + } + bookieContainers.values().forEach(consumer); + brokerContainers.values().forEach(consumer); + workerContainers.values().forEach(consumer); + externalServices.values().forEach(consumer); + } + public String getHttpServiceUrl() { return proxyContainer.getHttpServiceUrl(); }