From 782b4f895b2915afbe30a62f1cd28139228f1306 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 11 Oct 2022 11:04:55 -0700 Subject: [PATCH 01/19] Added option to set the method name for Basic authentication (#37) * Added option to set the method name for Basic authentication * Fixes * Added more tests * Update lib/auth/AuthBasic.cc Co-authored-by: Yunze Xu Co-authored-by: Yunze Xu --- include/pulsar/Authentication.h | 6 +++++ lib/Authentication.cc | 4 ++-- lib/auth/AuthBasic.cc | 28 +++++++++++++++++++---- lib/auth/AuthBasic.h | 6 ++++- tests/AuthBasicTest.cc | 39 +++++++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 7 deletions(-) diff --git a/include/pulsar/Authentication.h b/include/pulsar/Authentication.h index 7ab1e65a..7f8f7d25 100644 --- a/include/pulsar/Authentication.h +++ b/include/pulsar/Authentication.h @@ -321,6 +321,12 @@ class PULSAR_PUBLIC AuthBasic : public Authentication { */ static AuthenticationPtr create(const std::string& username, const std::string& password); + /** + * Create an AuthBasic with the required parameters + */ + static AuthenticationPtr create(const std::string& username, const std::string& password, + const std::string& method); + /** * @return “basic” */ diff --git a/lib/Authentication.cc b/lib/Authentication.cc index 8fc007db..4695a03c 100644 --- a/lib/Authentication.cc +++ b/lib/Authentication.cc @@ -130,7 +130,7 @@ AuthenticationPtr tryCreateBuiltinAuth(const std::string& pluginName, ParamMap& } else if (boost::iequals(pluginName, OAUTH2_TOKEN_PLUGIN_NAME) || boost::iequals(pluginName, OAUTH2_TOKEN_JAVA_PLUGIN_NAME)) { return AuthOauth2::create(paramMap); - } else if (boost::iequals(pluginName, BASIC_PLUGIN_NAME) || + } else if (boost::iequals(pluginName, DEFAULT_BASIC_METHOD_NAME) || boost::iequals(pluginName, BASIC_JAVA_PLUGIN_NAME)) { return AuthBasic::create(paramMap); } else { @@ -150,7 +150,7 @@ AuthenticationPtr tryCreateBuiltinAuth(const std::string& pluginName, const std: } else if (boost::iequals(pluginName, OAUTH2_TOKEN_PLUGIN_NAME) || boost::iequals(pluginName, OAUTH2_TOKEN_JAVA_PLUGIN_NAME)) { return AuthOauth2::create(authParamsString); - } else if (boost::iequals(pluginName, BASIC_PLUGIN_NAME) || + } else if (boost::iequals(pluginName, DEFAULT_BASIC_METHOD_NAME) || boost::iequals(pluginName, BASIC_JAVA_PLUGIN_NAME)) { return AuthBasic::create(authParamsString); } else { diff --git a/lib/auth/AuthBasic.cc b/lib/auth/AuthBasic.cc index 463e1474..ca74803a 100644 --- a/lib/auth/AuthBasic.cc +++ b/lib/auth/AuthBasic.cc @@ -38,9 +38,14 @@ std::string base64_encode(const std::string& s) { return data.append((3 - s.size() % 3) % 3, '='); } -AuthDataBasic::AuthDataBasic(const std::string& username, const std::string& password) { +AuthDataBasic::AuthDataBasic(const std::string& username, const std::string& password) + : AuthDataBasic(username, password, DEFAULT_BASIC_METHOD_NAME) {} + +AuthDataBasic::AuthDataBasic(const std::string& username, const std::string& password, + const std::string& methodName) { commandAuthToken_ = username + ":" + password; httpAuthToken_ = base64_encode(commandAuthToken_); + methodName_ = methodName; } AuthDataBasic::~AuthDataBasic() {} @@ -53,6 +58,8 @@ bool AuthDataBasic::hasDataFromCommand() { return true; } std::string AuthDataBasic::getCommandData() { return commandAuthToken_; } +const std::string& AuthDataBasic::getMethodName() const { return methodName_; } + // AuthBasic AuthBasic::AuthBasic(AuthenticationDataPtr& authDataBasic) { authDataBasic_ = authDataBasic; } @@ -64,6 +71,13 @@ AuthenticationPtr AuthBasic::create(const std::string& username, const std::stri return AuthenticationPtr(new AuthBasic(authDataBasic)); } +AuthenticationPtr AuthBasic::create(const std::string& username, const std::string& password, + const std::string& method) { + AuthenticationDataPtr authDataBasic = + AuthenticationDataPtr(new AuthDataBasic(username, password, method)); + return AuthenticationPtr(new AuthBasic(authDataBasic)); +} + ParamMap parseBasicAuthParamsString(const std::string& authParamsString) { ParamMap params; if (!authParamsString.empty()) { @@ -96,11 +110,17 @@ AuthenticationPtr AuthBasic::create(ParamMap& params) { if (passwordIt == params.end()) { throw std::runtime_error("No password provided for basic provider"); } - - return create(usernameIt->second, passwordIt->second); + auto methodIt = params.find("method"); + if (methodIt == params.end()) { + return create(usernameIt->second, passwordIt->second); + } else { + return create(usernameIt->second, passwordIt->second, methodIt->second); + } } -const std::string AuthBasic::getAuthMethodName() const { return "basic"; } +const std::string AuthBasic::getAuthMethodName() const { + return static_cast(authDataBasic_.get())->getMethodName(); +} Result AuthBasic::getAuthData(AuthenticationDataPtr& authDataBasic) { authDataBasic = authDataBasic_; diff --git a/lib/auth/AuthBasic.h b/lib/auth/AuthBasic.h index 89b995af..2bd9e11e 100644 --- a/lib/auth/AuthBasic.h +++ b/lib/auth/AuthBasic.h @@ -25,12 +25,13 @@ namespace pulsar { -const std::string BASIC_PLUGIN_NAME = "basic"; +const std::string DEFAULT_BASIC_METHOD_NAME = "basic"; const std::string BASIC_JAVA_PLUGIN_NAME = "org.apache.pulsar.client.impl.auth.AuthenticationBasic"; class AuthDataBasic : public AuthenticationDataProvider { public: AuthDataBasic(const std::string& username, const std::string& password); + AuthDataBasic(const std::string& username, const std::string& password, const std::string& methodName); ~AuthDataBasic(); bool hasDataForHttp(); @@ -38,9 +39,12 @@ class AuthDataBasic : public AuthenticationDataProvider { bool hasDataFromCommand(); std::string getCommandData(); + const std::string& getMethodName() const; + private: std::string commandAuthToken_; std::string httpAuthToken_; + std::string methodName_; }; } // namespace pulsar diff --git a/tests/AuthBasicTest.cc b/tests/AuthBasicTest.cc index 29a3ff51..29d66248 100644 --- a/tests/AuthBasicTest.cc +++ b/tests/AuthBasicTest.cc @@ -131,6 +131,16 @@ TEST(AuthPluginBasic, testLoadAuth) { ASSERT_EQ(data->hasDataForTls(), false); ASSERT_EQ(data->hasDataForHttp(), true); + auth = pulsar::AuthBasic::create( + "{\"username\":\"super-user\",\"password\":\"123789\",\"method\":\"my-method\"}"); + ASSERT_TRUE(auth != NULL); + ASSERT_EQ(auth->getAuthMethodName(), "my-method"); + ASSERT_EQ(auth->getAuthData(data), pulsar::ResultOk); + ASSERT_EQ(data->hasDataFromCommand(), true); + ASSERT_EQ(data->getCommandData(), "super-user:123789"); + ASSERT_EQ(data->hasDataForTls(), false); + ASSERT_EQ(data->hasDataForHttp(), true); + ParamMap p = ParamMap(); p["username"] = "super-user-2"; p["password"] = "456789"; @@ -142,6 +152,19 @@ TEST(AuthPluginBasic, testLoadAuth) { ASSERT_EQ(data->getCommandData(), "super-user-2:456789"); ASSERT_EQ(data->hasDataForTls(), false); ASSERT_EQ(data->hasDataForHttp(), true); + + p = ParamMap(); + p["username"] = "super-user-2"; + p["password"] = "456789"; + p["method"] = "my-method-2"; + auth = pulsar::AuthBasic::create(p); + ASSERT_TRUE(auth != NULL); + ASSERT_EQ(auth->getAuthMethodName(), "my-method-2"); + ASSERT_EQ(auth->getAuthData(data), pulsar::ResultOk); + ASSERT_EQ(data->hasDataFromCommand(), true); + ASSERT_EQ(data->getCommandData(), "super-user-2:456789"); + ASSERT_EQ(data->hasDataForTls(), false); + ASSERT_EQ(data->hasDataForHttp(), true); } TEST(AuthPluginBasic, testAuthBasicWithServiceUrlTlsWithTlsTransport) { @@ -253,3 +276,19 @@ TEST(AuthPluginBasic, testAuthBasicWithServiceUrlHttpsNoTlsTransport) { Result result = client.createProducer(topicName, producer); ASSERT_EQ(ResultLookupError, result); } + +TEST(AuthPluginBasic, testAuthBasicWithCustomMethodName) { + ClientConfiguration config = ClientConfiguration(); + + AuthenticationPtr auth = pulsar::AuthBasic::create("admin", "123456", "method-1"); + + ASSERT_TRUE(auth != NULL); + ASSERT_EQ(auth->getAuthMethodName(), "method-1"); + + pulsar::AuthenticationDataPtr data; + ASSERT_EQ(auth->getAuthData(data), pulsar::ResultOk); + ASSERT_EQ(data->hasDataFromCommand(), true); + ASSERT_EQ(data->getCommandData(), "admin:123456"); + ASSERT_EQ(data->hasDataForTls(), false); + ASSERT_EQ(data->hasDataForHttp(), true); +} From 176c8d75384c6229c76504eeecfa77758b88577a Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 11 Oct 2022 14:19:28 -0700 Subject: [PATCH 02/19] Use Alpine 3.12 to build APKs (#39) --- pkg/apk/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index 2a3d3a03..182c7d78 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM alpine:3.16 +FROM alpine:3.12 ARG PLATFORM @@ -48,7 +48,7 @@ RUN BOOST_VERSION=$(dep-version.py boost) && \ tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ cd boost_${BOOST_VERSION_UNDESRSCORE} && \ ./bootstrap.sh --with-libraries=regex && \ - ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ + ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} # Download and compile protobuf From 0fdcbe42de21d5a5a171f3a2ab8a03dd959ce920 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 11 Oct 2022 18:47:44 -0700 Subject: [PATCH 03/19] Improved the CI check completion (#40) --- .asf.yaml | 7 +------ .github/workflows/ci-pr-validation.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index cfdfe913..1d6e21eb 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -49,12 +49,7 @@ github: # # Contexts are the names of checks that must pass. # # See ./github/workflows/README.md for more documentation on this list. contexts: - - Run unit tests - - Build CPP Client on Windows x64 - - Build CPP Client on Windows x86 - - Build Debian Package - - Build RPM Package - - Build Alpine Linux APK Package + - Check Completion required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 229e6fa7..92157bfd 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -264,3 +264,13 @@ jobs: - name: Build APK packages run: pkg/apk/docker-build-apk-x86_64.sh build-apk-x86_64:latest + + + # Job that will be required to complete and depends on all the other jobs + check-completion: + name: Check Completion + runs-on: ubuntu-latest + needs: [unit-tests, cpp-build-windows, deb-packaging, rpm-packaging, apk-packaging] + + steps: + - run: true From aa8347724c217b8c2ff8513fbcaae00df5c97dc9 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 12 Oct 2022 01:14:16 -0700 Subject: [PATCH 04/19] Move main to next version (#41) --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 370e076c..7575d0ef 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.0-pre +3.1.0-pre From ebc44377a338d6cab45d4bcc82f2dca8f0c685e0 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 13 Oct 2022 01:06:08 +0800 Subject: [PATCH 05/19] Fix Ubuntu and macOS build guide (#44) * Fix Ubuntu and macOS build guide * Fixed readme Co-authored-by: Matteo Merli --- README.md | 52 +++++++--------------------------------------------- 1 file changed, 7 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index dbad7219..c7fc3021 100644 --- a/README.md +++ b/README.md @@ -92,37 +92,11 @@ Run unit tests: #### Install all dependencies: ```shell -apt-get install -y g++ cmake libssl-dev libcurl4-openssl-dev \ - libprotobuf-dev libboost-all-dev libgtest-dev google-mock \ +sudo apt-get install -y g++ cmake libssl-dev libcurl4-openssl-dev \ + libprotobuf-dev libboost-all-dev libgtest-dev libgmock-dev \ protobuf-compiler ``` -#### Compile and install Google Test: - -```shell -cd /usr/src/gtest -sudo cmake . -sudo make - -# Copy the libraries you just built to the OS library path. -sudo cp lib/*.a /usr/lib -``` - - -#### Compile and install Google Mock: - -```shell -cd /usr/src/gmock -sudo cmake . -sudo make - -# Copy the gmock headers to the OS include path. -sudo cp -r include/gmock /usr/include/ -# Copy the libraries you just built to the OS brary path. -sudo cp lib/*.a /usr/lib -``` - - #### Compile Pulsar client library: ```shell @@ -149,17 +123,7 @@ perf/perfConsumer #### Install all dependencies: ```shell -# For openSSL -brew install openssl -export OPENSSL_INCLUDE_DIR=/usr/local/opt/openssl/include/ -export OPENSSL_ROOT_DIR=/usr/local/opt/openssl/ - -# For Protobuf -brew install protobuf boost boost-python log4cxx jsoncpp -// If you are using python3, you need to install boost-python3 - -# For GoogleTest -brew install googletest +brew install openssl protobuf boost boost-python3 googletest zstd snappy ``` #### Compile Pulsar client library: @@ -258,18 +222,16 @@ pulsar-client-cpp/build/examples/Release ## Tests ```shell -# Source code -pulsar-client-cpp/tests/ - # Execution # Start standalone broker -pulsar-test-service-start.sh +./pulsar-test-service-start.sh # Run the tests -pulsar-client-cpp/tests/main +cd tests +./pulsar-tests # When no longer needed, stop standalone broker -pulsar-test-service-stop.sh +./pulsar-test-service-stop.sh ``` ## Requirements for Contributors From 748dad5673ceeba777ffef8096651a59d037e930 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 12 Oct 2022 10:07:36 -0700 Subject: [PATCH 06/19] Removed docker-build.sh script (#46) * Removed docker-build.sh script * Removed ref to docker-format.sh --- README.md | 18 +--------- docker-build.sh | 49 --------------------------- docker-format.sh | 47 -------------------------- docker-tests.sh | 88 ------------------------------------------------ 4 files changed, 1 insertion(+), 201 deletions(-) delete mode 100755 docker-build.sh delete mode 100755 docker-format.sh delete mode 100755 docker-tests.sh diff --git a/README.md b/README.md index c7fc3021..4cea378a 100644 --- a/README.md +++ b/README.md @@ -73,21 +73,7 @@ Pulsar C++ Client Library has been tested on: ## Compilation -### Compile within a Docker container - -You can compile the C++ client library within a Docker container that already -contains all the required dependencies. - -```shell -./docker-build.sh -``` - -Run unit tests: -```shell -./docker-tests.sh -``` - -### Compile on Ubuntu Server 20.04 +### Compile on Ubuntu #### Install all dependencies: @@ -238,6 +224,4 @@ cd tests It's required to install [LLVM](https://llvm.org/builds/) for `clang-tidy` and `clang-format`. Pulsar C++ client use `clang-format` 6.0+ to format files. `make format` automatically formats the files. -Use `pulsar-client-cpp/docker-format.sh` to ensure the C++ sources are correctly formatted. - We welcome contributions from the open source community, kindly make sure your changes are backward compatible with GCC 4.8 and Boost 1.53. diff --git a/docker-build.sh b/docker-build.sh deleted file mode 100755 index 796bf9bf..00000000 --- a/docker-build.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Build Pulsar C++ client within a Docker container - -# Fail script in case of errors -set -e - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd $ROOT_DIR - -BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" -BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-20.04}" - -IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" - -echo "---- Build Pulsar C++ client using image $IMAGE (pass for incremental build)" - -docker pull $IMAGE - -VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar-client-cpp"} -COMMAND="cd /pulsar-client-cpp && cmake . $CMAKE_ARGS && make check-format && make -j8" - -DOCKER_CMD="docker run -i ${VOLUME_OPTION} ${IMAGE}" - -# Remove any cached CMake relate file from previous builds -if [ "$1" != "skip-clean" ]; then - find . -name CMakeCache.txt | xargs rm -f - find . -name CMakeFiles | xargs rm -rf -fi - -$DOCKER_CMD bash -c "${COMMAND}" diff --git a/docker-format.sh b/docker-format.sh deleted file mode 100755 index 56a805fb..00000000 --- a/docker-format.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Build Pulsar C++ client within a Docker container - -# Fail script in case of errors -set -e - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd $ROOT_DIR - -BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" -BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-20.04}" - -IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" - -echo "---- Build Pulsar C++ client using image $IMAGE" - -docker pull $IMAGE - -VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar-client-cpp"} -COMMAND="cd /pulsar-client-cpp && cmake . $CMAKE_ARGS && make format" - -DOCKER_CMD="docker run -i ${VOLUME_OPTION} ${IMAGE}" - -# Remove any cached CMake relate file from previous builds -find . -name CMakeCache.txt | xargs rm -f -find . -name CMakeFiles | xargs rm -rf - -$DOCKER_CMD bash -c "${COMMAND}" diff --git a/docker-tests.sh b/docker-tests.sh deleted file mode 100755 index 2095eb4d..00000000 --- a/docker-tests.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Run C++ unit tests within a Docker container - -# Fail script in case of errors -set -e -x - -if [ "$1" = "--help" ]; then - echo "Usage:" - echo "--tests=\"\" (eg: --test=\"BasicEndToEndTest.*\")" - exit 0 -fi - - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd $ROOT_DIR - -BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" -BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-20.04}" - -IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" - -echo "---- Testing Pulsar C++ client using image $IMAGE (type --help for more options)" - -docker pull $IMAGE - -CONTAINER_LABEL="pulsartests=$$" -export GTEST_COLOR=${GTEST_COLOR:-no} -DOCKER_CMD="docker run -e GTEST_COLOR -i -l $CONTAINER_LABEL -v $ROOT_DIR:/pulsar-client-cpp $IMAGE" - - -for args in "$@" -do - arg=$(echo $args | cut -f1 -d=) - val=$(echo $args | cut -f2 -d=) - - case "$arg" in - --tests) tests=${val} ;; - *) - esac -done - -# Start 2 Pulsar standalone instances (one with TLS and one without) -# and execute the tests -set +e -DISABLE_COLOR_OUTPUT="" -if [ "$GTEST_COLOR" = "no" ]; then - DISABLE_COLOR_OUTPUT="| cat" -fi - -# Java17 is required for CLI e.g) bin/pulsar create-token -$DOCKER_CMD bash -c "apt-get -y install openjdk-17-jre-headless &&\ - set -o pipefail; cd /pulsar-client-cpp && git config --global --add safe.directory /pulsar-client-cpp && ./run-unit-tests.sh ${tests} $DISABLE_COLOR_OUTPUT" -RES=$? -if [ $RES -ne 0 ]; then - ( - cd "$ROOT_DIR" - mkdir -p test-logs - cd test-logs - container_id=$(docker ps -a -q --filter "label=$CONTAINER_LABEL") - if [ -n "$container_id" ]; then - # copy logs from the container that ran the tests - docker commit $container_id pulsartests/$container_id - docker run -i --rm pulsartests/$container_id \ - bash -c "cd /tmp; tar zcf - gtest-parallel-logs gtest_parallel_results.json pulsar-test-dist/logs" \ - | tar zxvf - - fi - ) -fi -exit $RES From 0fbe25ed0aa0cf16187806810648848d709c1ffe Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 12 Oct 2022 11:31:09 -0700 Subject: [PATCH 07/19] Fixed typos in stage-release.sh (#47) --- build-support/stage-release.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build-support/stage-release.sh b/build-support/stage-release.sh index d435ca25..93b5c412 100755 --- a/build-support/stage-release.sh +++ b/build-support/stage-release.sh @@ -25,8 +25,8 @@ if [ $# -neq 2 ]; then exit 1 fi -DEST_PATH=$1 -WORKFLOW_ID=$1 +DEST_PATH=$(readlink -f $1) +WORKFLOW_ID=$2 pushd $(dirname "$0") PULSAR_CPP_PATH=$(git rev-parse --show-toplevel) @@ -34,8 +34,7 @@ popd mkdir -p $DEST_PATH -cd PULSAR_CPP_PATH -VERSION=$(cat version.txt | xargs) +cd $PULSAR_CPP_PATH build-support/generate-source-archive.sh $DEST_PATH build-support/download-release-artifacts.py $WORKFLOW_ID $DEST_PATH From ce8fe5a68f5594f4c79b709a0818005811ea8999 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 12 Oct 2022 19:30:35 -0700 Subject: [PATCH 08/19] Use parametrized job to build packages (#36) ### Motivation Consolidated all the RPM/DEB/APK jobs into a single matrix --- .../workflows/ci-build-binary-artifacts.yaml | 213 ++---------------- .github/workflows/ci-pr-validation.yaml | 91 ++------ 2 files changed, 46 insertions(+), 258 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index dd482313..21e13db0 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -29,188 +29,21 @@ concurrency: jobs: - deb-packaging-x86_64: - name: Build Debian Package - x86_64 - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/deb - load: true - tags: build-deb-x86_64:latest - build-args: PLATFORM=x86_64 - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build Debian packages - run: pkg/deb/docker-build-deb-x86_64.sh build-deb-x86_64:latest - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: deb-x86_64 - path: pkg/deb/BUILD/DEB - - deb-packaging-arm64: - name: Build Debian Package - Arm64 - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/deb - load: true - tags: build-deb-arm64:latest - build-args: PLATFORM=aarch64 - platforms: linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build Debian packages - run: pkg/deb/docker-build-deb-arm64.sh build-deb-arm64:latest - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: deb-arm64 - path: pkg/deb/BUILD/DEB - - rpm-packaging-x86_64: - name: Build RPM Package - x86_64 - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/rpm - load: true - tags: build-rpm-x86_64:latest - build-args: PLATFORM=x86_64 - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-x86_64.sh build-rpm-x86_64:latest - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: rpm-x86_64 - path: pkg/rpm/RPMS - - rpm-packaging-arm64: - name: Build RPM Package - arm64 - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/rpm - load: true - tags: build-rpm-arm64:latest - build-args: PLATFORM=aarch64 - platforms: linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-arm64.sh build-rpm-arm64:latest - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: rpm-arm64 - path: pkg/rpm/RPMS - - apk-packaging-x86_64: - name: Build Alpine Linux APK Package - x86_64 - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/apk - load: true - tags: build-apk-x86_64:latest - build-args: PLATFORM=x86_64 - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build APK packages - run: pkg/apk/docker-build-apk-x86_64.sh build-apk-x86_64:latest - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: apk-x86_64 - path: pkg/apk/build/x86_64 - - apk-packaging-arm64: - name: Build Alpine Linux APK Package - arm64 - runs-on: ubuntu-20.04 - timeout-minutes: 120 + package: + name: Build ${{matrix.pkg.name}} ${{matrix.cpu.platform}} + runs-on: ubuntu-22.04 + timeout-minutes: 500 + + strategy: + fail-fast: false + matrix: + pkg: + - { name: 'RPM', type: 'rpm', path: 'pkg/rpm/RPMS' } + - { name: 'Deb', type: 'deb', path: 'pkg/deb/BUILD/DEB' } + - { name: 'Alpine', type: 'apk', path: 'pkg/apk/build' } + cpu: + - { arch: 'x86_64', platform: 'x86_64' } + - { arch: 'aarch64', platform: 'arm64' } steps: - name: checkout @@ -228,19 +61,19 @@ jobs: - name: Build dependencies Docker image uses: docker/build-push-action@v3 with: - context: ./pkg/apk + context: ./pkg/${{matrix.pkg.type}} load: true - tags: build-apk-arm64:latest - build-args: PLATFORM=aarch64 - platforms: linux/arm64 + tags: build:latest + platforms: linux/${{matrix.cpu.platform}} + build-args: PLATFORM=${{matrix.cpu.arch}} cache-from: type=gha cache-to: type=gha,mode=max - - name: Build APK packages - run: pkg/apk/docker-build-apk-arm64.sh build-apk-arm64:latest + - name: Build packages + run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest - name: Upload artifacts uses: actions/upload-artifact@v3 with: - name: apk-arm64 - path: pkg/apk/build/aarch64 + name: ${{matrix.pkg.type}}-${{matrix.pkg.platform}} + path: ${{matrix.pkg.path}} diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 92157bfd..4fbc83c4 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -176,75 +176,28 @@ jobs: cmake --build ./build-1 --parallel --config Release fi - deb-packaging: - name: Build Debian Package - runs-on: ubuntu-20.04 + package: + name: Build ${{matrix.pkg.name}} ${{matrix.cpu.platform}} + runs-on: ubuntu-22.04 needs: unit-tests - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh + timeout-minutes: 500 - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/deb - load: true - tags: build-deb-x86_64:latest - build-args: PLATFORM=x86_64 - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build Debian packages - run: pkg/deb/docker-build-deb-x86_64.sh build-deb-x86_64:latest - - - rpm-packaging: - name: Build RPM Package - runs-on: ubuntu-20.04 - needs: unit-tests - timeout-minutes: 120 + strategy: + fail-fast: true + matrix: + pkg: + - { name: 'RPM', type: 'rpm', path: 'pkg/rpm/RPMS' } + - { name: 'Deb', type: 'deb', path: 'pkg/deb/BUILD/DEB' } + - { name: 'Alpine', type: 'apk', path: 'pkg/apk/build' } + cpu: + - { arch: 'x86_64', platform: 'x86_64' } steps: - name: checkout uses: actions/checkout@v2 - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/rpm - load: true - tags: build-rpm-x86_64:latest - build-args: PLATFORM=x86_64 - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-x86_64.sh build-rpm-x86_64:latest - - - apk-packaging: - name: Build Alpine Linux APK Package - runs-on: ubuntu-20.04 - needs: unit-tests - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 - name: Package Pulsar source run: build-support/generate-source-archive.sh @@ -255,22 +208,24 @@ jobs: - name: Build dependencies Docker image uses: docker/build-push-action@v3 with: - context: ./pkg/apk + context: ./pkg/${{matrix.pkg.type}} load: true - tags: build-apk-x86_64:latest - build-args: PLATFORM=x86_64 + tags: build:latest + platforms: linux/${{matrix.cpu.platform}} + build-args: PLATFORM=${{matrix.cpu.arch}} cache-from: type=gha cache-to: type=gha,mode=max - - name: Build APK packages - run: pkg/apk/docker-build-apk-x86_64.sh build-apk-x86_64:latest + - name: Build packages + run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest # Job that will be required to complete and depends on all the other jobs check-completion: name: Check Completion runs-on: ubuntu-latest - needs: [unit-tests, cpp-build-windows, deb-packaging, rpm-packaging, apk-packaging] + needs: [unit-tests, cpp-build-windows, package] steps: - run: true + From ba1a7e1c28ab186ed1d51b5cab47334cca1b2aea Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 13 Oct 2022 11:14:01 -0700 Subject: [PATCH 09/19] Updated Github actions versions to avoid deprecations (#49) --- .github/workflows/ci-build-binary-artifacts.yaml | 4 ++-- .github/workflows/ci-pr-validation.yaml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 21e13db0..f2a931ef 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -47,10 +47,10 @@ jobs: steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Package Pulsar source run: build-support/generate-source-archive.sh diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 4fbc83c4..14857084 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -36,7 +36,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install deps run: | @@ -102,10 +102,10 @@ jobs: steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Restore vcpkg and its artifacts. - uses: actions/cache@v2 + uses: actions/cache@v3 id: vcpkg-cache with: path: | @@ -194,10 +194,10 @@ jobs: steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Package Pulsar source run: build-support/generate-source-archive.sh From c1a98084ba6c41c8b8e4f822b0224a25d173f258 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sat, 15 Oct 2022 08:33:12 -0700 Subject: [PATCH 10/19] Fixed validation of cluster/namespace names containing legal non-alpha characters (#52) * Fixed validation of cluster/namespace names containing legal non-alpha characters * Fixed formatting --- lib/NamedEntity.cc | 24 +++++++++++++++++------- tests/NamespaceNameTest.cc | 8 ++++++++ tests/TopicNameTest.cc | 9 +++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/lib/NamedEntity.cc b/lib/NamedEntity.cc index ad7c385c..484c8b1f 100644 --- a/lib/NamedEntity.cc +++ b/lib/NamedEntity.cc @@ -18,19 +18,29 @@ */ #include "NamedEntity.h" +#include + +/** + * Allowed characters for property, namespace, cluster and topic names are + * alphanumeric (a-zA-Z_0-9) and these special chars -=:. + * @param name + * @return + */ bool NamedEntity::checkName(const std::string& name) { for (char c : name) { + if (isalnum(c)) { + continue; + } + switch (c) { + case '-': case '=': case ':': - case ' ': - case '!': - case '\t': - case '\r': - case '\n': - return false; + case '.': + continue; default: - break; + // Invalid character was found + return false; } } diff --git a/tests/NamespaceNameTest.cc b/tests/NamespaceNameTest.cc index 85a57494..51975c82 100644 --- a/tests/NamespaceNameTest.cc +++ b/tests/NamespaceNameTest.cc @@ -42,3 +42,11 @@ TEST(NamespaceNameTest, testNamespaceNameV2) { std::shared_ptr nn2 = NamespaceName::get("property", "namespace"); ASSERT_TRUE(*nn1 == *nn2); } + +TEST(NamespaceNameTest, testNamespaceNameLegalCharacters) { + std::shared_ptr nn1 = NamespaceName::get("cluster-1:=.", "namespace-1:=."); + ASSERT_EQ("cluster-1:=.", nn1->getProperty()); + ASSERT_TRUE(nn1->getCluster().empty()); + ASSERT_EQ("namespace-1:=.", nn1->getLocalName()); + ASSERT_TRUE(nn1->isV2()); +} diff --git a/tests/TopicNameTest.cc b/tests/TopicNameTest.cc index 377a9319..3db2a6c0 100644 --- a/tests/TopicNameTest.cc +++ b/tests/TopicNameTest.cc @@ -140,6 +140,15 @@ TEST(TopicNameTest, testIllegalCharacters) { ASSERT_FALSE(topicName); } +TEST(TopicNameTest, testLegalNonAlphaCharacters) { + std::shared_ptr topicName = TopicName::get("persistent://cluster-1:=./namespace-1:=./topic"); + ASSERT_TRUE(topicName); + ASSERT_EQ("cluster-1:=.", topicName->getProperty()); + ASSERT_EQ("namespace-1:=.", topicName->getNamespacePortion()); + ASSERT_EQ("persistent", topicName->getDomain()); + ASSERT_EQ("topic", topicName->getLocalName()); +} + TEST(TopicNameTest, testIllegalUrl) { std::shared_ptr topicName = TopicName::get("persistent:::/property/cluster/namespace/topic"); ASSERT_FALSE(topicName); From 7f7653b694996f38cf422d8bef24aa1fdfb21c59 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Tue, 18 Oct 2022 10:33:41 +0800 Subject: [PATCH 11/19] [feat] Consumer support batch receive messages. (#21) ### Motivation https://github.com/apache/pulsar/issues/17140 This PR has been reviewed in [pulsar repo](https://github.com/apache/pulsar/pull/17429). ### Modifications - Consumer support batch receives messages. - Abstract common implementation to `ConsumerImplBase`. --- include/pulsar/BatchReceivePolicy.h | 90 ++++++++++++ include/pulsar/Consumer.h | 25 ++++ include/pulsar/ConsumerConfiguration.h | 18 +++ lib/BatchReceivePolicy.cc | 57 ++++++++ lib/BatchReceivePolicyImpl.h | 29 ++++ lib/ClientImpl.h | 5 +- lib/Consumer.cc | 18 +++ lib/ConsumerConfiguration.cc | 9 ++ lib/ConsumerConfigurationImpl.h | 1 + lib/ConsumerImpl.cc | 174 +++++++++++++----------- lib/ConsumerImpl.h | 15 +- lib/ConsumerImplBase.cc | 141 +++++++++++++++++++ lib/ConsumerImplBase.h | 45 +++++- lib/HandlerBase.h | 1 + lib/MessagesImpl.cc | 58 ++++++++ lib/MessagesImpl.h | 46 +++++++ lib/MultiTopicsConsumerImpl.cc | 142 +++++++++++++------ lib/MultiTopicsConsumerImpl.h | 22 ++- tests/BasicEndToEndTest.cc | 181 +++++++++++++++++++++++++ tests/BatchReceivePolicyTest.cc | 40 ++++++ tests/ConsumerConfigurationTest.cc | 8 ++ tests/MessagesImplTest.cc | 61 +++++++++ 22 files changed, 1037 insertions(+), 149 deletions(-) create mode 100644 include/pulsar/BatchReceivePolicy.h create mode 100644 lib/BatchReceivePolicy.cc create mode 100644 lib/BatchReceivePolicyImpl.h create mode 100644 lib/ConsumerImplBase.cc create mode 100644 lib/MessagesImpl.cc create mode 100644 lib/MessagesImpl.h create mode 100644 tests/BatchReceivePolicyTest.cc create mode 100644 tests/MessagesImplTest.cc diff --git a/include/pulsar/BatchReceivePolicy.h b/include/pulsar/BatchReceivePolicy.h new file mode 100644 index 00000000..3c66da2f --- /dev/null +++ b/include/pulsar/BatchReceivePolicy.h @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef BATCH_RECEIVE_POLICY_HPP_ +#define BATCH_RECEIVE_POLICY_HPP_ + +#include +#include + +namespace pulsar { + +struct BatchReceivePolicyImpl; + +/** + * Configuration for message batch receive {@link Consumer#batchReceive()} {@link + * Consumer#batchReceiveAsync()}. + * + *

Batch receive policy can limit the number and bytes of messages in a single batch, and can specify a + * timeout for waiting for enough messages for this batch. + * + *

A batch receive action is completed as long as any one of the + * conditions (the batch has enough number or size of messages, or the waiting timeout is passed) are met. + * + *

Examples: + * 1.If set maxNumMessages = 10, maxSizeOfMessages = 1MB and without timeout, it + * means {@link Consumer#batchReceive()} will always wait until there is enough messages. + * 2.If set maxNumberOfMessages = 0, maxNumBytes = 0 and timeout = 100ms, it + * means {@link Consumer#batchReceive()} will wait for 100ms no matter whether there are enough messages. + * + *

Note: + * Must specify messages limitation(maxNumMessages, maxNumBytes) or wait timeout. + * Otherwise, {@link Messages} ingest {@link Message} will never end. + * + * @since 2.4.1 + */ +class PULSAR_PUBLIC BatchReceivePolicy { + public: + /** + * Default value: {maxNumMessage: -1, maxNumBytes: 10 * 1024 * 1024, timeoutMs: 100} + */ + BatchReceivePolicy(); + + /** + * + * @param maxNumMessage Max num message, if less than 0, it means no limit. + * @param maxNumBytes Max num bytes, if less than 0, it means no limit. + * @param timeoutMs If less than 0, it means no limit. + */ + BatchReceivePolicy(int maxNumMessage, long maxNumBytes, long timeoutMs); + + /** + * Get max time out ms. + * + * @return + */ + long getTimeoutMs() const; + + /** + * Get the maximum number of messages. + * @return + */ + int getMaxNumMessages() const; + + /** + * Get max num bytes. + * @return + */ + long getMaxNumBytes() const; + + private: + std::shared_ptr impl_; +}; +} // namespace pulsar + +#endif /* BATCH_RECEIVE_POLICY_HPP_ */ diff --git a/include/pulsar/Consumer.h b/include/pulsar/Consumer.h index 6c0ab27b..c7911b98 100644 --- a/include/pulsar/Consumer.h +++ b/include/pulsar/Consumer.h @@ -113,6 +113,31 @@ class PULSAR_PUBLIC Consumer { */ void receiveAsync(ReceiveCallback callback); + /** + * Batch receiving messages. + * + *

This calls blocks until has enough messages or wait timeout, more details to see {@link + * BatchReceivePolicy}. + * + * @param msgs a non-const reference where the received messages will be copied + * @return ResultOk when a message is received + * @return ResultInvalidConfiguration if a message listener had been set in the configuration + */ + Result batchReceive(Messages& msgs); + + /** + * Async Batch receiving messages. + *

+ * Retrieves a message when it will be available and completes callback with received message. + *

+ *

+ * batchReceiveAsync() should be called subsequently once callback gets completed with received message. + * Else it creates backlog of receive requests in the application. + *

+ * @param BatchReceiveCallback will be completed when messages are available. + */ + void batchReceiveAsync(BatchReceiveCallback callback); + /** * Acknowledge the reception of a single message. * diff --git a/include/pulsar/ConsumerConfiguration.h b/include/pulsar/ConsumerConfiguration.h index 4347c3b2..13d5cc02 100644 --- a/include/pulsar/ConsumerConfiguration.h +++ b/include/pulsar/ConsumerConfiguration.h @@ -31,6 +31,7 @@ #include #include #include +#include "BatchReceivePolicy.h" namespace pulsar { @@ -38,8 +39,10 @@ class Consumer; class PulsarWrapper; /// Callback definition for non-data operation +typedef std::vector Messages; typedef std::function ResultCallback; typedef std::function ReceiveCallback; +typedef std::function BatchReceiveCallback; typedef std::function GetLastMessageIdCallback; /// Callback definition for MessageListener @@ -378,6 +381,21 @@ class PULSAR_PUBLIC ConsumerConfiguration { */ InitialPosition getSubscriptionInitialPosition() const; + /** + * Set batch receive policy. + * + * @param batchReceivePolicy the default is + * {maxNumMessage: -1, maxNumBytes: 10 * 1024 * 1024, timeoutMs: 100} + */ + void setBatchReceivePolicy(const BatchReceivePolicy& batchReceivePolicy); + + /** + * Get batch receive policy. + * + * @return batch receive policy + */ + const BatchReceivePolicy& getBatchReceivePolicy() const; + /** * Set whether the subscription status should be replicated. * The default value is `false`. diff --git a/lib/BatchReceivePolicy.cc b/lib/BatchReceivePolicy.cc new file mode 100644 index 00000000..08aa3687 --- /dev/null +++ b/lib/BatchReceivePolicy.cc @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include "BatchReceivePolicyImpl.h" +#include "LogUtils.h" + +using namespace pulsar; + +namespace pulsar { + +DECLARE_LOG_OBJECT() + +BatchReceivePolicy::BatchReceivePolicy() : BatchReceivePolicy(-1, 10 * 1024 * 1024, 100) {} + +BatchReceivePolicy::BatchReceivePolicy(int maxNumMessage, long maxNumBytes, long timeoutMs) + : impl_(std::make_shared()) { + if (maxNumMessage <= 0 && maxNumBytes <= 0 && timeoutMs <= 0) { + throw std::invalid_argument( + "At least one of maxNumMessages, maxNumBytes and timeoutMs must be specified."); + } + if (maxNumMessage <= 0 && maxNumBytes <= 0) { + impl_->maxNumMessage = -1; + impl_->maxNumBytes = 10 * 1024 * 1024; + LOG_WARN( + "BatchReceivePolicy maxNumMessages and maxNumBytes is less than 0. Reset to default: " + "maxNumMessage(-1), maxNumBytes(10 * 1024 * 10)"); + } else { + impl_->maxNumMessage = maxNumMessage; + impl_->maxNumBytes = maxNumBytes; + } + impl_->timeoutMs = timeoutMs; +} + +long BatchReceivePolicy::getTimeoutMs() const { return impl_->timeoutMs; } + +int BatchReceivePolicy::getMaxNumMessages() const { return impl_->maxNumMessage; } + +long BatchReceivePolicy::getMaxNumBytes() const { return impl_->maxNumBytes; } + +} // namespace pulsar diff --git a/lib/BatchReceivePolicyImpl.h b/lib/BatchReceivePolicyImpl.h new file mode 100644 index 00000000..e7ba4317 --- /dev/null +++ b/lib/BatchReceivePolicyImpl.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +namespace pulsar { + +struct BatchReceivePolicyImpl { + int maxNumMessage; + long maxNumBytes; + long timeoutMs; +}; + +} // namespace pulsar diff --git a/lib/ClientImpl.h b/lib/ClientImpl.h index 466461ae..e8e77082 100644 --- a/lib/ClientImpl.h +++ b/lib/ClientImpl.h @@ -28,14 +28,12 @@ #include #include #include "ProducerImplBase.h" -#include "ConsumerImplBase.h" #include #include #include "ServiceNameResolver.h" namespace pulsar { -class ClientImpl; class PulsarFriend; typedef std::shared_ptr ClientImplPtr; typedef std::weak_ptr ClientImplWeakPtr; @@ -44,6 +42,9 @@ class ReaderImpl; typedef std::shared_ptr ReaderImplPtr; typedef std::weak_ptr ReaderImplWeakPtr; +class ConsumerImplBase; +typedef std::weak_ptr ConsumerImplBaseWeakPtr; + std::string generateRandomName(); class ClientImpl : public std::enable_shared_from_this { diff --git a/lib/Consumer.cc b/lib/Consumer.cc index 5d163629..13fb9f4a 100644 --- a/lib/Consumer.cc +++ b/lib/Consumer.cc @@ -82,6 +82,24 @@ void Consumer::receiveAsync(ReceiveCallback callback) { impl_->receiveAsync(callback); } +Result Consumer::batchReceive(Messages& msgs) { + if (!impl_) { + return ResultConsumerNotInitialized; + } + Promise promise; + impl_->batchReceiveAsync(WaitForCallbackValue(promise)); + return promise.getFuture().get(msgs); +} + +void Consumer::batchReceiveAsync(BatchReceiveCallback callback) { + if (!impl_) { + Messages msgs; + callback(ResultConsumerNotInitialized, msgs); + return; + } + impl_->batchReceiveAsync(callback); +} + Result Consumer::acknowledge(const Message& message) { return acknowledge(message.getMessageId()); } Result Consumer::acknowledge(const MessageId& messageId) { diff --git a/lib/ConsumerConfiguration.cc b/lib/ConsumerConfiguration.cc index f9fe499b..0705cca3 100644 --- a/lib/ConsumerConfiguration.cc +++ b/lib/ConsumerConfiguration.cc @@ -19,6 +19,7 @@ #include #include +#include namespace pulsar { @@ -267,4 +268,12 @@ ConsumerConfiguration& ConsumerConfiguration::setStartMessageIdInclusive(bool st bool ConsumerConfiguration::isStartMessageIdInclusive() const { return impl_->startMessageIdInclusive; } +void ConsumerConfiguration::setBatchReceivePolicy(const BatchReceivePolicy& batchReceivePolicy) { + impl_->batchReceivePolicy = batchReceivePolicy; +} + +const BatchReceivePolicy& ConsumerConfiguration::getBatchReceivePolicy() const { + return impl_->batchReceivePolicy; +} + } // namespace pulsar diff --git a/lib/ConsumerConfigurationImpl.h b/lib/ConsumerConfigurationImpl.h index cca83a38..444fedf9 100644 --- a/lib/ConsumerConfigurationImpl.h +++ b/lib/ConsumerConfigurationImpl.h @@ -45,6 +45,7 @@ struct ConsumerConfigurationImpl { ConsumerCryptoFailureAction cryptoFailureAction{ConsumerCryptoFailureAction::FAIL}; bool readCompacted{false}; InitialPosition subscriptionInitialPosition{InitialPosition::InitialPositionLatest}; + BatchReceivePolicy batchReceivePolicy{}; int patternAutoDiscoveryPeriod{60}; bool replicateSubscriptionStateEnabled{false}; std::map properties; diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 37fcd952..54e346f0 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -18,6 +18,7 @@ */ #include "ConsumerImpl.h" #include "MessageImpl.h" +#include "MessagesImpl.h" #include "Commands.h" #include "LogUtils.h" #include "TimeUtils.h" @@ -43,7 +44,8 @@ ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, bool hasParent /* = false by default */, const ConsumerTopicType consumerTopicType /* = NonPartitioned by default */, Commands::SubscriptionMode subscriptionMode, Optional startMessageId) - : HandlerBase(client, topic, Backoff(milliseconds(100), seconds(60), milliseconds(0))), + : ConsumerImplBase(client, topic, Backoff(milliseconds(100), seconds(60), milliseconds(0)), conf, + listenerExecutor ? listenerExecutor : client->getListenerExecutorProvider()->get()), waitingForZeroQueueSizeMessage(false), config_(conf), subscription_(subscriptionName), @@ -85,13 +87,6 @@ ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerDisabled()); } - // Initialize listener executor. - if (listenerExecutor) { - listenerExecutor_ = listenerExecutor; - } else { - listenerExecutor_ = client->getListenerExecutorProvider()->get(); - } - // Setup stats reporter. unsigned int statsIntervalInSeconds = client->getClientConfig().getStatsIntervalInSeconds(); if (statsIntervalInSeconds) { @@ -145,12 +140,12 @@ const std::string& ConsumerImpl::getTopic() const { return topic_; } void ConsumerImpl::start() { HandlerBase::start(); - // Initialize ackGroupingTrackerPtr_ here because the shared_from_this() was not initialized until the + // Initialize ackGroupingTrackerPtr_ here because the get_shared_this_ptr() was not initialized until the // constructor completed. if (TopicName::get(topic_)->isPersistent()) { if (config_.getAckGroupingTimeMs() > 0) { ackGroupingTrackerPtr_.reset(new AckGroupingTrackerEnabled( - client_.lock(), shared_from_this(), consumerId_, config_.getAckGroupingTimeMs(), + client_.lock(), get_shared_this_ptr(), consumerId_, config_.getAckGroupingTimeMs(), config_.getAckGroupingMaxSize())); } else { ackGroupingTrackerPtr_.reset(new AckGroupingTrackerDisabled(*this, consumerId_)); @@ -169,7 +164,7 @@ void ConsumerImpl::connectionOpened(const ClientConnectionPtr& cnx) { // Register consumer so that we can handle other incomming commands (e.g. ACTIVE_CONSUMER_CHANGE) after // sending the subscribe request. - cnx->registerConsumer(consumerId_, shared_from_this()); + cnx->registerConsumer(consumerId_, get_shared_this_ptr()); if (duringSeek_) { ackGroupingTrackerPtr_->flushAndClean(); @@ -195,13 +190,13 @@ void ConsumerImpl::connectionOpened(const ClientConnectionPtr& cnx) { config_.getSchema(), getInitialPosition(), config_.isReplicateSubscriptionStateEnabled(), config_.getKeySharedPolicy(), config_.getPriorityLevel()); cnx->sendRequestWithId(cmd, requestId) - .addListener( - std::bind(&ConsumerImpl::handleCreateConsumer, shared_from_this(), cnx, std::placeholders::_1)); + .addListener(std::bind(&ConsumerImpl::handleCreateConsumer, get_shared_this_ptr(), cnx, + std::placeholders::_1)); } void ConsumerImpl::connectionFailed(Result result) { // Keep a reference to ensure object is kept alive - ConsumerImplPtr ptr = shared_from_this(); + auto ptr = get_shared_this_ptr(); if (consumerCreatedPromise_.setFailed(result)) { state_ = Failed; @@ -244,7 +239,7 @@ void ConsumerImpl::handleCreateConsumer(const ClientConnectionPtr& cnx, Result r sendFlowPermitsToBroker(cnx, 1); } } - consumerCreatedPromise_.setValue(shared_from_this()); + consumerCreatedPromise_.setValue(get_shared_this_ptr()); } else { if (result == ResultTimeout) { // Creating the consumer has timed out. We need to ensure the broker closes the consumer @@ -257,12 +252,12 @@ void ConsumerImpl::handleCreateConsumer(const ClientConnectionPtr& cnx, Result r if (consumerCreatedPromise_.isComplete()) { // Consumer had already been initially created, we need to retry connecting in any case LOG_WARN(getName() << "Failed to reconnect consumer: " << strResult(result)); - scheduleReconnection(shared_from_this()); + scheduleReconnection(get_shared_this_ptr()); } else { // Consumer was not yet created, retry to connect to broker if it's possible if (isRetriableError(result) && (creationTimestamp_ + operationTimeut_ < TimeUtils::now())) { LOG_WARN(getName() << "Temporary error in creating consumer : " << strResult(result)); - scheduleReconnection(shared_from_this()); + scheduleReconnection(get_shared_this_ptr()); } else { LOG_ERROR(getName() << "Failed to create consumer: " << strResult(result)); consumerCreatedPromise_.setFailed(result); @@ -292,7 +287,7 @@ void ConsumerImpl::unsubscribeAsync(ResultCallback callback) { int requestId = client->newRequestId(); SharedBuffer cmd = Commands::newUnsubscribe(consumerId_, requestId); cnx->sendRequestWithId(cmd, requestId) - .addListener(std::bind(&ConsumerImpl::handleUnsubscribe, shared_from_this(), + .addListener(std::bind(&ConsumerImpl::handleUnsubscribe, get_shared_this_ptr(), std::placeholders::_1, callback)); } else { Result result = ResultNotConnected; @@ -460,34 +455,7 @@ void ConsumerImpl::messageReceived(const ClientConnectionPtr& cnx, const proto:: << startMessageId.value()); return; } - - Lock lock(pendingReceiveMutex_); - // if asyncReceive is waiting then notify callback without adding to incomingMessages queue - bool asyncReceivedWaiting = !pendingReceives_.empty(); - ReceiveCallback callback; - if (asyncReceivedWaiting) { - callback = pendingReceives_.front(); - pendingReceives_.pop(); - } - lock.unlock(); - - if (asyncReceivedWaiting) { - listenerExecutor_->postWork(std::bind(&ConsumerImpl::notifyPendingReceivedCallback, - shared_from_this(), ResultOk, m, callback)); - return; - } - - // config_.getReceiverQueueSize() != 0 or waiting For ZeroQueueSize Message` - if (config_.getReceiverQueueSize() != 0 || - (config_.getReceiverQueueSize() == 0 && messageListener_)) { - incomingMessages_.push(m); - } else { - Lock lock(mutex_); - if (waitingForZeroQueueSizeMessage) { - lock.unlock(); - incomingMessages_.push(m); - } - } + executeNotifyCallback(m); } if (messageListener_) { @@ -496,7 +464,7 @@ void ConsumerImpl::messageReceived(const ClientConnectionPtr& cnx, const proto:: } // Trigger message listener callback in a separate thread while (numOfMessageReceived--) { - listenerExecutor_->postWork(std::bind(&ConsumerImpl::internalListener, shared_from_this())); + listenerExecutor_->postWork(std::bind(&ConsumerImpl::internalListener, get_shared_this_ptr())); } } } @@ -504,16 +472,16 @@ void ConsumerImpl::messageReceived(const ClientConnectionPtr& cnx, const proto:: void ConsumerImpl::activeConsumerChanged(bool isActive) { if (eventListener_) { listenerExecutor_->postWork( - std::bind(&ConsumerImpl::internalConsumerChangeListener, shared_from_this(), isActive)); + std::bind(&ConsumerImpl::internalConsumerChangeListener, get_shared_this_ptr(), isActive)); } } void ConsumerImpl::internalConsumerChangeListener(bool isActive) { try { if (isActive) { - eventListener_->becameActive(Consumer(shared_from_this()), partitionIndex_); + eventListener_->becameActive(Consumer(get_shared_this_ptr()), partitionIndex_); } else { - eventListener_->becameInactive(Consumer(shared_from_this()), partitionIndex_); + eventListener_->becameInactive(Consumer(get_shared_this_ptr()), partitionIndex_); } } catch (const std::exception& e) { LOG_ERROR(getName() << "Exception thrown from event listener " << e.what()); @@ -527,11 +495,56 @@ void ConsumerImpl::failPendingReceiveCallback() { ReceiveCallback callback = pendingReceives_.front(); pendingReceives_.pop(); listenerExecutor_->postWork(std::bind(&ConsumerImpl::notifyPendingReceivedCallback, - shared_from_this(), ResultAlreadyClosed, msg, callback)); + get_shared_this_ptr(), ResultAlreadyClosed, msg, callback)); } lock.unlock(); } +void ConsumerImpl::executeNotifyCallback(Message& msg) { + Lock lock(pendingReceiveMutex_); + // if asyncReceive is waiting then notify callback without adding to incomingMessages queue + bool asyncReceivedWaiting = !pendingReceives_.empty(); + ReceiveCallback callback; + if (asyncReceivedWaiting) { + callback = pendingReceives_.front(); + pendingReceives_.pop(); + } + lock.unlock(); + + // has pending receive, direct callback. + if (asyncReceivedWaiting) { + listenerExecutor_->postWork(std::bind(&ConsumerImpl::notifyPendingReceivedCallback, + get_shared_this_ptr(), ResultOk, msg, callback)); + return; + } + + // try to add incoming messages. + // config_.getReceiverQueueSize() != 0 or waiting For ZeroQueueSize Message` + if (messageListener_ || config_.getReceiverQueueSize() != 0 || waitingForZeroQueueSizeMessage) { + incomingMessages_.push(msg); + incomingMessagesSize_.fetch_add(msg.getLength()); + } + + // try trigger pending batch messages + Lock batchOptionLock(batchReceiveOptionMutex_); + if (hasEnoughMessagesForBatchReceive()) { + ConsumerImplBase::notifyBatchPendingReceivedCallback(); + } +} + +void ConsumerImpl::notifyBatchPendingReceivedCallback(const BatchReceiveCallback& callback) { + auto messages = std::make_shared(batchReceivePolicy_.getMaxNumMessages(), + batchReceivePolicy_.getMaxNumBytes()); + Message peekMsg; + while (incomingMessages_.pop(peekMsg, std::chrono::milliseconds(0)) && messages->canAdd(peekMsg)) { + messageProcessed(peekMsg); + messages->add(peekMsg); + } + auto self = get_shared_this_ptr(); + listenerExecutor_->postWork( + [callback, messages, self]() { callback(ResultOk, messages->getMessageList()); }); +} + void ConsumerImpl::notifyPendingReceivedCallback(Result result, Message& msg, const ReceiveCallback& callback) { if (result == ResultOk && config_.getReceiverQueueSize() != 0) { @@ -573,19 +586,7 @@ uint32_t ConsumerImpl::receiveIndividualMessagesFromBatch(const ClientConnection } } - // - Lock lock(pendingReceiveMutex_); - if (!pendingReceives_.empty()) { - ReceiveCallback callback = pendingReceives_.front(); - pendingReceives_.pop(); - lock.unlock(); - listenerExecutor_->postWork(std::bind(&ConsumerImpl::notifyPendingReceivedCallback, - shared_from_this(), ResultOk, msg, callback)); - } else { - // Regular path, append individual message to incoming messages queue - incomingMessages_.push(msg); - lock.unlock(); - } + executeNotifyCallback(msg); } if (skippedMessages > 0) { @@ -698,7 +699,7 @@ void ConsumerImpl::internalListener() { try { consumerStatsBasePtr_->receivedMessage(msg, ResultOk); lastDequedMessageId_ = msg.getMessageId(); - messageListener_(Consumer(shared_from_this()), msg); + messageListener_(Consumer(get_shared_this_ptr()), msg); } catch (const std::exception& e) { LOG_ERROR(getName() << "Exception thrown from listener" << e.what()); } @@ -721,9 +722,7 @@ Result ConsumerImpl::fetchSingleMessageFromBroker(Message& msg) { getName() << "The incoming message queue should never be greater than 0 when Queue size is 0"); incomingMessages_.clear(); } - Lock localLock(mutex_); waitingForZeroQueueSizeMessage = true; - localLock.unlock(); sendFlowPermitsToBroker(currentCnx, 1); @@ -745,7 +744,6 @@ Result ConsumerImpl::fetchSingleMessageFromBroker(Message& msg) { } } } - return ResultOk; } Result ConsumerImpl::receive(Message& msg) { @@ -837,6 +835,8 @@ void ConsumerImpl::messageProcessed(Message& msg, bool track) { lastDequedMessageId_ = msg.getMessageId(); lock.unlock(); + incomingMessagesSize_.fetch_sub(msg.getLength()); + ClientConnectionPtr currentCnx = getCnx().lock(); if (currentCnx && msg.impl_->cnx_ != currentCnx.get()) { LOG_DEBUG(getName() << "Not adding permit since connection is different."); @@ -934,7 +934,7 @@ void ConsumerImpl::statsCallback(Result res, ResultCallback callback, proto::Com } void ConsumerImpl::acknowledgeAsync(const MessageId& msgId, ResultCallback callback) { - ResultCallback cb = std::bind(&ConsumerImpl::statsCallback, shared_from_this(), std::placeholders::_1, + ResultCallback cb = std::bind(&ConsumerImpl::statsCallback, get_shared_this_ptr(), std::placeholders::_1, callback, proto::CommandAck_AckType_Individual); if (msgId.batchIndex() != -1 && !batchAcknowledgementTracker_.isBatchReady(msgId, proto::CommandAck_AckType_Individual)) { @@ -945,7 +945,7 @@ void ConsumerImpl::acknowledgeAsync(const MessageId& msgId, ResultCallback callb } void ConsumerImpl::acknowledgeCumulativeAsync(const MessageId& msgId, ResultCallback callback) { - ResultCallback cb = std::bind(&ConsumerImpl::statsCallback, shared_from_this(), std::placeholders::_1, + ResultCallback cb = std::bind(&ConsumerImpl::statsCallback, get_shared_this_ptr(), std::placeholders::_1, callback, proto::CommandAck_AckType_Cumulative); if (!isCumulativeAcknowledgementAllowed(config_.getConsumerType())) { cb(ResultCumulativeAcknowledgementNotAllowedError); @@ -993,12 +993,12 @@ void ConsumerImpl::disconnectConsumer() { Lock lock(mutex_); connection_.reset(); lock.unlock(); - scheduleReconnection(shared_from_this()); + scheduleReconnection(get_shared_this_ptr()); } void ConsumerImpl::closeAsync(ResultCallback callback) { // Keep a reference to ensure object is kept alive - ConsumerImplPtr ptr = shared_from_this(); + ConsumerImplPtr ptr = get_shared_this_ptr(); if (state_ != Ready) { if (callback) { @@ -1041,12 +1041,16 @@ void ConsumerImpl::closeAsync(ResultCallback callback) { cnx->sendRequestWithId(Commands::newCloseConsumer(consumerId_, requestId), requestId); if (callback) { // Pass the shared pointer "ptr" to the handler to prevent the object from being destroyed - future.addListener( - std::bind(&ConsumerImpl::handleClose, shared_from_this(), std::placeholders::_1, callback, ptr)); + future.addListener(std::bind(&ConsumerImpl::handleClose, get_shared_this_ptr(), std::placeholders::_1, + callback, ptr)); } // fail pendingReceive callback failPendingReceiveCallback(); + failPendingBatchReceiveCallback(); + + // cancel timer + batchReceiveTimer_->cancel(); } void ConsumerImpl::handleClose(Result result, ResultCallback callback, ConsumerImplPtr consumer) { @@ -1102,7 +1106,7 @@ Result ConsumerImpl::resumeMessageListener() { for (size_t i = 0; i < count; i++) { // Trigger message listener callback in a separate thread - listenerExecutor_->postWork(std::bind(&ConsumerImpl::internalListener, shared_from_this())); + listenerExecutor_->postWork(std::bind(&ConsumerImpl::internalListener, get_shared_this_ptr())); } // Check current permits and determine whether to send FLOW command this->increaseAvailablePermits(getCnx().lock(), 0); @@ -1167,7 +1171,7 @@ void ConsumerImpl::getBrokerConsumerStatsAsync(BrokerConsumerStatsCallback callb << ", requestId - " << requestId); cnx->newConsumerStats(consumerId_, requestId) - .addListener(std::bind(&ConsumerImpl::brokerConsumerStatsListener, shared_from_this(), + .addListener(std::bind(&ConsumerImpl::brokerConsumerStatsListener, get_shared_this_ptr(), std::placeholders::_1, std::placeholders::_2, callback)); return; } else { @@ -1303,7 +1307,7 @@ void ConsumerImpl::internalGetLastMessageIdAsync(const BackoffPtr& backoff, Time LOG_DEBUG(getName() << " Sending getLastMessageId Command for Consumer - " << getConsumerId() << ", requestId - " << requestId); - auto self = shared_from_this(); + auto self = get_shared_this_ptr(); cnx->newGetLastMessageId(consumerId_, requestId) .addListener([this, self, callback](Result result, const GetLastMessageIdResponse& response) { if (result == ResultOk) { @@ -1384,7 +1388,7 @@ void ConsumerImpl::seekAsyncInternal(long requestId, SharedBuffer seek, const Me LOG_INFO(getName() << " Seeking subscription to " << seekId); } - std::weak_ptr weakSelf{shared_from_this()}; + std::weak_ptr weakSelf{get_shared_this_ptr()}; cnx->sendRequestWithId(seek, requestId) .addListener([this, weakSelf, callback, originalSeekMessageId](Result result, @@ -1419,4 +1423,18 @@ bool ConsumerImpl::isPriorEntryIndex(int64_t idx) { : idx <= startMessageId_.get().value().entryId(); } +bool ConsumerImpl::hasEnoughMessagesForBatchReceive() const { + if (batchReceivePolicy_.getMaxNumMessages() <= 0 && batchReceivePolicy_.getMaxNumBytes() <= 0) { + return false; + } + return (batchReceivePolicy_.getMaxNumMessages() > 0 && + incomingMessages_.size() >= batchReceivePolicy_.getMaxNumMessages()) || + (batchReceivePolicy_.getMaxNumBytes() > 0 && + incomingMessagesSize_ >= batchReceivePolicy_.getMaxNumBytes()); +} + +std::shared_ptr ConsumerImpl::get_shared_this_ptr() { + return std::dynamic_pointer_cast(shared_from_this()); +} + } /* namespace pulsar */ diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 1ad3a4c3..09d2c5cf 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -65,9 +65,7 @@ enum ConsumerTopicType Partitioned }; -class ConsumerImpl : public ConsumerImplBase, - public HandlerBase, - public std::enable_shared_from_this { +class ConsumerImpl : public ConsumerImplBase { public: ConsumerImpl(const ClientImplPtr client, const std::string& topic, const std::string& subscriptionName, const ConsumerConfiguration&, bool isPersistent, @@ -147,7 +145,10 @@ class ConsumerImpl : public ConsumerImplBase, // overrided methods from HandlerBase void connectionOpened(const ClientConnectionPtr& cnx) override; void connectionFailed(Result result) override; - HandlerBaseWeakPtr get_weak_from_this() override { return shared_from_this(); } + + // impl methods from ConsumerImpl base + bool hasEnoughMessagesForBatchReceive() const override; + void notifyBatchPendingReceivedCallback(const BatchReceiveCallback& callback) override; void handleCreateConsumer(const ClientConnectionPtr& cnx, Result result); @@ -159,7 +160,8 @@ class ConsumerImpl : public ConsumerImplBase, ConsumerStatsBasePtr consumerStatsBasePtr_; private: - bool waitingForZeroQueueSizeMessage; + std::atomic_bool waitingForZeroQueueSizeMessage; + std::shared_ptr get_shared_this_ptr(); bool uncompressMessageIfNeeded(const ClientConnectionPtr& cnx, const proto::MessageIdData& messageIdData, const proto::MessageMetadata& metadata, SharedBuffer& payload, bool checkMaxMessageSize); @@ -180,6 +182,7 @@ class ConsumerImpl : public ConsumerImplBase, Result receiveHelper(Message& msg); Result receiveHelper(Message& msg, int timeout); void statsCallback(Result, ResultCallback, proto::CommandAck_AckType); + void executeNotifyCallback(Message& msg); void notifyPendingReceivedCallback(Result result, Message& message, const ReceiveCallback& callback); void failPendingReceiveCallback(); void setNegativeAcknowledgeEnabledForTesting(bool enabled) override; @@ -199,13 +202,13 @@ class ConsumerImpl : public ConsumerImplBase, const bool isPersistent_; MessageListener messageListener_; ConsumerEventListenerPtr eventListener_; - ExecutorServicePtr listenerExecutor_; bool hasParent_; ConsumerTopicType consumerTopicType_; const Commands::SubscriptionMode subscriptionMode_; UnboundedBlockingQueue incomingMessages_; + std::atomic_int incomingMessagesSize_ = {0}; std::queue pendingReceives_; std::atomic_int availablePermits_; const int receiverQueueRefillThreshold_; diff --git a/lib/ConsumerImplBase.cc b/lib/ConsumerImplBase.cc new file mode 100644 index 00000000..4a8c0276 --- /dev/null +++ b/lib/ConsumerImplBase.cc @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "ConsumerImpl.h" +#include "MessageImpl.h" +#include "MessagesImpl.h" +#include "LogUtils.h" +#include "TimeUtils.h" +#include "pulsar/Result.h" +#include "MessageIdUtil.h" +#include "AckGroupingTracker.h" +#include "ConsumerImplBase.h" + +#include + +DECLARE_LOG_OBJECT() + +namespace pulsar { + +ConsumerImplBase::ConsumerImplBase(ClientImplPtr client, const std::string& topic, Backoff backoff, + const ConsumerConfiguration& conf, ExecutorServicePtr listenerExecutor) + : HandlerBase(client, topic, backoff), + listenerExecutor_(listenerExecutor), + batchReceivePolicy_(conf.getBatchReceivePolicy()) { + auto userBatchReceivePolicy = conf.getBatchReceivePolicy(); + if (userBatchReceivePolicy.getMaxNumMessages() > conf.getReceiverQueueSize()) { + batchReceivePolicy_ = + BatchReceivePolicy(conf.getReceiverQueueSize(), userBatchReceivePolicy.getMaxNumBytes(), + userBatchReceivePolicy.getTimeoutMs()); + LOG_WARN("BatchReceivePolicy maxNumMessages: {" << userBatchReceivePolicy.getMaxNumMessages() + << "} is greater than maxReceiverQueueSize: {" + << conf.getReceiverQueueSize() + << "}, reset to " + "maxReceiverQueueSize. "); + } + batchReceiveTimer_ = listenerExecutor_->createDeadlineTimer(); +} + +void ConsumerImplBase::triggerBatchReceiveTimerTask(long timeoutMs) { + if (timeoutMs > 0) { + batchReceiveTimer_->expires_from_now(boost::posix_time::milliseconds(timeoutMs)); + std::weak_ptr weakSelf{shared_from_this()}; + batchReceiveTimer_->async_wait([weakSelf](const boost::system::error_code& ec) { + auto self = weakSelf.lock(); + if (self && !ec) { + self->doBatchReceiveTimeTask(); + } + }); + } +} + +void ConsumerImplBase::doBatchReceiveTimeTask() { + if (state_ != Ready) { + return; + } + + bool hasPendingReceives = false; + long timeToWaitMs; + + Lock lock(batchPendingReceiveMutex_); + while (!batchPendingReceives_.empty()) { + OpBatchReceive& batchReceive = batchPendingReceives_.front(); + long diff = + batchReceivePolicy_.getTimeoutMs() - (TimeUtils::currentTimeMillis() - batchReceive.createAt_); + if (diff <= 0) { + Lock batchOptionLock(batchReceiveOptionMutex_); + notifyBatchPendingReceivedCallback(batchReceive.batchReceiveCallback_); + batchOptionLock.unlock(); + batchPendingReceives_.pop(); + } else { + hasPendingReceives = true; + timeToWaitMs = diff; + break; + } + } + lock.unlock(); + + if (hasPendingReceives) { + triggerBatchReceiveTimerTask(timeToWaitMs); + } +} + +void ConsumerImplBase::failPendingBatchReceiveCallback() { + Lock lock(batchPendingReceiveMutex_); + while (!batchPendingReceives_.empty()) { + OpBatchReceive opBatchReceive = batchPendingReceives_.front(); + batchPendingReceives_.pop(); + listenerExecutor_->postWork( + [opBatchReceive]() { opBatchReceive.batchReceiveCallback_(ResultAlreadyClosed, {}); }); + } +} + +void ConsumerImplBase::notifyBatchPendingReceivedCallback() { + Lock lock(batchPendingReceiveMutex_); + if (!batchPendingReceives_.empty()) { + OpBatchReceive& batchReceive = batchPendingReceives_.front(); + batchPendingReceives_.pop(); + lock.unlock(); + notifyBatchPendingReceivedCallback(batchReceive.batchReceiveCallback_); + } +} + +void ConsumerImplBase::batchReceiveAsync(BatchReceiveCallback callback) { + // fail the callback if consumer is closing or closed + if (state_ != Ready) { + callback(ResultAlreadyClosed, Messages()); + return; + } + + Lock batchOptionLock(batchReceiveOptionMutex_); + if (hasEnoughMessagesForBatchReceive()) { + notifyBatchPendingReceivedCallback(callback); + batchOptionLock.unlock(); + } else { + OpBatchReceive opBatchReceive(callback); + Lock lock(batchPendingReceiveMutex_); + batchPendingReceives_.emplace(opBatchReceive); + lock.unlock(); + triggerBatchReceiveTimerTask(batchReceivePolicy_.getTimeoutMs()); + } +} + +OpBatchReceive::OpBatchReceive(const BatchReceiveCallback& batchReceiveCallback) + : batchReceiveCallback_(batchReceiveCallback), createAt_(TimeUtils::currentTimeMillis()) {} + +} /* namespace pulsar */ diff --git a/lib/ConsumerImplBase.h b/lib/ConsumerImplBase.h index 693d4da9..18b8bc1c 100644 --- a/lib/ConsumerImplBase.h +++ b/lib/ConsumerImplBase.h @@ -20,23 +20,38 @@ #define PULSAR_CONSUMER_IMPL_BASE_HEADER #include #include - +#include "HandlerBase.h" +#include #include namespace pulsar { class ConsumerImplBase; +class HandlerBase; typedef std::weak_ptr ConsumerImplBaseWeakPtr; -class ConsumerImplBase { +class OpBatchReceive { public: - virtual ~ConsumerImplBase() {} + OpBatchReceive(); + explicit OpBatchReceive(const BatchReceiveCallback& batchReceiveCallback); + const BatchReceiveCallback batchReceiveCallback_; + const int64_t createAt_; +}; + +class ConsumerImplBase : public HandlerBase, public std::enable_shared_from_this { + public: + virtual ~ConsumerImplBase(){}; + ConsumerImplBase(ClientImplPtr client, const std::string& topic, Backoff backoff, + const ConsumerConfiguration& conf, ExecutorServicePtr listenerExecutor); + + // interface by consumer virtual Future getConsumerCreatedFuture() = 0; - virtual const std::string& getSubscriptionName() const = 0; virtual const std::string& getTopic() const = 0; + virtual const std::string& getSubscriptionName() const = 0; virtual Result receive(Message& msg) = 0; virtual Result receive(Message& msg, int timeout) = 0; virtual void receiveAsync(ReceiveCallback& callback) = 0; + void batchReceiveAsync(BatchReceiveCallback callback); virtual void unsubscribeAsync(ResultCallback callback) = 0; virtual void acknowledgeAsync(const MessageId& msgId, ResultCallback callback) = 0; virtual void acknowledgeCumulativeAsync(const MessageId& msgId, ResultCallback callback) = 0; @@ -49,7 +64,6 @@ class ConsumerImplBase { virtual Result resumeMessageListener() = 0; virtual void redeliverUnacknowledgedMessages() = 0; virtual void redeliverUnacknowledgedMessages(const std::set& messageIds) = 0; - virtual const std::string& getName() const = 0; virtual int getNumOfPrefetchedMessages() const = 0; virtual void getBrokerConsumerStatsAsync(BrokerConsumerStatsCallback callback) = 0; virtual void seekAsync(const MessageId& msgId, ResultCallback callback) = 0; @@ -57,6 +71,27 @@ class ConsumerImplBase { virtual void negativeAcknowledge(const MessageId& msgId) = 0; virtual bool isConnected() const = 0; virtual uint64_t getNumberOfConnectedConsumer() = 0; + // overrided methods from HandlerBase + virtual const std::string& getName() const override = 0; + + protected: + // overrided methods from HandlerBase + void connectionOpened(const ClientConnectionPtr& cnx) override {} + void connectionFailed(Result result) override {} + HandlerBaseWeakPtr get_weak_from_this() override { return shared_from_this(); } + + // consumer impl generic method. + ExecutorServicePtr listenerExecutor_; + std::queue batchPendingReceives_; + BatchReceivePolicy batchReceivePolicy_; + DeadlineTimerPtr batchReceiveTimer_; + std::mutex batchReceiveOptionMutex_; + void triggerBatchReceiveTimerTask(long timeoutMs); + void doBatchReceiveTimeTask(); + void failPendingBatchReceiveCallback(); + void notifyBatchPendingReceivedCallback(); + virtual void notifyBatchPendingReceivedCallback(const BatchReceiveCallback& callback) = 0; + virtual bool hasEnoughMessagesForBatchReceive() const = 0; private: virtual void setNegativeAcknowledgeEnabledForTesting(bool enabled) = 0; diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index 1184746d..6fc3603d 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -90,6 +90,7 @@ class HandlerBase { ExecutorServicePtr executor_; mutable std::mutex mutex_; std::mutex pendingReceiveMutex_; + std::mutex batchPendingReceiveMutex_; ptime creationTimestamp_; const TimeDuration operationTimeut_; diff --git a/lib/MessagesImpl.cc b/lib/MessagesImpl.cc new file mode 100644 index 00000000..7d45cddc --- /dev/null +++ b/lib/MessagesImpl.cc @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "MessagesImpl.h" +#include "stdexcept" + +MessagesImpl::MessagesImpl(int maxNumberOfMessages, long maxSizeOfMessages) + : maxNumberOfMessages_(maxNumberOfMessages), + maxSizeOfMessages_(maxSizeOfMessages), + currentSizeOfMessages_(0) {} + +const std::vector& MessagesImpl::getMessageList() const { return messageList_; } + +bool MessagesImpl::canAdd(const Message& message) const { + if (messageList_.size() == 0) { + return true; + } + + if (maxNumberOfMessages_ > 0 && messageList_.size() + 1 > maxNumberOfMessages_) { + return false; + } + + if (maxSizeOfMessages_ > 0 && currentSizeOfMessages_ + message.getLength() > maxSizeOfMessages_) { + return false; + } + + return true; +} + +void MessagesImpl::add(const Message& message) { + if (!canAdd(message)) { + throw std::invalid_argument("No more space to add messages."); + } + currentSizeOfMessages_ += message.getLength(); + messageList_.emplace_back(message); +} + +int MessagesImpl::size() const { return messageList_.size(); } + +void MessagesImpl::clear() { + currentSizeOfMessages_ = 0; + messageList_.clear(); +} diff --git a/lib/MessagesImpl.h b/lib/MessagesImpl.h new file mode 100644 index 00000000..0c12768f --- /dev/null +++ b/lib/MessagesImpl.h @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef PULSAR_CPP_MESSAGESIMPL_H +#define PULSAR_CPP_MESSAGESIMPL_H + +#include +#include + +using namespace pulsar; + +namespace pulsar { + +class MessagesImpl { + public: + MessagesImpl(const int maxNumberOfMessages, const long maxSizeOfMessages); + const std::vector& getMessageList() const; + bool canAdd(const Message& message) const; + void add(const Message& message); + int size() const; + void clear(); + + private: + std::vector messageList_; + const int maxNumberOfMessages_; + const long maxSizeOfMessages_; + long currentSizeOfMessages_; +}; + +} // namespace pulsar +#endif // PULSAR_CPP_MESSAGESIMPL_H diff --git a/lib/MultiTopicsConsumerImpl.cc b/lib/MultiTopicsConsumerImpl.cc index 0d730e15..573c33d9 100644 --- a/lib/MultiTopicsConsumerImpl.cc +++ b/lib/MultiTopicsConsumerImpl.cc @@ -18,6 +18,7 @@ */ #include "MultiTopicsConsumerImpl.h" #include "MultiResultCallback.h" +#include "MessagesImpl.h" DECLARE_LOG_OBJECT() @@ -27,12 +28,13 @@ MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std const std::string& subscriptionName, TopicNamePtr topicName, const ConsumerConfiguration& conf, LookupServicePtr lookupServicePtr) - : client_(client), + : ConsumerImplBase(client, topicName ? topicName->toString() : "EmptyTopics", + Backoff(milliseconds(100), seconds(60), milliseconds(0)), conf, + client->getListenerExecutorProvider()->get()), + client_(client), subscriptionName_(subscriptionName), - topic_(topicName ? topicName->toString() : "EmptyTopics"), conf_(conf), - messages_(conf.getReceiverQueueSize()), - listenerExecutor_(client->getListenerExecutorProvider()->get()), + incomingMessages_(conf.getReceiverQueueSize()), messageListener_(conf.getMessageListener()), lookupServicePtr_(lookupServicePtr), numberTopicPartitions_(std::make_shared>(0)), @@ -59,14 +61,16 @@ MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std partitionsUpdateInterval_ = boost::posix_time::seconds(partitionsUpdateInterval); lookupServicePtr_ = client_->getLookup(); } + + state_ = Pending; } void MultiTopicsConsumerImpl::start() { if (topics_.empty()) { - MultiTopicsConsumerState state = Pending; + State state = Pending; if (state_.compare_exchange_strong(state, Ready)) { LOG_DEBUG("No topics passed in when create MultiTopicsConsumer."); - multiTopicsConsumerCreatedPromise_.setValue(shared_from_this()); + multiTopicsConsumerCreatedPromise_.setValue(get_shared_this_ptr()); return; } else { LOG_ERROR("Consumer " << consumerStr_ << " in wrong state: " << state_); @@ -81,7 +85,7 @@ void MultiTopicsConsumerImpl::start() { // subscribe for each passed in topic for (std::vector::const_iterator itr = topics_.begin(); itr != topics_.end(); itr++) { subscribeOneTopicAsync(*itr).addListener(std::bind(&MultiTopicsConsumerImpl::handleOneTopicSubscribed, - shared_from_this(), std::placeholders::_1, + get_shared_this_ptr(), std::placeholders::_1, std::placeholders::_2, *itr, topicsNeedCreate)); } } @@ -100,10 +104,10 @@ void MultiTopicsConsumerImpl::handleOneTopicSubscribed(Result result, Consumer c } if (--(*topicsNeedCreate) == 0) { - MultiTopicsConsumerState state = Pending; + State state = Pending; if (state_.compare_exchange_strong(state, Ready)) { LOG_INFO("Successfully Subscribed to Topics"); - multiTopicsConsumerCreatedPromise_.setValue(shared_from_this()); + multiTopicsConsumerCreatedPromise_.setValue(get_shared_this_ptr()); } else { LOG_ERROR("Unable to create Consumer - " << consumerStr_ << " Error - " << result); // unsubscribed all of the successfully subscribed partitioned consumers @@ -162,7 +166,7 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(int numPartitions, TopicN ConsumerConfiguration config = conf_.clone(); ExecutorServicePtr internalListenerExecutor = client_->getPartitionListenerExecutorProvider()->get(); - config.setMessageListener(std::bind(&MultiTopicsConsumerImpl::messageReceived, shared_from_this(), + config.setMessageListener(std::bind(&MultiTopicsConsumerImpl::messageReceived, get_shared_this_ptr(), std::placeholders::_1, std::placeholders::_2)); int partitions = numPartitions == 0 ? 1 : numPartitions; @@ -186,8 +190,8 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(int numPartitions, TopicN topicName->isPersistent(), internalListenerExecutor, true, NonPartitioned); consumer->getConsumerCreatedFuture().addListener(std::bind( - &MultiTopicsConsumerImpl::handleSingleConsumerCreated, shared_from_this(), std::placeholders::_1, - std::placeholders::_2, partitionsNeedCreate, topicSubResultPromise)); + &MultiTopicsConsumerImpl::handleSingleConsumerCreated, get_shared_this_ptr(), + std::placeholders::_1, std::placeholders::_2, partitionsNeedCreate, topicSubResultPromise)); consumers_.emplace(topicName->toString(), consumer); LOG_DEBUG("Creating Consumer for - " << topicName << " - " << consumerStr_); consumer->start(); @@ -199,7 +203,7 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(int numPartitions, TopicN topicName->isPersistent(), internalListenerExecutor, true, Partitioned); consumer->getConsumerCreatedFuture().addListener(std::bind( - &MultiTopicsConsumerImpl::handleSingleConsumerCreated, shared_from_this(), + &MultiTopicsConsumerImpl::handleSingleConsumerCreated, get_shared_this_ptr(), std::placeholders::_1, std::placeholders::_2, partitionsNeedCreate, topicSubResultPromise)); consumer->setPartitionIndex(i); consumers_.emplace(topicPartitionName, consumer); @@ -236,7 +240,7 @@ void MultiTopicsConsumerImpl::handleSingleConsumerCreated( if (partitionsUpdateTimer_) { runPartitionUpdateTask(); } - topicSubResultPromise->setValue(Consumer(shared_from_this())); + topicSubResultPromise->setValue(Consumer(get_shared_this_ptr())); } } @@ -252,7 +256,7 @@ void MultiTopicsConsumerImpl::unsubscribeAsync(ResultCallback callback) { state_ = Closing; std::shared_ptr> consumerUnsubed = std::make_shared>(0); - auto self = shared_from_this(); + auto self = get_shared_this_ptr(); int numConsumers = 0; consumers_.forEachValue( [&numConsumers, &consumerUnsubed, &self, callback](const ConsumerImplPtr& consumer) { @@ -329,7 +333,7 @@ void MultiTopicsConsumerImpl::unsubscribeOneTopicAsync(const std::string& topic, } optConsumer.value()->unsubscribeAsync( - std::bind(&MultiTopicsConsumerImpl::handleOneTopicUnsubscribedAsync, shared_from_this(), + std::bind(&MultiTopicsConsumerImpl::handleOneTopicUnsubscribedAsync, get_shared_this_ptr(), std::placeholders::_1, consumerUnsubed, numberPartitions, topicName, topicPartitionName, callback)); } @@ -385,7 +389,7 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { state_ = Closing; - std::weak_ptr weakSelf{shared_from_this()}; + std::weak_ptr weakSelf{get_shared_this_ptr()}; int numConsumers = 0; consumers_.clear( [this, weakSelf, &numConsumers, callback](const std::string& name, const ConsumerImplPtr& consumer) { @@ -414,7 +418,7 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { } // closed all consumers if (numConsumersLeft == 0) { - messages_.clear(); + incomingMessages_.clear(); topicsPartitions_.clear(); unAckedMessageTrackerPtr_->clear(); @@ -440,6 +444,10 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { // fail pending receive failPendingReceiveCallback(); + failPendingBatchReceiveCallback(); + + // cancel timer + batchReceiveTimer_->cancel(); } void MultiTopicsConsumerImpl::messageReceived(Consumer consumer, const Message& msg) { @@ -454,25 +462,39 @@ void MultiTopicsConsumerImpl::messageReceived(Consumer consumer, const Message& pendingReceives_.pop(); lock.unlock(); listenerExecutor_->postWork(std::bind(&MultiTopicsConsumerImpl::notifyPendingReceivedCallback, - shared_from_this(), ResultOk, msg, callback)); - } else { - if (messages_.full()) { - lock.unlock(); - } + get_shared_this_ptr(), ResultOk, msg, callback)); + return; + } - if (messages_.push(msg) && messageListener_) { - listenerExecutor_->postWork( - std::bind(&MultiTopicsConsumerImpl::internalListener, shared_from_this(), consumer)); - } + if (incomingMessages_.full()) { + lock.unlock(); + } + + // add message to block queue. + // when messages queue is full, will block listener thread on ConsumerImpl, + // then will not send permits to broker, will broker stop push message. + incomingMessages_.push(msg); + incomingMessagesSize_.fetch_add(msg.getLength()); + + // try trigger pending batch messages + Lock batchOptionLock(batchReceiveOptionMutex_); + if (hasEnoughMessagesForBatchReceive()) { + ConsumerImplBase::notifyBatchPendingReceivedCallback(); + } + batchOptionLock.unlock(); + + if (messageListener_) { + listenerExecutor_->postWork( + std::bind(&MultiTopicsConsumerImpl::internalListener, get_shared_this_ptr(), consumer)); } } void MultiTopicsConsumerImpl::internalListener(Consumer consumer) { Message m; - messages_.pop(m); - unAckedMessageTrackerPtr_->add(m.getMessageId()); + incomingMessages_.pop(m); try { - messageListener_(Consumer(shared_from_this()), m); + messageListener_(Consumer(get_shared_this_ptr()), m); + messageProcessed(m); } catch (const std::exception& e) { LOG_ERROR("Exception thrown from listener of Partitioned Consumer" << e.what()); } @@ -487,9 +509,9 @@ Result MultiTopicsConsumerImpl::receive(Message& msg) { LOG_ERROR("Can not receive when a listener has been set"); return ResultInvalidConfiguration; } - messages_.pop(msg); + incomingMessages_.pop(msg); + messageProcessed(msg); - unAckedMessageTrackerPtr_->add(msg.getMessageId()); return ResultOk; } @@ -503,8 +525,8 @@ Result MultiTopicsConsumerImpl::receive(Message& msg, int timeout) { return ResultInvalidConfiguration; } - if (messages_.pop(msg, std::chrono::milliseconds(timeout))) { - unAckedMessageTrackerPtr_->add(msg.getMessageId()); + if (incomingMessages_.pop(msg, std::chrono::milliseconds(timeout))) { + messageProcessed(msg); return ResultOk; } else { if (state_ != Ready) { @@ -524,9 +546,9 @@ void MultiTopicsConsumerImpl::receiveAsync(ReceiveCallback& callback) { } Lock lock(pendingReceiveMutex_); - if (messages_.pop(msg, std::chrono::milliseconds(0))) { + if (incomingMessages_.pop(msg, std::chrono::milliseconds(0))) { lock.unlock(); - unAckedMessageTrackerPtr_->add(msg.getMessageId()); + messageProcessed(msg); callback(ResultOk, msg); } else { pendingReceives_.push(callback); @@ -536,14 +558,14 @@ void MultiTopicsConsumerImpl::receiveAsync(ReceiveCallback& callback) { void MultiTopicsConsumerImpl::failPendingReceiveCallback() { Message msg; - messages_.close(); + incomingMessages_.close(); Lock lock(pendingReceiveMutex_); while (!pendingReceives_.empty()) { ReceiveCallback callback = pendingReceives_.front(); pendingReceives_.pop(); listenerExecutor_->postWork(std::bind(&MultiTopicsConsumerImpl::notifyPendingReceivedCallback, - shared_from_this(), ResultAlreadyClosed, msg, callback)); + get_shared_this_ptr(), ResultAlreadyClosed, msg, callback)); } lock.unlock(); } @@ -649,7 +671,7 @@ void MultiTopicsConsumerImpl::redeliverUnacknowledgedMessages(const std::set(numberTopicPartitions_->load()); lock.unlock(); - auto self = shared_from_this(); + auto self = get_shared_this_ptr(); size_t i = 0; consumers_.forEachValue([&self, &latchPtr, &statsPtr, &i, callback](const ConsumerImplPtr& consumer) { size_t index = i++; @@ -750,7 +772,7 @@ uint64_t MultiTopicsConsumerImpl::getNumberOfConnectedConsumer() { } void MultiTopicsConsumerImpl::runPartitionUpdateTask() { partitionsUpdateTimer_->expires_from_now(partitionsUpdateInterval_); - std::weak_ptr weakSelf{shared_from_this()}; + std::weak_ptr weakSelf{get_shared_this_ptr()}; partitionsUpdateTimer_->async_wait([weakSelf](const boost::system::error_code& ec) { // If two requests call runPartitionUpdateTask at the same time, the timer will fail, and it // cannot continue at this time, and the request needs to be ignored. @@ -769,7 +791,7 @@ void MultiTopicsConsumerImpl::topicPartitionUpdate() { auto topicName = TopicName::get(item.first); auto currentNumPartitions = item.second; lookupServicePtr_->getPartitionMetadataAsync(topicName).addListener( - std::bind(&MultiTopicsConsumerImpl::handleGetPartitions, shared_from_this(), topicName, + std::bind(&MultiTopicsConsumerImpl::handleGetPartitions, get_shared_this_ptr(), topicName, std::placeholders::_1, std::placeholders::_2, currentNumPartitions)); } } @@ -810,7 +832,7 @@ void MultiTopicsConsumerImpl::subscribeSingleNewConsumer( std::shared_ptr> partitionsNeedCreate) { ConsumerConfiguration config = conf_.clone(); ExecutorServicePtr internalListenerExecutor = client_->getPartitionListenerExecutorProvider()->get(); - config.setMessageListener(std::bind(&MultiTopicsConsumerImpl::messageReceived, shared_from_this(), + config.setMessageListener(std::bind(&MultiTopicsConsumerImpl::messageReceived, get_shared_this_ptr(), std::placeholders::_1, std::placeholders::_2)); // Apply total limit of receiver queue size across partitions @@ -824,7 +846,7 @@ void MultiTopicsConsumerImpl::subscribeSingleNewConsumer( topicName->isPersistent(), internalListenerExecutor, true, Partitioned); consumer->getConsumerCreatedFuture().addListener( - std::bind(&MultiTopicsConsumerImpl::handleSingleConsumerCreated, shared_from_this(), + std::bind(&MultiTopicsConsumerImpl::handleSingleConsumerCreated, get_shared_this_ptr(), std::placeholders::_1, std::placeholders::_2, partitionsNeedCreate, topicSubResultPromise)); consumer->setPartitionIndex(partitionIndex); consumer->start(); @@ -832,3 +854,35 @@ void MultiTopicsConsumerImpl::subscribeSingleNewConsumer( LOG_INFO("Add Creating Consumer for - " << topicPartitionName << " - " << consumerStr_ << " consumerSize: " << consumers_.size()); } + +bool MultiTopicsConsumerImpl::hasEnoughMessagesForBatchReceive() const { + if (batchReceivePolicy_.getMaxNumMessages() <= 0 && batchReceivePolicy_.getMaxNumBytes() <= 0) { + return false; + } + return (batchReceivePolicy_.getMaxNumMessages() > 0 && + incomingMessages_.size() >= batchReceivePolicy_.getMaxNumMessages()) || + (batchReceivePolicy_.getMaxNumBytes() > 0 && + incomingMessagesSize_ >= batchReceivePolicy_.getMaxNumBytes()); +} + +void MultiTopicsConsumerImpl::notifyBatchPendingReceivedCallback(const BatchReceiveCallback& callback) { + auto messages = std::make_shared(batchReceivePolicy_.getMaxNumMessages(), + batchReceivePolicy_.getMaxNumBytes()); + Message peekMsg; + while (incomingMessages_.pop(peekMsg, std::chrono::milliseconds(0)) && messages->canAdd(peekMsg)) { + messageProcessed(peekMsg); + messages->add(peekMsg); + } + auto self = get_shared_this_ptr(); + listenerExecutor_->postWork( + [callback, messages, self]() { callback(ResultOk, messages->getMessageList()); }); +} + +void MultiTopicsConsumerImpl::messageProcessed(Message& msg) { + incomingMessagesSize_.fetch_sub(msg.getLength()); + unAckedMessageTrackerPtr_->add(msg.getMessageId()); +} + +std::shared_ptr MultiTopicsConsumerImpl::get_shared_this_ptr() { + return std::dynamic_pointer_cast(shared_from_this()); +} diff --git a/lib/MultiTopicsConsumerImpl.h b/lib/MultiTopicsConsumerImpl.h index 8769d59b..044f4173 100644 --- a/lib/MultiTopicsConsumerImpl.h +++ b/lib/MultiTopicsConsumerImpl.h @@ -38,17 +38,8 @@ namespace pulsar { typedef std::shared_ptr> ConsumerSubResultPromisePtr; class MultiTopicsConsumerImpl; -class MultiTopicsConsumerImpl : public ConsumerImplBase, - public std::enable_shared_from_this { +class MultiTopicsConsumerImpl : public ConsumerImplBase { public: - enum MultiTopicsConsumerState - { - Pending, - Ready, - Closing, - Closed, - Failed - }; MultiTopicsConsumerImpl(ClientImplPtr client, const std::vector& topics, const std::string& subscriptionName, TopicNamePtr topicName, const ConsumerConfiguration& conf, LookupServicePtr lookupServicePtr_); @@ -99,16 +90,14 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, const ClientImplPtr client_; const std::string subscriptionName_; std::string consumerStr_; - std::string topic_; const ConsumerConfiguration conf_; typedef SynchronizedHashMap ConsumerMap; ConsumerMap consumers_; std::map topicsPartitions_; mutable std::mutex mutex_; std::mutex pendingReceiveMutex_; - std::atomic state_{Pending}; - BlockingQueue messages_; - const ExecutorServicePtr listenerExecutor_; + BlockingQueue incomingMessages_; + std::atomic_int incomingMessagesSize_ = {0}; MessageListener messageListener_; DeadlineTimerPtr partitionsUpdateTimer_; boost::posix_time::time_duration partitionsUpdateInterval_; @@ -125,6 +114,7 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, unsigned int partitionIndex); void notifyResult(CloseCallback closeCallback); void messageReceived(Consumer consumer, const Message& msg); + void messageProcessed(Message& msg); void internalListener(Consumer consumer); void receiveMessages(); void failPendingReceiveCallback(); @@ -149,8 +139,12 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase, void subscribeSingleNewConsumer(int numPartitions, TopicNamePtr topicName, int partitionIndex, ConsumerSubResultPromisePtr topicSubResultPromise, std::shared_ptr> partitionsNeedCreate); + // impl consumer base virtual method + bool hasEnoughMessagesForBatchReceive() const override; + void notifyBatchPendingReceivedCallback(const BatchReceiveCallback& callback) override; private: + std::shared_ptr get_shared_this_ptr(); void setNegativeAcknowledgeEnabledForTesting(bool enabled) override; FRIEND_TEST(ConsumerTest, testMultiTopicsConsumerUnAckedMessageRedelivery); diff --git a/tests/BasicEndToEndTest.cc b/tests/BasicEndToEndTest.cc index 54a07bbc..d3e424e6 100644 --- a/tests/BasicEndToEndTest.cc +++ b/tests/BasicEndToEndTest.cc @@ -4098,3 +4098,184 @@ TEST(BasicEndToEndTest, testUnAckedMessageTrackerEnabledCumulativeAck) { consumer.close(); client.close(); } + +void testBatchReceive(bool multiConsumer) { + ClientConfiguration config; + Client client(lookupUrl); + + std::string uniqueChunk = unique_str(); + std::string topicName = "persistent://public/default/test-batch-receive" + uniqueChunk; + + if (multiConsumer) { + // call admin api to make it partitioned + std::string url = + adminUrl + "admin/v2/persistent/public/default/test-batch-receive" + uniqueChunk + "/partitions"; + int res = makePutRequest(url, "5"); + LOG_INFO("res = " << res); + ASSERT_FALSE(res != 204 && res != 409); + } + + std::string subName = "subscription-name"; + Producer producer; + + Promise producerPromise; + client.createProducerAsync(topicName, WaitForCallbackValue(producerPromise)); + Future producerFuture = producerPromise.getFuture(); + Result result = producerFuture.get(producer); + ASSERT_EQ(ResultOk, result); + + Consumer consumer; + ConsumerConfiguration consumerConfig; + // when receiver queue size > maxNumMessages, use receiver queue size. + consumerConfig.setBatchReceivePolicy(BatchReceivePolicy(1000, -1, -1)); + consumerConfig.setReceiverQueueSize(10); + consumerConfig.setProperty("consumer-name", "test-consumer-name"); + consumerConfig.setProperty("consumer-id", "test-consumer-id"); + Promise consumerPromise; + client.subscribeAsync(topicName, subName, consumerConfig, + WaitForCallbackValue(consumerPromise)); + Future consumerFuture = consumerPromise.getFuture(); + result = consumerFuture.get(consumer); + ASSERT_EQ(ResultOk, result); + + // sync batch receive test + std::string prefix = "batch-receive-msg"; + int numOfMessages = 10; + for (int i = 0; i < numOfMessages; i++) { + std::string messageContent = prefix + std::to_string(i); + Message msg = MessageBuilder().setContent(messageContent).build(); + producer.send(msg); + } + + Messages messages; + Result receive = consumer.batchReceive(messages); + ASSERT_EQ(receive, ResultOk); + ASSERT_EQ(messages.size(), numOfMessages); + + // async batch receive test + Latch latch(1); + BatchReceiveCallback batchReceiveCallback = [&latch, numOfMessages](Result result, Messages messages) { + ASSERT_EQ(result, ResultOk); + ASSERT_EQ(messages.size(), numOfMessages); + latch.countdown(); + }; + consumer.batchReceiveAsync(batchReceiveCallback); + for (int i = 0; i < numOfMessages; i++) { + std::string messageContent = prefix + std::to_string(i); + Message msg = MessageBuilder().setContent(messageContent).build(); + producer.send(msg); + } + ASSERT_TRUE(latch.wait(std::chrono::seconds(10))); + + producer.close(); + consumer.close(); + client.close(); +} + +TEST(BasicEndToEndTest, testBatchReceive) { testBatchReceive(false); } + +TEST(BasicEndToEndTest, testBatchReceiveWithMultiConsumer) { testBatchReceive(true); } + +void testBatchReceiveTimeout(bool multiConsumer) { + ClientConfiguration config; + Client client(lookupUrl); + std::string uniqueChunk = unique_str(); + std::string topicName = "persistent://public/default/test-batch-receive-timeout" + uniqueChunk; + + if (multiConsumer) { + // call admin api to make it partitioned + std::string url = adminUrl + "admin/v2/persistent/public/default/test-batch-receive-timeout" + + uniqueChunk + "/partitions"; + int res = makePutRequest(url, "5"); + LOG_INFO("res = " << res); + ASSERT_FALSE(res != 204 && res != 409); + } + + std::string subName = "subscription-name"; + Producer producer; + + Promise producerPromise; + client.createProducerAsync(topicName, WaitForCallbackValue(producerPromise)); + Future producerFuture = producerPromise.getFuture(); + Result result = producerFuture.get(producer); + ASSERT_EQ(ResultOk, result); + + Consumer consumer; + ConsumerConfiguration consumerConfig; + consumerConfig.setBatchReceivePolicy(BatchReceivePolicy(1000, -1, 1000)); + Promise consumerPromise; + client.subscribeAsync(topicName, subName, consumerConfig, + WaitForCallbackValue(consumerPromise)); + Future consumerFuture = consumerPromise.getFuture(); + result = consumerFuture.get(consumer); + ASSERT_EQ(ResultOk, result); + + std::string prefix = "batch-receive-msg"; + int numOfMessages = 10; + + for (int i = 0; i < numOfMessages; i++) { + std::string messageContent = prefix + std::to_string(i); + Message msg = MessageBuilder().setContent(messageContent).build(); + producer.send(msg); + } + + Latch latch(1); + BatchReceiveCallback batchReceiveCallback = [&latch, numOfMessages](Result result, Messages messages) { + ASSERT_EQ(result, ResultOk); + ASSERT_EQ(messages.size(), numOfMessages); + latch.countdown(); + }; + consumer.batchReceiveAsync(batchReceiveCallback); + ASSERT_TRUE(latch.wait(std::chrono::seconds(10))); + + producer.close(); + consumer.close(); + client.close(); +} + +TEST(BasicEndToEndTest, testBatchReceiveTimeout) { testBatchReceiveTimeout(false); } + +TEST(BasicEndToEndTest, testBatchReceiveTimeoutWithMultiConsumer) { testBatchReceiveTimeout(true); } + +void testBatchReceiveClose(bool multiConsumer) { + ClientConfiguration config; + Client client(lookupUrl); + + std::string uniqueChunk = unique_str(); + std::string topicName = "persistent://public/default/test-batch-receive-close" + uniqueChunk; + + if (multiConsumer) { + // call admin api to make it partitioned + std::string url = adminUrl + "admin/v2/persistent/public/default/test-batch-receive-close" + + uniqueChunk + "/partitions"; + int res = makePutRequest(url, "5"); + LOG_INFO("res = " << res); + ASSERT_FALSE(res != 204 && res != 409); + } + + std::string subName = "subscription-name"; + Consumer consumer; + ConsumerConfiguration consumerConfig; + consumerConfig.setBatchReceivePolicy(BatchReceivePolicy(1000, -1, 1000)); + Promise consumerPromise; + client.subscribeAsync(topicName, subName, consumerConfig, + WaitForCallbackValue(consumerPromise)); + Future consumerFuture = consumerPromise.getFuture(); + Result result = consumerFuture.get(consumer); + ASSERT_EQ(ResultOk, result); + + Latch latch(1); + BatchReceiveCallback batchReceiveCallback = [&latch](Result result, Messages messages) { + ASSERT_EQ(result, ResultAlreadyClosed); + latch.countdown(); + }; + consumer.batchReceiveAsync(batchReceiveCallback); + consumer.close(); + client.close(); + + ASSERT_TRUE(latch.wait(std::chrono::seconds(10))); +} + +TEST(BasicEndToEndTest, testBatchReceiveClose) { testBatchReceiveClose(false); } + +TEST(BasicEndToEndTest, testBatchReceiveCloseWithMultiConsumer) { testBatchReceiveClose(true); } diff --git a/tests/BatchReceivePolicyTest.cc b/tests/BatchReceivePolicyTest.cc new file mode 100644 index 00000000..ab9ffcc0 --- /dev/null +++ b/tests/BatchReceivePolicyTest.cc @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include + +using namespace pulsar; + +TEST(BatchReceivePolicyTest, testBatchReceivePolicy) { + ASSERT_THROW(BatchReceivePolicy(-1, -1, -1), std::invalid_argument); + + { + BatchReceivePolicy batchReceivePolicy; + ASSERT_EQ(batchReceivePolicy.getMaxNumMessages(), -1); + ASSERT_EQ(batchReceivePolicy.getMaxNumBytes(), 10 * 1024 * 1024); + ASSERT_EQ(batchReceivePolicy.getTimeoutMs(), 100); + } + + { + BatchReceivePolicy batchReceivePolicy(-1, -1, 123); + ASSERT_EQ(batchReceivePolicy.getMaxNumMessages(), -1); + ASSERT_EQ(batchReceivePolicy.getMaxNumBytes(), 10 * 1024 * 1024); + ASSERT_EQ(batchReceivePolicy.getTimeoutMs(), 123); + } +} diff --git a/tests/ConsumerConfigurationTest.cc b/tests/ConsumerConfigurationTest.cc index 24f541b5..20cd8f4b 100644 --- a/tests/ConsumerConfigurationTest.cc +++ b/tests/ConsumerConfigurationTest.cc @@ -61,6 +61,9 @@ TEST(ConsumerConfigurationTest, testDefaultConfig) { ASSERT_EQ(conf.getPriorityLevel(), 0); ASSERT_EQ(conf.getMaxPendingChunkedMessage(), 10); ASSERT_EQ(conf.isAutoAckOldestChunkedMessageOnQueueFull(), false); + ASSERT_EQ(conf.getBatchReceivePolicy().getMaxNumMessages(), -1); + ASSERT_EQ(conf.getBatchReceivePolicy().getMaxNumBytes(), 10 * 1024 * 1024); + ASSERT_EQ(conf.getBatchReceivePolicy().getTimeoutMs(), 100); } TEST(ConsumerConfigurationTest, testCustomConfig) { @@ -151,6 +154,11 @@ TEST(ConsumerConfigurationTest, testCustomConfig) { conf.setAutoAckOldestChunkedMessageOnQueueFull(true); ASSERT_TRUE(conf.isAutoAckOldestChunkedMessageOnQueueFull()); + + conf.setBatchReceivePolicy(BatchReceivePolicy(10, 10, 100)); + ASSERT_EQ(conf.getBatchReceivePolicy().getMaxNumMessages(), 10); + ASSERT_EQ(conf.getBatchReceivePolicy().getMaxNumBytes(), 10); + ASSERT_EQ(conf.getBatchReceivePolicy().getTimeoutMs(), 100); } TEST(ConsumerConfigurationTest, testReadCompactPersistentExclusive) { diff --git a/tests/MessagesImplTest.cc b/tests/MessagesImplTest.cc new file mode 100644 index 00000000..e963501a --- /dev/null +++ b/tests/MessagesImplTest.cc @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include "pulsar/MessageBuilder.h" + +using namespace pulsar; + +TEST(MessagesImplTest, testMessage) { + // 0. test not limits + { + MessagesImpl messages(-1, -1); + ASSERT_TRUE(messages.canAdd(Message())); + } + + // 1. test max number of messages. + { + Message msg = MessageBuilder().setContent("c").build(); + MessagesImpl messages(10, -1); + for (int i = 0; i < 10; i++) { + messages.add(msg); + } + ASSERT_FALSE(messages.canAdd(msg)); + ASSERT_EQ(messages.size(), 10); + ASSERT_THROW(messages.add(msg), std::invalid_argument); + messages.clear(); + ASSERT_TRUE(messages.canAdd(msg)); + ASSERT_EQ(messages.size(), 0); + } + + // 2. test max size of messages. + { + Message msg = MessageBuilder().setContent("c").build(); + MessagesImpl messages(-1, 10); + for (int i = 0; i < 10; i++) { + messages.add(msg); + } + ASSERT_FALSE(messages.canAdd(msg)); + ASSERT_EQ(messages.size(), 10); + ASSERT_THROW(messages.add(msg), std::invalid_argument); + messages.clear(); + ASSERT_TRUE(messages.canAdd(msg)); + ASSERT_EQ(messages.size(), 0); + } +} From 55b4bc9406f884c3a7a2a7287d098558c12d003b Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 20 Oct 2022 22:45:20 +0800 Subject: [PATCH 12/19] [fix] Fix memory leak caused by incorrect close and destruction (#54) Fixes https://github.com/apache/pulsar-client-cpp/issues/55 ### Motivation 1. When a producer or consumer is closed, the reference is still stored in `ClientImpl`. If a client kept creating producers or consumers, the memory usage would not reduce. 2. When the `HandlerBase::connection_` field is modified, the `removeProducer` or `removeConsumer` method is not called. Then these producers and consumers will be cached in the connection until the connection is closed. 3. The `PartitionedProducerImpl` and `MultiTopicsConsumerImpl` have cyclic references, when a `Producer` or `Consumer` instance goes out of the scope, the destructors are not called. When I used GDB to debug them, I found the reference counts were both greater than 1. ### Modifications Let's use "handlers" to represent "producers and consumers". 1. In `ClientImpl`, use `SynchronizedHashMap` to store references of handlers, as well as the `cleanupXXX` methods to remove a handler. 2. Add `HandlerBase::beforeConnectionChange` method, which is called before `connection_` is modified. Disallow the access to `connection_` from derived classes. 3. Avoid `shared_from_this()` being passed into callbacks in ASIO executors for `PartitionedProducerImpl` and `MultiTopicsConsumerImpl`. This PR also unifies the `shutdown` implementations for handlers and call `shutdown` in the destructors. 1. Cancel the timers 2. Unregister itself from `ClientImpl` and `ClientConnection` 3. Set the create future with `ResultAlreadyClosed` 4. Set the state to `Closed` It's called when: - the destructor is called - `closeAsync` is completed - `unsubscribeAsync` is completed with ResultOk ### Verifications `ShutdownTest` is added to verify the following cases: - a single topic - a partitioned topic (multiple topics) - a partitioned topic with regex subscription `testClose` verifies `shutdown` when `closeAsync` and `unsubscribeAsync` are called. `testDestructor` verifies `shutdown` when handlers go out of the scope and the destructors are called. --- .github/workflows/ci-pr-validation.yaml | 2 +- lib/ClientConnection.h | 2 +- lib/ClientImpl.cc | 89 +++++----- lib/ClientImpl.h | 13 +- lib/ConnectionPool.h | 4 +- lib/ConsumerImpl.cc | 125 ++++++-------- lib/ConsumerImpl.h | 5 +- lib/HandlerBase.cc | 24 ++- lib/HandlerBase.h | 20 ++- lib/MultiTopicsConsumerImpl.cc | 217 +++++++++++++++++------- lib/MultiTopicsConsumerImpl.h | 11 +- lib/PartitionedProducerImpl.cc | 83 ++++++--- lib/PartitionedProducerImpl.h | 5 +- lib/PatternMultiTopicsConsumerImpl.cc | 19 ++- lib/PatternMultiTopicsConsumerImpl.h | 1 + lib/PeriodicTask.cc | 5 +- lib/PeriodicTask.h | 2 +- lib/ProducerImpl.cc | 90 ++++------ lib/ProducerImpl.h | 5 +- lib/SynchronizedHashMap.h | 18 +- tests/ClientTest.cc | 28 ++- tests/PulsarFriend.h | 35 +++- tests/ShutdownTest.cc | 121 +++++++++++++ tests/WaitUtils.h | 43 ----- 24 files changed, 610 insertions(+), 357 deletions(-) create mode 100644 tests/ShutdownTest.cc delete mode 100644 tests/WaitUtils.h diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 14857084..bd47a812 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -68,7 +68,7 @@ jobs: run: ./pulsar-test-service-start.sh - name: Run unit tests - run: ./run-unit-tests.sh + run: RETRY_FAILED=3 ./run-unit-tests.sh - name: Stop Pulsar service run: ./pulsar-test-service-stop.sh diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 418a5831..8a48408e 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -314,7 +314,7 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this> PendingGetNamespaceTopicsMap; PendingGetNamespaceTopicsMap pendingGetNamespaceTopicsRequests_; - std::mutex mutex_; + mutable std::mutex mutex_; typedef std::unique_lock Lock; // Pending buffers to write on the socket diff --git a/lib/ClientImpl.cc b/lib/ClientImpl.cc index 29e92f3b..025727af 100644 --- a/lib/ClientImpl.cc +++ b/lib/ClientImpl.cc @@ -189,9 +189,15 @@ void ClientImpl::handleCreateProducer(const Result result, const LookupDataResul void ClientImpl::handleProducerCreated(Result result, ProducerImplBaseWeakPtr producerBaseWeakPtr, CreateProducerCallback callback, ProducerImplBasePtr producer) { if (result == ResultOk) { - Lock lock(mutex_); - producers_.push_back(producer); - lock.unlock(); + auto pair = producers_.emplace(producer.get(), producer); + if (!pair.second) { + auto existingProducer = pair.first->second.lock(); + LOG_ERROR("Unexpected existing producer at the same address: " + << pair.first->first << ", producer: " + << (existingProducer ? existingProducer->getProducerName() : "(null)")); + callback(ResultUnknownError, {}); + return; + } callback(result, Producer(producer)); } else { callback(result, {}); @@ -241,9 +247,18 @@ void ClientImpl::handleReaderMetadataLookup(const Result result, const LookupDat ConsumerImplBasePtr consumer = reader->getConsumer().lock(); auto self = shared_from_this(); reader->start(startMessageId, [this, self](const ConsumerImplBaseWeakPtr& weakConsumerPtr) { - Lock lock(mutex_); - consumers_.push_back(weakConsumerPtr); - lock.unlock(); + auto consumer = weakConsumerPtr.lock(); + if (consumer) { + auto pair = consumers_.emplace(consumer.get(), consumer); + if (!pair.second) { + auto existingConsumer = pair.first->second.lock(); + LOG_ERROR("Unexpected existing consumer at the same address: " + << pair.first->first + << ", consumer: " << (existingConsumer ? existingConsumer->getName() : "(null)")); + } + } else { + LOG_ERROR("Unexpected case: the consumer is somehow expired"); + } }); } @@ -397,9 +412,15 @@ void ClientImpl::handleSubscribe(const Result result, const LookupDataResultPtr void ClientImpl::handleConsumerCreated(Result result, ConsumerImplBaseWeakPtr consumerImplBaseWeakPtr, SubscribeCallback callback, ConsumerImplBasePtr consumer) { if (result == ResultOk) { - Lock lock(mutex_); - consumers_.push_back(consumer); - lock.unlock(); + auto pair = consumers_.emplace(consumer.get(), consumer); + if (!pair.second) { + auto existingConsumer = pair.first->second.lock(); + LOG_ERROR("Unexpected existing consumer at the same address: " + << pair.first->first + << ", consumer: " << (existingConsumer ? existingConsumer->getName() : "(null)")); + callback(ResultUnknownError, {}); + return; + } callback(result, Consumer(consumer)); } else { callback(result, {}); @@ -477,27 +498,26 @@ void ClientImpl::getPartitionsForTopicAsync(const std::string& topic, GetPartiti } void ClientImpl::closeAsync(CloseCallback callback) { - Lock lock(mutex_); - ProducersList producers(producers_); - ConsumersList consumers(consumers_); - - if (state_ != Open && callback) { - lock.unlock(); - callback(ResultAlreadyClosed); + if (state_ != Open) { + if (callback) { + callback(ResultAlreadyClosed); + } return; } // Set the state to Closing so that no producers could get added state_ = Closing; - lock.unlock(); memoryLimitController_.close(); + auto producers = producers_.move(); + auto consumers = consumers_.move(); + SharedInt numberOfOpenHandlers = std::make_shared(producers.size() + consumers.size()); LOG_INFO("Closing Pulsar client with " << producers.size() << " producers and " << consumers.size() << " consumers"); - for (ProducersList::iterator it = producers.begin(); it != producers.end(); ++it) { - ProducerImplBasePtr producer = it->lock(); + for (auto&& kv : producers) { + ProducerImplBasePtr producer = kv.second.lock(); if (producer && !producer->isClosed()) { producer->closeAsync(std::bind(&ClientImpl::handleClose, shared_from_this(), std::placeholders::_1, numberOfOpenHandlers, callback)); @@ -507,8 +527,8 @@ void ClientImpl::closeAsync(CloseCallback callback) { } } - for (ConsumersList::iterator it = consumers.begin(); it != consumers.end(); ++it) { - ConsumerImplBasePtr consumer = it->lock(); + for (auto&& kv : consumers) { + ConsumerImplBasePtr consumer = kv.second.lock(); if (consumer && !consumer->isClosed()) { consumer->closeAsync(std::bind(&ClientImpl::handleClose, shared_from_this(), std::placeholders::_1, numberOfOpenHandlers, callback)); @@ -562,23 +582,18 @@ void ClientImpl::handleClose(Result result, SharedInt numberOfOpenHandlers, Resu } void ClientImpl::shutdown() { - Lock lock(mutex_); - ProducersList producers; - ConsumersList consumers; + auto producers = producers_.move(); + auto consumers = consumers_.move(); - producers.swap(producers_); - consumers.swap(consumers_); - lock.unlock(); - - for (ProducersList::iterator it = producers.begin(); it != producers.end(); ++it) { - ProducerImplBasePtr producer = it->lock(); + for (auto&& kv : producers) { + ProducerImplBasePtr producer = kv.second.lock(); if (producer) { producer->shutdown(); } } - for (ConsumersList::iterator it = consumers.begin(); it != consumers.end(); ++it) { - ConsumerImplBasePtr consumer = it->lock(); + for (auto&& kv : consumers) { + ConsumerImplBasePtr consumer = kv.second.lock(); if (consumer) { consumer->shutdown(); } @@ -631,26 +646,24 @@ uint64_t ClientImpl::newRequestId() { } uint64_t ClientImpl::getNumberOfProducers() { - Lock lock(mutex_); uint64_t numberOfAliveProducers = 0; - for (const auto& producer : producers_) { + producers_.forEachValue([&numberOfAliveProducers](const ProducerImplBaseWeakPtr& producer) { const auto& producerImpl = producer.lock(); if (producerImpl) { numberOfAliveProducers += producerImpl->getNumberOfConnectedProducer(); } - } + }); return numberOfAliveProducers; } uint64_t ClientImpl::getNumberOfConsumers() { - Lock lock(mutex_); uint64_t numberOfAliveConsumers = 0; - for (const auto& consumer : consumers_) { + consumers_.forEachValue([&numberOfAliveConsumers](const ConsumerImplBaseWeakPtr& consumer) { const auto consumerImpl = consumer.lock(); if (consumerImpl) { numberOfAliveConsumers += consumerImpl->getNumberOfConnectedConsumer(); } - } + }); return numberOfAliveConsumers; } diff --git a/lib/ClientImpl.h b/lib/ClientImpl.h index e8e77082..50ddeffe 100644 --- a/lib/ClientImpl.h +++ b/lib/ClientImpl.h @@ -31,6 +31,7 @@ #include #include #include "ServiceNameResolver.h" +#include "SynchronizedHashMap.h" namespace pulsar { @@ -91,6 +92,11 @@ class ClientImpl : public std::enable_shared_from_this { ExecutorServiceProviderPtr getListenerExecutorProvider(); ExecutorServiceProviderPtr getPartitionListenerExecutorProvider(); LookupServicePtr getLookup(); + + void cleanupProducer(ProducerImplBase* address) { producers_.remove(address); } + + void cleanupConsumer(ConsumerImplBase* address) { consumers_.remove(address); } + friend class PulsarFriend; private: @@ -147,11 +153,8 @@ class ClientImpl : public std::enable_shared_from_this { uint64_t consumerIdGenerator_; uint64_t requestIdGenerator_; - typedef std::vector ProducersList; - ProducersList producers_; - - typedef std::vector ConsumersList; - ConsumersList consumers_; + SynchronizedHashMap producers_; + SynchronizedHashMap consumers_; std::atomic closingError; diff --git a/lib/ConnectionPool.h b/lib/ConnectionPool.h index 21d439e7..996df54e 100644 --- a/lib/ConnectionPool.h +++ b/lib/ConnectionPool.h @@ -74,10 +74,10 @@ class PULSAR_PUBLIC ConnectionPool { typedef std::map PoolMap; PoolMap pool_; bool poolConnections_; - std::mutex mutex_; + mutable std::mutex mutex_; std::atomic_bool closed_{false}; - friend class ConnectionPoolTest; + friend class PulsarFriend; }; } // namespace pulsar #endif //_PULSAR_CONNECTION_POOL_HEADER_ diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 54e346f0..7be5a6aa 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -104,7 +104,6 @@ ConsumerImpl::ConsumerImpl(const ClientImplPtr client, const std::string& topic, ConsumerImpl::~ConsumerImpl() { LOG_DEBUG(getName() << "~ConsumerImpl"); - incomingMessages_.clear(); if (state_ == Ready) { // this could happen at least in this condition: // consumer seek, caused reconnection, if consumer close happened before connection ready, @@ -121,6 +120,7 @@ ConsumerImpl::~ConsumerImpl() { LOG_INFO(getName() << "Closed consumer for race condition: " << consumerId_); } } + shutdown(); } void ConsumerImpl::setPartitionIndex(int partitionIndex) { partitionIndex_ = partitionIndex; } @@ -156,6 +156,8 @@ void ConsumerImpl::start() { ackGroupingTrackerPtr_->start(); } +void ConsumerImpl::beforeConnectionChange(ClientConnection& cnx) { cnx.removeConsumer(consumerId_); } + void ConsumerImpl::connectionOpened(const ClientConnectionPtr& cnx) { if (state_ == Closed) { LOG_DEBUG(getName() << "connectionOpened : Consumer is already closed"); @@ -220,7 +222,7 @@ void ConsumerImpl::handleCreateConsumer(const ClientConnectionPtr& cnx, Result r LOG_INFO(getName() << "Created consumer on broker " << cnx->cnxString()); { Lock lock(mutex_); - connection_ = cnx; + setCnx(cnx); incomingMessages_.clear(); state_ = Ready; backoff_.reset(); @@ -267,13 +269,24 @@ void ConsumerImpl::handleCreateConsumer(const ClientConnectionPtr& cnx, Result r } } -void ConsumerImpl::unsubscribeAsync(ResultCallback callback) { +void ConsumerImpl::unsubscribeAsync(ResultCallback originalCallback) { LOG_INFO(getName() << "Unsubscribing"); + auto callback = [this, originalCallback](Result result) { + if (result == ResultOk) { + shutdown(); + LOG_INFO(getName() << "Unsubscribed successfully"); + } else { + state_ = Ready; + LOG_WARN(getName() << "Failed to unsubscribe: " << result); + } + if (originalCallback) { + originalCallback(result); + } + }; + if (state_ != Ready) { callback(ResultAlreadyClosed); - LOG_ERROR(getName() << "Can not unsubscribe a closed subscription, please call subscribe again and " - "then call unsubscribe"); return; } @@ -286,9 +299,9 @@ void ConsumerImpl::unsubscribeAsync(ResultCallback callback) { lock.unlock(); int requestId = client->newRequestId(); SharedBuffer cmd = Commands::newUnsubscribe(consumerId_, requestId); + auto self = get_shared_this_ptr(); cnx->sendRequestWithId(cmd, requestId) - .addListener(std::bind(&ConsumerImpl::handleUnsubscribe, get_shared_this_ptr(), - std::placeholders::_1, callback)); + .addListener([self, callback](Result result, const ResponseData&) { callback(result); }); } else { Result result = ResultNotConnected; lock.unlock(); @@ -297,16 +310,6 @@ void ConsumerImpl::unsubscribeAsync(ResultCallback callback) { } } -void ConsumerImpl::handleUnsubscribe(Result result, ResultCallback callback) { - if (result == ResultOk) { - state_ = Closed; - LOG_INFO(getName() << "Unsubscribed successfully"); - } else { - LOG_WARN(getName() << "Failed to unsubscribe: " << strResult(result)); - } - callback(result); -} - Optional ConsumerImpl::processMessageChunk(const SharedBuffer& payload, const proto::MessageMetadata& metadata, const MessageId& messageId, @@ -990,20 +993,25 @@ void ConsumerImpl::negativeAcknowledge(const MessageId& messageId) { void ConsumerImpl::disconnectConsumer() { LOG_INFO("Broker notification of Closed consumer: " << consumerId_); - Lock lock(mutex_); - connection_.reset(); - lock.unlock(); + resetCnx(); scheduleReconnection(get_shared_this_ptr()); } -void ConsumerImpl::closeAsync(ResultCallback callback) { - // Keep a reference to ensure object is kept alive - ConsumerImplPtr ptr = get_shared_this_ptr(); +void ConsumerImpl::closeAsync(ResultCallback originalCallback) { + auto callback = [this, originalCallback](Result result) { + shutdown(); + if (result == ResultOk) { + LOG_INFO(getName() << "Closed consumer " << consumerId_); + } else { + LOG_WARN(getName() << "Failed to close consumer: " << result); + } + if (originalCallback) { + originalCallback(result); + } + }; if (state_ != Ready) { - if (callback) { - callback(ResultAlreadyClosed); - } + callback(ResultAlreadyClosed); return; } @@ -1018,66 +1026,40 @@ void ConsumerImpl::closeAsync(ResultCallback callback) { ClientConnectionPtr cnx = getCnx().lock(); if (!cnx) { - state_ = Closed; // If connection is gone, also the consumer is closed on the broker side - if (callback) { - callback(ResultOk); - } + callback(ResultOk); return; } ClientImplPtr client = client_.lock(); if (!client) { - state_ = Closed; // Client was already destroyed - if (callback) { - callback(ResultOk); - } + callback(ResultOk); return; } - int requestId = client->newRequestId(); - Future future = - cnx->sendRequestWithId(Commands::newCloseConsumer(consumerId_, requestId), requestId); - if (callback) { - // Pass the shared pointer "ptr" to the handler to prevent the object from being destroyed - future.addListener(std::bind(&ConsumerImpl::handleClose, get_shared_this_ptr(), std::placeholders::_1, - callback, ptr)); - } + cancelTimers(); - // fail pendingReceive callback - failPendingReceiveCallback(); - failPendingBatchReceiveCallback(); - - // cancel timer - batchReceiveTimer_->cancel(); -} - -void ConsumerImpl::handleClose(Result result, ResultCallback callback, ConsumerImplPtr consumer) { - if (result == ResultOk) { - state_ = Closed; - - ClientConnectionPtr cnx = getCnx().lock(); - if (cnx) { - cnx->removeConsumer(consumerId_); - } - - LOG_INFO(getName() << "Closed consumer " << consumerId_); - } else { - LOG_ERROR(getName() << "Failed to close consumer: " << result); - } - - if (callback) { - callback(result); - } + int requestId = client->newRequestId(); + auto self = get_shared_this_ptr(); + cnx->sendRequestWithId(Commands::newCloseConsumer(consumerId_, requestId), requestId) + .addListener([self, callback](Result result, const ResponseData&) { callback(result); }); } const std::string& ConsumerImpl::getName() const { return consumerStr_; } void ConsumerImpl::shutdown() { - state_ = Closed; - + incomingMessages_.clear(); + resetCnx(); + auto client = client_.lock(); + if (client) { + client->cleanupConsumer(this); + } + cancelTimers(); consumerCreatedPromise_.setFailed(ResultAlreadyClosed); + failPendingReceiveCallback(); + failPendingBatchReceiveCallback(); + state_ = Closed; } bool ConsumerImpl::isClosed() { return state_ == Closed; } @@ -1437,4 +1419,9 @@ std::shared_ptr ConsumerImpl::get_shared_this_ptr() { return std::dynamic_pointer_cast(shared_from_this()); } +void ConsumerImpl::cancelTimers() noexcept { + boost::system::error_code ec; + batchReceiveTimer_->cancel(ec); +} + } /* namespace pulsar */ diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 09d2c5cf..3aa632a7 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -84,7 +84,6 @@ class ConsumerImpl : public ConsumerImplBase { void activeConsumerChanged(bool isActive); inline proto::CommandSubscribe_SubType getSubType(); inline proto::CommandSubscribe_InitialPosition getInitialPosition(); - void handleUnsubscribe(Result result, ResultCallback callback); /** * Send individual ACK request of given message ID to broker. @@ -140,6 +139,7 @@ class ConsumerImpl : public ConsumerImplBase { virtual bool isReadCompacted(); virtual void hasMessageAvailableAsync(HasMessageAvailableCallback callback); virtual void getLastMessageIdAsync(BrokerGetLastMessageIdCallback callback); + void beforeConnectionChange(ClientConnection& cnx) override; protected: // overrided methods from HandlerBase @@ -156,7 +156,8 @@ class ConsumerImpl : public ConsumerImplBase { void internalConsumerChangeListener(bool isActive); - void handleClose(Result result, ResultCallback callback, ConsumerImplPtr consumer); + void cancelTimers() noexcept; + ConsumerStatsBasePtr consumerStatsBasePtr_; private: diff --git a/lib/HandlerBase.cc b/lib/HandlerBase.cc index 506207ea..1f4ce6e9 100644 --- a/lib/HandlerBase.cc +++ b/lib/HandlerBase.cc @@ -30,7 +30,6 @@ namespace pulsar { HandlerBase::HandlerBase(const ClientImplPtr& client, const std::string& topic, const Backoff& backoff) : client_(client), topic_(topic), - connection_(), executor_(client->getIOExecutorProvider()->get()), mutex_(), creationTimestamp_(TimeUtils::now()), @@ -50,14 +49,25 @@ void HandlerBase::start() { } } +ClientConnectionWeakPtr HandlerBase::getCnx() const { + Lock lock(connectionMutex_); + return connection_; +} + +void HandlerBase::setCnx(const ClientConnectionPtr& cnx) { + Lock lock(connectionMutex_); + auto previousCnx = connection_.lock(); + if (previousCnx) { + beforeConnectionChange(*previousCnx); + } + connection_ = cnx; +} + void HandlerBase::grabCnx() { - Lock lock(mutex_); - if (connection_.lock()) { - lock.unlock(); + if (getCnx().lock()) { LOG_INFO(getName() << "Ignoring reconnection request since we're already connected"); return; } - lock.unlock(); LOG_INFO(getName() << "Getting connection from pool"); ClientImplPtr client = client_.lock(); Future future = client->getConnection(topic_); @@ -96,14 +106,14 @@ void HandlerBase::handleDisconnection(Result result, ClientConnectionWeakPtr con State state = handler->state_; - ClientConnectionPtr currentConnection = handler->connection_.lock(); + ClientConnectionPtr currentConnection = handler->getCnx().lock(); if (currentConnection && connection.lock().get() != currentConnection.get()) { LOG_WARN(handler->getName() << "Ignoring connection closed since we are already attached to a newer connection"); return; } - handler->connection_.reset(); + handler->resetCnx(); if (result == ResultRetryable) { scheduleReconnection(handler); diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index 6fc3603d..6616ec40 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -44,11 +44,9 @@ class HandlerBase { void start(); - /* - * get method for derived class to access weak ptr to connection so that they - * have to check if they can get a shared_ptr out of it or not - */ - ClientConnectionWeakPtr getCnx() const { return connection_; } + ClientConnectionWeakPtr getCnx() const; + void setCnx(const ClientConnectionPtr& cnx); + void resetCnx() { setCnx(nullptr); } protected: /* @@ -65,6 +63,14 @@ class HandlerBase { * Should we retry in error that are transient */ bool isRetriableError(Result result); + + /** + * Do some cleanup work before changing `connection_` to `cnx`. + * + * @param cnx the current connection + */ + virtual void beforeConnectionChange(ClientConnection& cnx) = 0; + /* * connectionOpened will be implemented by derived class to receive notification */ @@ -86,7 +92,6 @@ class HandlerBase { protected: ClientImplWeakPtr client_; const std::string topic_; - ClientConnectionWeakPtr connection_; ExecutorServicePtr executor_; mutable std::mutex mutex_; std::mutex pendingReceiveMutex_; @@ -112,6 +117,9 @@ class HandlerBase { private: DeadlineTimerPtr timer_; + + mutable std::mutex connectionMutex_; + ClientConnectionWeakPtr connection_; friend class ClientConnection; friend class PulsarFriend; }; diff --git a/lib/MultiTopicsConsumerImpl.cc b/lib/MultiTopicsConsumerImpl.cc index 573c33d9..c54f8e8e 100644 --- a/lib/MultiTopicsConsumerImpl.cc +++ b/lib/MultiTopicsConsumerImpl.cc @@ -19,6 +19,7 @@ #include "MultiTopicsConsumerImpl.h" #include "MultiResultCallback.h" #include "MessagesImpl.h" +#include DECLARE_LOG_OBJECT() @@ -55,11 +56,11 @@ MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std } else { unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerDisabled()); } - auto partitionsUpdateInterval = static_cast(client_->conf().getPartitionsUpdateInterval()); + auto partitionsUpdateInterval = static_cast(client->conf().getPartitionsUpdateInterval()); if (partitionsUpdateInterval > 0) { partitionsUpdateTimer_ = listenerExecutor_->createDeadlineTimer(); partitionsUpdateInterval_ = boost::posix_time::seconds(partitionsUpdateInterval); - lookupServicePtr_ = client_->getLookup(); + lookupServicePtr_ = client->getLookup(); } state_ = Pending; @@ -83,10 +84,16 @@ void MultiTopicsConsumerImpl::start() { int topicsNumber = topics_.size(); std::shared_ptr> topicsNeedCreate = std::make_shared>(topicsNumber); // subscribe for each passed in topic + auto weakSelf = weak_from_this(); for (std::vector::const_iterator itr = topics_.begin(); itr != topics_.end(); itr++) { - subscribeOneTopicAsync(*itr).addListener(std::bind(&MultiTopicsConsumerImpl::handleOneTopicSubscribed, - get_shared_this_ptr(), std::placeholders::_1, - std::placeholders::_2, *itr, topicsNeedCreate)); + auto topic = *itr; + subscribeOneTopicAsync(topic).addListener( + [this, weakSelf, topic, topicsNeedCreate](Result result, const Consumer& consumer) { + auto self = weakSelf.lock(); + if (self) { + handleOneTopicSubscribed(result, consumer, topic, topicsNeedCreate); + } + }); } } @@ -111,9 +118,9 @@ void MultiTopicsConsumerImpl::handleOneTopicSubscribed(Result result, Consumer c } else { LOG_ERROR("Unable to create Consumer - " << consumerStr_ << " Error - " << result); // unsubscribed all of the successfully subscribed partitioned consumers - // It's safe to capture only this here, because the callback can be called only when this is valid - closeAsync( - [this](Result result) { multiTopicsConsumerCreatedPromise_.setFailed(failedResult.load()); }); + // `shutdown()`, which set multiTopicsConsumerCreatedPromise_ with `failedResult`, will be called + // when `closeAsync` completes. + closeAsync(nullptr); } } } @@ -164,10 +171,20 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(int numPartitions, TopicN ConsumerSubResultPromisePtr topicSubResultPromise) { std::shared_ptr consumer; ConsumerConfiguration config = conf_.clone(); - ExecutorServicePtr internalListenerExecutor = client_->getPartitionListenerExecutorProvider()->get(); + auto client = client_.lock(); + if (!client) { + topicSubResultPromise->setFailed(ResultAlreadyClosed); + return; + } + ExecutorServicePtr internalListenerExecutor = client->getPartitionListenerExecutorProvider()->get(); - config.setMessageListener(std::bind(&MultiTopicsConsumerImpl::messageReceived, get_shared_this_ptr(), - std::placeholders::_1, std::placeholders::_2)); + auto weakSelf = weak_from_this(); + config.setMessageListener([this, weakSelf](Consumer consumer, const Message& msg) { + auto self = weakSelf.lock(); + if (self) { + messageReceived(consumer, msg); + } + }); int partitions = numPartitions == 0 ? 1 : numPartitions; @@ -186,7 +203,7 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(int numPartitions, TopicN // non-partitioned topic if (numPartitions == 0) { // We don't have to add partition-n suffix - consumer = std::make_shared(client_, topicName->toString(), subscriptionName_, config, + consumer = std::make_shared(client, topicName->toString(), subscriptionName_, config, topicName->isPersistent(), internalListenerExecutor, true, NonPartitioned); consumer->getConsumerCreatedFuture().addListener(std::bind( @@ -199,7 +216,7 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(int numPartitions, TopicN } else { for (int i = 0; i < numPartitions; i++) { std::string topicPartitionName = topicName->getTopicPartitionName(i); - consumer = std::make_shared(client_, topicPartitionName, subscriptionName_, config, + consumer = std::make_shared(client, topicPartitionName, subscriptionName_, config, topicName->isPersistent(), internalListenerExecutor, true, Partitioned); consumer->getConsumerCreatedFuture().addListener(std::bind( @@ -244,12 +261,24 @@ void MultiTopicsConsumerImpl::handleSingleConsumerCreated( } } -void MultiTopicsConsumerImpl::unsubscribeAsync(ResultCallback callback) { +void MultiTopicsConsumerImpl::unsubscribeAsync(ResultCallback originalCallback) { LOG_INFO("[ Topics Consumer " << topic_ << "," << subscriptionName_ << "] Unsubscribing"); + auto callback = [this, originalCallback](Result result) { + if (result == ResultOk) { + shutdown(); + LOG_INFO(getName() << "Unsubscribed successfully"); + } else { + state_ = Ready; + LOG_WARN(getName() << "Failed to unsubscribe: " << result); + } + if (originalCallback) { + originalCallback(result); + } + }; + const auto state = state_.load(); if (state == Closing || state == Closed) { - LOG_INFO(consumerStr_ << " already closed"); callback(ResultAlreadyClosed); return; } @@ -284,12 +313,9 @@ void MultiTopicsConsumerImpl::handleUnsubscribedAsync(Result result, if (consumerUnsubed->load() == numberTopicPartitions_->load()) { LOG_DEBUG("Unsubscribed all of the partition consumer for TopicsConsumer. - " << consumerStr_); - consumers_.clear(); - topicsPartitions_.clear(); - unAckedMessageTrackerPtr_->clear(); - Result result1 = (state_ != Failed) ? ResultOk : ResultUnknownError; - state_ = Closed; + // The `callback` is a wrapper of user provided callback, it's not null and will call `shutdown()` if + // unsubscribe succeeds. callback(result1); return; } @@ -376,20 +402,27 @@ void MultiTopicsConsumerImpl::handleOneTopicUnsubscribedAsync( } } -void MultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { +void MultiTopicsConsumerImpl::closeAsync(ResultCallback originalCallback) { + auto callback = [this, originalCallback](Result result) { + shutdown(); + if (result != ResultOk) { + LOG_WARN(getName() << "Failed to close consumer: " << result); + } + if (originalCallback) { + originalCallback(result); + } + }; const auto state = state_.load(); if (state == Closing || state == Closed) { - LOG_ERROR("TopicsConsumer already closed " - << " topic" << topic_ << " consumer - " << consumerStr_); - if (callback) { - callback(ResultAlreadyClosed); - } + callback(ResultAlreadyClosed); return; } state_ = Closing; - std::weak_ptr weakSelf{get_shared_this_ptr()}; + cancelTimers(); + + auto weakSelf = weak_from_this(); int numConsumers = 0; consumers_.clear( [this, weakSelf, &numConsumers, callback](const std::string& name, const ConsumerImplPtr& consumer) { @@ -418,27 +451,14 @@ void MultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { } // closed all consumers if (numConsumersLeft == 0) { - incomingMessages_.clear(); - topicsPartitions_.clear(); - unAckedMessageTrackerPtr_->clear(); - - if (state_ != Failed) { - state_ = Closed; - } - - if (callback) { - callback(result); - } + callback(result); } }); }); if (numConsumers == 0) { LOG_DEBUG("TopicsConsumer have no consumers to close " << " topic" << topic_ << " subscription - " << subscriptionName_); - state_ = Closed; - if (callback) { - callback(ResultAlreadyClosed); - } + callback(ResultAlreadyClosed); return; } @@ -461,8 +481,13 @@ void MultiTopicsConsumerImpl::messageReceived(Consumer consumer, const Message& ReceiveCallback callback = pendingReceives_.front(); pendingReceives_.pop(); lock.unlock(); - listenerExecutor_->postWork(std::bind(&MultiTopicsConsumerImpl::notifyPendingReceivedCallback, - get_shared_this_ptr(), ResultOk, msg, callback)); + auto weakSelf = weak_from_this(); + listenerExecutor_->postWork([this, weakSelf, msg, callback]() { + auto self = weakSelf.lock(); + if (self) { + notifyPendingReceivedCallback(ResultOk, msg, callback); + } + }); return; } @@ -564,13 +589,18 @@ void MultiTopicsConsumerImpl::failPendingReceiveCallback() { while (!pendingReceives_.empty()) { ReceiveCallback callback = pendingReceives_.front(); pendingReceives_.pop(); - listenerExecutor_->postWork(std::bind(&MultiTopicsConsumerImpl::notifyPendingReceivedCallback, - get_shared_this_ptr(), ResultAlreadyClosed, msg, callback)); + auto weakSelf = weak_from_this(); + listenerExecutor_->postWork([this, weakSelf, msg, callback]() { + auto self = weakSelf.lock(); + if (self) { + notifyPendingReceivedCallback(ResultAlreadyClosed, msg, callback); + } + }); } lock.unlock(); } -void MultiTopicsConsumerImpl::notifyPendingReceivedCallback(Result result, Message& msg, +void MultiTopicsConsumerImpl::notifyPendingReceivedCallback(Result result, const Message& msg, const ReceiveCallback& callback) { if (result == ResultOk) { unAckedMessageTrackerPtr_->add(msg.getMessageId()); @@ -609,7 +639,7 @@ void MultiTopicsConsumerImpl::negativeAcknowledge(const MessageId& msgId) { } } -MultiTopicsConsumerImpl::~MultiTopicsConsumerImpl() {} +MultiTopicsConsumerImpl::~MultiTopicsConsumerImpl() { shutdown(); } Future MultiTopicsConsumerImpl::getConsumerCreatedFuture() { return multiTopicsConsumerCreatedPromise_.getFuture(); @@ -620,7 +650,24 @@ const std::string& MultiTopicsConsumerImpl::getTopic() const { return topic_; } const std::string& MultiTopicsConsumerImpl::getName() const { return consumerStr_; } -void MultiTopicsConsumerImpl::shutdown() {} +void MultiTopicsConsumerImpl::shutdown() { + cancelTimers(); + incomingMessages_.clear(); + topicsPartitions_.clear(); + unAckedMessageTrackerPtr_->clear(); + auto client = client_.lock(); + if (client) { + client->cleanupConsumer(this); + } + consumers_.clear(); + topicsPartitions_.clear(); + if (failedResult != ResultOk) { + multiTopicsConsumerCreatedPromise_.setFailed(failedResult); + } else { + multiTopicsConsumerCreatedPromise_.setFailed(ResultAlreadyClosed); + } + state_ = Closed; +} bool MultiTopicsConsumerImpl::isClosed() { return state_ == Closed; } @@ -684,13 +731,16 @@ void MultiTopicsConsumerImpl::getBrokerConsumerStatsAsync(BrokerConsumerStatsCal LatchPtr latchPtr = std::make_shared(numberTopicPartitions_->load()); lock.unlock(); - auto self = get_shared_this_ptr(); size_t i = 0; - consumers_.forEachValue([&self, &latchPtr, &statsPtr, &i, callback](const ConsumerImplPtr& consumer) { + consumers_.forEachValue([this, &latchPtr, &statsPtr, &i, callback](const ConsumerImplPtr& consumer) { size_t index = i++; + auto weakSelf = weak_from_this(); consumer->getBrokerConsumerStatsAsync( - [self, latchPtr, statsPtr, index, callback](Result result, BrokerConsumerStats stats) { - self->handleGetConsumerStats(result, stats, latchPtr, statsPtr, index, callback); + [this, weakSelf, latchPtr, statsPtr, index, callback](Result result, BrokerConsumerStats stats) { + auto self = weakSelf.lock(); + if (self) { + handleGetConsumerStats(result, stats, latchPtr, statsPtr, index, callback); + } }); }); } @@ -772,7 +822,7 @@ uint64_t MultiTopicsConsumerImpl::getNumberOfConnectedConsumer() { } void MultiTopicsConsumerImpl::runPartitionUpdateTask() { partitionsUpdateTimer_->expires_from_now(partitionsUpdateInterval_); - std::weak_ptr weakSelf{get_shared_this_ptr()}; + auto weakSelf = weak_from_this(); partitionsUpdateTimer_->async_wait([weakSelf](const boost::system::error_code& ec) { // If two requests call runPartitionUpdateTask at the same time, the timer will fail, and it // cannot continue at this time, and the request needs to be ignored. @@ -790,9 +840,15 @@ void MultiTopicsConsumerImpl::topicPartitionUpdate() { for (const auto& item : topicsPartitions) { auto topicName = TopicName::get(item.first); auto currentNumPartitions = item.second; + auto weakSelf = weak_from_this(); lookupServicePtr_->getPartitionMetadataAsync(topicName).addListener( - std::bind(&MultiTopicsConsumerImpl::handleGetPartitions, get_shared_this_ptr(), topicName, - std::placeholders::_1, std::placeholders::_2, currentNumPartitions)); + [this, weakSelf, topicName, currentNumPartitions](Result result, + const LookupDataResultPtr& lookupDataResult) { + auto self = weakSelf.lock(); + if (self) { + this->handleGetPartitions(topicName, result, lookupDataResult, currentNumPartitions); + } + }); } } void MultiTopicsConsumerImpl::handleGetPartitions(TopicNamePtr topicName, Result result, @@ -831,9 +887,19 @@ void MultiTopicsConsumerImpl::subscribeSingleNewConsumer( ConsumerSubResultPromisePtr topicSubResultPromise, std::shared_ptr> partitionsNeedCreate) { ConsumerConfiguration config = conf_.clone(); - ExecutorServicePtr internalListenerExecutor = client_->getPartitionListenerExecutorProvider()->get(); - config.setMessageListener(std::bind(&MultiTopicsConsumerImpl::messageReceived, get_shared_this_ptr(), - std::placeholders::_1, std::placeholders::_2)); + auto client = client_.lock(); + if (!client) { + topicSubResultPromise->setFailed(ResultAlreadyClosed); + return; + } + ExecutorServicePtr internalListenerExecutor = client->getPartitionListenerExecutorProvider()->get(); + auto weakSelf = weak_from_this(); + config.setMessageListener([this, weakSelf](Consumer consumer, const Message& msg) { + auto self = weakSelf.lock(); + if (self) { + messageReceived(consumer, msg); + } + }); // Apply total limit of receiver queue size across partitions config.setReceiverQueueSize( @@ -842,12 +908,18 @@ void MultiTopicsConsumerImpl::subscribeSingleNewConsumer( std::string topicPartitionName = topicName->getTopicPartitionName(partitionIndex); - auto consumer = std::make_shared(client_, topicPartitionName, subscriptionName_, config, + auto consumer = std::make_shared(client, topicPartitionName, subscriptionName_, config, topicName->isPersistent(), internalListenerExecutor, true, Partitioned); consumer->getConsumerCreatedFuture().addListener( - std::bind(&MultiTopicsConsumerImpl::handleSingleConsumerCreated, get_shared_this_ptr(), - std::placeholders::_1, std::placeholders::_2, partitionsNeedCreate, topicSubResultPromise)); + [this, weakSelf, partitionsNeedCreate, topicSubResultPromise]( + Result result, const ConsumerImplBaseWeakPtr& consumerImplBaseWeakPtr) { + auto self = weakSelf.lock(); + if (self) { + handleSingleConsumerCreated(result, consumerImplBaseWeakPtr, partitionsNeedCreate, + topicSubResultPromise); + } + }); consumer->setPartitionIndex(partitionIndex); consumer->start(); consumers_.emplace(topicPartitionName, consumer); @@ -873,9 +945,13 @@ void MultiTopicsConsumerImpl::notifyBatchPendingReceivedCallback(const BatchRece messageProcessed(peekMsg); messages->add(peekMsg); } - auto self = get_shared_this_ptr(); - listenerExecutor_->postWork( - [callback, messages, self]() { callback(ResultOk, messages->getMessageList()); }); + auto weakSelf = weak_from_this(); + listenerExecutor_->postWork([weakSelf, callback, messages]() { + auto self = weakSelf.lock(); + if (self) { + callback(ResultOk, messages->getMessageList()); + } + }); } void MultiTopicsConsumerImpl::messageProcessed(Message& msg) { @@ -886,3 +962,14 @@ void MultiTopicsConsumerImpl::messageProcessed(Message& msg) { std::shared_ptr MultiTopicsConsumerImpl::get_shared_this_ptr() { return std::dynamic_pointer_cast(shared_from_this()); } + +void MultiTopicsConsumerImpl::beforeConnectionChange(ClientConnection& cnx) { + throw std::runtime_error("The connection_ field should not be modified for a MultiTopicsConsumerImpl"); +} + +void MultiTopicsConsumerImpl::cancelTimers() noexcept { + if (partitionsUpdateTimer_) { + boost::system::error_code ec; + partitionsUpdateTimer_->cancel(ec); + } +} diff --git a/lib/MultiTopicsConsumerImpl.h b/lib/MultiTopicsConsumerImpl.h index 044f4173..7c83da9d 100644 --- a/lib/MultiTopicsConsumerImpl.h +++ b/lib/MultiTopicsConsumerImpl.h @@ -87,7 +87,7 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase { Future subscribeOneTopicAsync(const std::string& topic); protected: - const ClientImplPtr client_; + const ClientImplWeakPtr client_; const std::string subscriptionName_; std::string consumerStr_; const ConsumerConfiguration conf_; @@ -118,7 +118,8 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase { void internalListener(Consumer consumer); void receiveMessages(); void failPendingReceiveCallback(); - void notifyPendingReceivedCallback(Result result, Message& message, const ReceiveCallback& callback); + void notifyPendingReceivedCallback(Result result, const Message& message, + const ReceiveCallback& callback); void handleOneTopicSubscribed(Result result, Consumer consumer, const std::string& topic, std::shared_ptr> topicsNeedCreate); @@ -142,10 +143,16 @@ class MultiTopicsConsumerImpl : public ConsumerImplBase { // impl consumer base virtual method bool hasEnoughMessagesForBatchReceive() const override; void notifyBatchPendingReceivedCallback(const BatchReceiveCallback& callback) override; + void beforeConnectionChange(ClientConnection& cnx) override; private: std::shared_ptr get_shared_this_ptr(); void setNegativeAcknowledgeEnabledForTesting(bool enabled) override; + void cancelTimers() noexcept; + + std::weak_ptr weak_from_this() noexcept { + return std::static_pointer_cast(shared_from_this()); + } FRIEND_TEST(ConsumerTest, testMultiTopicsConsumerUnAckedMessageRedelivery); FRIEND_TEST(ConsumerTest, testPartitionedConsumerUnAckedMessageRedelivery); diff --git a/lib/PartitionedProducerImpl.cc b/lib/PartitionedProducerImpl.cc index 469ecc9e..3d383ffb 100644 --- a/lib/PartitionedProducerImpl.cc +++ b/lib/PartitionedProducerImpl.cc @@ -46,12 +46,12 @@ PartitionedProducerImpl::PartitionedProducerImpl(ClientImplPtr client, const Top (int)(config.getMaxPendingMessagesAcrossPartitions() / numPartitions)); conf_.setMaxPendingMessages(maxPendingMessagesPerPartition); - auto partitionsUpdateInterval = static_cast(client_->conf().getPartitionsUpdateInterval()); + auto partitionsUpdateInterval = static_cast(client->conf().getPartitionsUpdateInterval()); if (partitionsUpdateInterval > 0) { - listenerExecutor_ = client_->getListenerExecutorProvider()->get(); + listenerExecutor_ = client->getListenerExecutorProvider()->get(); partitionsUpdateTimer_ = listenerExecutor_->createDeadlineTimer(); partitionsUpdateInterval_ = boost::posix_time::seconds(partitionsUpdateInterval); - lookupServicePtr_ = client_->getLookup(); + lookupServicePtr_ = client->getLookup(); } } @@ -71,7 +71,7 @@ MessageRoutingPolicyPtr PartitionedProducerImpl::getMessageRouter() { } } -PartitionedProducerImpl::~PartitionedProducerImpl() {} +PartitionedProducerImpl::~PartitionedProducerImpl() { shutdown(); } // override const std::string& PartitionedProducerImpl::getTopic() const { return topic_; } @@ -86,7 +86,11 @@ unsigned int PartitionedProducerImpl::getNumPartitionsWithLock() const { ProducerImplPtr PartitionedProducerImpl::newInternalProducer(unsigned int partition, bool lazy) { using namespace std::placeholders; - auto producer = std::make_shared(client_, *topicName_, conf_, partition); + auto client = client_.lock(); + auto producer = std::make_shared(client, *topicName_, conf_, partition); + if (!client) { + return producer; + } if (lazy) { createLazyPartitionProducer(partition); @@ -211,7 +215,15 @@ void PartitionedProducerImpl::sendAsync(const Message& msg, SendCallback callbac } // override -void PartitionedProducerImpl::shutdown() { state_ = Closed; } +void PartitionedProducerImpl::shutdown() { + cancelTimers(); + auto client = client_.lock(); + if (client) { + client->cleanupProducer(this); + } + partitionedProducerCreatedPromise_.setFailed(ResultAlreadyClosed); + state_ = Closed; +} const std::string& PartitionedProducerImpl::getProducerName() const { Lock producersLock(producersMutex_); @@ -239,11 +251,25 @@ int64_t PartitionedProducerImpl::getLastSequenceId() const { * if createProducerCallback is set, it means the closeAsync is called from CreateProducer API which failed to * create one or many producers for partitions. So, we have to notify with ERROR on createProducerFailure */ -void PartitionedProducerImpl::closeAsync(CloseCallback closeCallback) { - if (state_ == Closing || state_ == Closed) { +void PartitionedProducerImpl::closeAsync(CloseCallback originalCallback) { + auto closeCallback = [this, originalCallback](Result result) { + if (result == ResultOk) { + shutdown(); + } + if (originalCallback) { + originalCallback(result); + } + }; + if (state_ == Closed) { + closeCallback(ResultAlreadyClosed); + return; + } + State expectedState = Ready; + if (!state_.compare_exchange_strong(expectedState, Closing)) { return; } - state_ = Closing; + + cancelTimers(); unsigned int producerAlreadyClosed = 0; @@ -271,12 +297,12 @@ void PartitionedProducerImpl::closeAsync(CloseCallback closeCallback) { * c. If closeAsync called due to failure in creating just one sub producer then state is set by * handleSinglePartitionProducerCreated */ - if (producerAlreadyClosed == numProducers && closeCallback) { - state_ = Closed; + if (producerAlreadyClosed == numProducers) { closeCallback(ResultOk); } } +// `callback` is a wrapper of user provided callback, it's not null and will call `shutdown()` void PartitionedProducerImpl::handleSinglePartitionProducerClose(Result result, const unsigned int partitionIndex, CloseCallback callback) { @@ -285,11 +311,9 @@ void PartitionedProducerImpl::handleSinglePartitionProducerClose(Result result, return; } if (result != ResultOk) { - state_ = Failed; LOG_ERROR("Closing the producer failed for partition - " << partitionIndex); - if (callback) { - callback(result); - } + callback(result); + state_ = Failed; return; } assert(partitionIndex < getNumPartitionsWithLock()); @@ -298,16 +322,13 @@ void PartitionedProducerImpl::handleSinglePartitionProducerClose(Result result, } // closed all successfully if (!numProducersCreated_) { - state_ = Closed; // set the producerCreatedPromise to failure, if client called // closeAsync and it's not failure to create producer, the promise // is set second time here, first time it was successful. So check // if there's any adverse effect of setting it again. It should not // be but must check. MUSTCHECK changeme partitionedProducerCreatedPromise_.setFailed(ResultUnknownError); - if (callback) { - callback(result); - } + callback(result); return; } } @@ -371,15 +392,26 @@ void PartitionedProducerImpl::flushAsync(FlushCallback callback) { } void PartitionedProducerImpl::runPartitionUpdateTask() { + auto weakSelf = weak_from_this(); partitionsUpdateTimer_->expires_from_now(partitionsUpdateInterval_); - partitionsUpdateTimer_->async_wait( - std::bind(&PartitionedProducerImpl::getPartitionMetadata, shared_from_this())); + partitionsUpdateTimer_->async_wait([weakSelf](const boost::system::error_code& ec) { + auto self = weakSelf.lock(); + if (self) { + self->getPartitionMetadata(); + } + }); } void PartitionedProducerImpl::getPartitionMetadata() { using namespace std::placeholders; + auto weakSelf = weak_from_this(); lookupServicePtr_->getPartitionMetadataAsync(topicName_) - .addListener(std::bind(&PartitionedProducerImpl::handleGetPartitions, shared_from_this(), _1, _2)); + .addListener([weakSelf](Result result, const LookupDataResultPtr& lookupDataResult) { + auto self = weakSelf.lock(); + if (self) { + self->handleGetPartitions(result, lookupDataResult); + } + }); } void PartitionedProducerImpl::handleGetPartitions(Result result, @@ -446,4 +478,11 @@ uint64_t PartitionedProducerImpl::getNumberOfConnectedProducer() { return numberOfConnectedProducer; } +void PartitionedProducerImpl::cancelTimers() noexcept { + if (partitionsUpdateTimer_) { + boost::system::error_code ec; + partitionsUpdateTimer_->cancel(ec); + } +} + } // namespace pulsar diff --git a/lib/PartitionedProducerImpl.h b/lib/PartitionedProducerImpl.h index 0a8c10e2..cc7a4e00 100644 --- a/lib/PartitionedProducerImpl.h +++ b/lib/PartitionedProducerImpl.h @@ -73,10 +73,12 @@ class PartitionedProducerImpl : public ProducerImplBase, void notifyResult(CloseCallback closeCallback); + std::weak_ptr weak_from_this() noexcept { return shared_from_this(); } + friend class PulsarFriend; private: - const ClientImplPtr client_; + ClientImplWeakPtr client_; const TopicNamePtr topicName_; const std::string topic_; @@ -119,6 +121,7 @@ class PartitionedProducerImpl : public ProducerImplBase, void runPartitionUpdateTask(); void getPartitionMetadata(); void handleGetPartitions(const Result result, const LookupDataResultPtr& partitionMetadata); + void cancelTimers() noexcept; }; } // namespace pulsar diff --git a/lib/PatternMultiTopicsConsumerImpl.cc b/lib/PatternMultiTopicsConsumerImpl.cc index 79ed1969..8014078d 100644 --- a/lib/PatternMultiTopicsConsumerImpl.cc +++ b/lib/PatternMultiTopicsConsumerImpl.cc @@ -32,7 +32,7 @@ PatternMultiTopicsConsumerImpl::PatternMultiTopicsConsumerImpl(ClientImplPtr cli lookupServicePtr_), patternString_(pattern), pattern_(PULSAR_REGEX_NAMESPACE::regex(pattern)), - autoDiscoveryTimer_(), + autoDiscoveryTimer_(client->getIOExecutorProvider()->get()->createDeadlineTimer()), autoDiscoveryRunning_(false) { namespaceName_ = TopicName::get(pattern)->getNamespaceName(); } @@ -215,9 +215,7 @@ void PatternMultiTopicsConsumerImpl::start() { LOG_DEBUG("PatternMultiTopicsConsumerImpl start autoDiscoveryTimer_."); - // Init autoDiscoveryTimer task only once, wait for the timeout to happen - if (!autoDiscoveryTimer_ && conf_.getPatternAutoDiscoveryPeriod() > 0) { - autoDiscoveryTimer_ = client_->getIOExecutorProvider()->get()->createDeadlineTimer(); + if (conf_.getPatternAutoDiscoveryPeriod() > 0) { autoDiscoveryTimer_->expires_from_now(seconds(conf_.getPatternAutoDiscoveryPeriod())); autoDiscoveryTimer_->async_wait( std::bind(&PatternMultiTopicsConsumerImpl::autoDiscoveryTimerTask, this, std::placeholders::_1)); @@ -225,13 +223,16 @@ void PatternMultiTopicsConsumerImpl::start() { } void PatternMultiTopicsConsumerImpl::shutdown() { - Lock lock(mutex_); - state_ = Closed; - autoDiscoveryTimer_->cancel(); - multiTopicsConsumerCreatedPromise_.setFailed(ResultAlreadyClosed); + cancelTimers(); + MultiTopicsConsumerImpl::shutdown(); } void PatternMultiTopicsConsumerImpl::closeAsync(ResultCallback callback) { + cancelTimers(); MultiTopicsConsumerImpl::closeAsync(callback); - autoDiscoveryTimer_->cancel(); +} + +void PatternMultiTopicsConsumerImpl::cancelTimers() noexcept { + boost::system::error_code ec; + autoDiscoveryTimer_->cancel(ec); } diff --git a/lib/PatternMultiTopicsConsumerImpl.h b/lib/PatternMultiTopicsConsumerImpl.h index 408d68e3..448f2e39 100644 --- a/lib/PatternMultiTopicsConsumerImpl.h +++ b/lib/PatternMultiTopicsConsumerImpl.h @@ -72,6 +72,7 @@ class PatternMultiTopicsConsumerImpl : public MultiTopicsConsumerImpl { bool autoDiscoveryRunning_; NamespaceNamePtr namespaceName_; + void cancelTimers() noexcept; void resetAutoDiscoveryTimer(); void timerGetTopicsOfNamespace(const Result result, const NamespaceTopicsPtr topics); void onTopicsAdded(NamespaceTopicsPtr addedTopics, ResultCallback callback); diff --git a/lib/PeriodicTask.cc b/lib/PeriodicTask.cc index 4e91ef5f..65bdf234 100644 --- a/lib/PeriodicTask.cc +++ b/lib/PeriodicTask.cc @@ -38,12 +38,13 @@ void PeriodicTask::start() { } } -void PeriodicTask::stop() { +void PeriodicTask::stop() noexcept { State state = Ready; if (!state_.compare_exchange_strong(state, Closing)) { return; } - timer_.cancel(); + ErrorCode ec; + timer_.cancel(ec); state_ = Pending; } diff --git a/lib/PeriodicTask.h b/lib/PeriodicTask.h index 57d07348..159c86a8 100644 --- a/lib/PeriodicTask.h +++ b/lib/PeriodicTask.h @@ -55,7 +55,7 @@ class PeriodicTask : public std::enable_shared_from_this { void start(); - void stop(); + void stop() noexcept; void setCallback(CallbackType callback) noexcept { callback_ = callback; } diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 20133c50..e228c836 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -109,7 +109,7 @@ ProducerImpl::ProducerImpl(ClientImplPtr client, const TopicName& topicName, ProducerImpl::~ProducerImpl() { LOG_DEBUG(getName() << "~ProducerImpl"); - cancelTimers(); + shutdown(); printStats(); if (state_ == Ready || state_ == Pending) { LOG_WARN(getName() << "Destroyed producer which was not properly closed"); @@ -124,6 +124,10 @@ int64_t ProducerImpl::getLastSequenceId() const { return lastSequenceIdPublished const std::string& ProducerImpl::getSchemaVersion() const { return schemaVersion_; } +void ProducerImpl::beforeConnectionChange(ClientConnection& connection) { + connection.removeProducer(producerId_); +} + void ProducerImpl::connectionOpened(const ClientConnectionPtr& cnx) { if (state_ == Closed) { LOG_DEBUG(getName() << "connectionOpened : Producer is already closed"); @@ -185,7 +189,7 @@ void ProducerImpl::handleCreateProducer(const ClientConnectionPtr& cnx, Result r msgSequenceGenerator_ = lastSequenceIdPublished_ + 1; } resendMessages(cnx); - connection_ = cnx; + setCnx(cnx); state_ = Ready; backoff_.reset(); lock.unlock(); @@ -645,7 +649,19 @@ void ProducerImpl::printStats() { } } -void ProducerImpl::closeAsync(CloseCallback callback) { +void ProducerImpl::closeAsync(CloseCallback originalCallback) { + auto callback = [this, originalCallback](Result result) { + if (result == ResultOk) { + LOG_INFO(getName() << "Closed producer " << producerId_); + shutdown(); + } else { + LOG_ERROR(getName() << "Failed to close producer: " << strResult(result)); + } + if (originalCallback) { + originalCallback(result); + } + }; + // if the producer was never started then there is nothing to clean up State expectedState = NotStarted; if (state_.compare_exchange_strong(expectedState, Closed)) { @@ -653,9 +669,6 @@ void ProducerImpl::closeAsync(CloseCallback callback) { return; } - // Keep a reference to ensure object is kept alive - ProducerImplPtr ptr = shared_from_this(); - cancelTimers(); if (semaphore_) { @@ -669,10 +682,7 @@ void ProducerImpl::closeAsync(CloseCallback callback) { // just like Java's `getAndUpdate` method on an atomic variable const auto state = state_.load(); if (state != Ready && state != Pending) { - state_ = Closed; - if (callback) { - callback(ResultAlreadyClosed); - } + callback(ResultAlreadyClosed); return; } @@ -681,53 +691,24 @@ void ProducerImpl::closeAsync(CloseCallback callback) { ClientConnectionPtr cnx = getCnx().lock(); if (!cnx) { - state_ = Closed; - - if (callback) { - callback(ResultOk); - } + callback(ResultOk); return; } // Detach the producer from the connection to avoid sending any other // message from the producer - connection_.reset(); + resetCnx(); ClientImplPtr client = client_.lock(); if (!client) { - state_ = Closed; - // Client was already destroyed - if (callback) { - callback(ResultOk); - } + callback(ResultOk); return; } int requestId = client->newRequestId(); - Future future = - cnx->sendRequestWithId(Commands::newCloseProducer(producerId_, requestId), requestId); - if (callback) { - // Pass the shared pointer "ptr" to the handler to prevent the object from being destroyed - future.addListener( - std::bind(&ProducerImpl::handleClose, shared_from_this(), std::placeholders::_1, callback, ptr)); - } -} - -void ProducerImpl::handleClose(Result result, ResultCallback callback, ProducerImplPtr producer) { - if (result == ResultOk) { - state_ = Closed; - LOG_INFO(getName() << "Closed producer " << producerId_); - ClientConnectionPtr cnx = getCnx().lock(); - if (cnx) { - cnx->removeProducer(producerId_); - } - } else { - LOG_ERROR(getName() << "Failed to close producer: " << strResult(result)); - } - - if (callback) { - callback(result); - } + auto self = shared_from_this(); + cnx->sendRequestWithId(Commands::newCloseProducer(producerId_, requestId), requestId) + .addListener([self, callback](Result result, const ResponseData&) { callback(result); }); } Future ProducerImpl::getProducerCreatedFuture() { @@ -868,9 +849,7 @@ bool ProducerImpl::encryptMessage(proto::MessageMetadata& metadata, SharedBuffer void ProducerImpl::disconnectProducer() { LOG_DEBUG("Broker notification of Closed producer: " << producerId_); - Lock lock(mutex_); - connection_.reset(); - lock.unlock(); + resetCnx(); scheduleReconnection(shared_from_this()); } @@ -885,16 +864,21 @@ void ProducerImpl::start() { } void ProducerImpl::shutdown() { - Lock lock(mutex_); - state_ = Closed; + resetCnx(); + auto client = client_.lock(); + if (client) { + client->cleanupProducer(this); + } cancelTimers(); producerCreatedPromise_.setFailed(ResultAlreadyClosed); + state_ = Closed; } -void ProducerImpl::cancelTimers() { +void ProducerImpl::cancelTimers() noexcept { dataKeyRefreshTask_.stop(); - batchTimer_.cancel(); - sendTimer_.cancel(); + boost::system::error_code ec; + batchTimer_.cancel(ec); + sendTimer_.cancel(ec); } bool ProducerImplCmp::operator()(const ProducerImplPtr& a, const ProducerImplPtr& b) const { diff --git a/lib/ProducerImpl.h b/lib/ProducerImpl.h index 74eee610..05595154 100644 --- a/lib/ProducerImpl.h +++ b/lib/ProducerImpl.h @@ -109,6 +109,7 @@ class ProducerImpl : public HandlerBase, friend class BatchMessageContainer; // overrided methods from HandlerBase + void beforeConnectionChange(ClientConnection& connection) override; void connectionOpened(const ClientConnectionPtr& connection) override; void connectionFailed(Result result) override; HandlerBaseWeakPtr get_weak_from_this() override { return shared_from_this(); } @@ -120,8 +121,6 @@ class ProducerImpl : public HandlerBase, void handleCreateProducer(const ClientConnectionPtr& cnx, Result result, const ResponseData& responseData); - void handleClose(Result result, ResultCallback callback, ProducerImplPtr producer); - void resendMessages(ClientConnectionPtr cnx); void refreshEncryptionKey(const boost::system::error_code& ec); @@ -143,7 +142,7 @@ class ProducerImpl : public HandlerBase, void releaseSemaphore(uint32_t payloadSize); void releaseSemaphoreForSendOp(const OpSendMsg& op); - void cancelTimers(); + void cancelTimers() noexcept; bool isValidProducerState(const SendCallback& callback) const; bool canAddToBatch(const Message& msg) const; diff --git a/lib/SynchronizedHashMap.h b/lib/SynchronizedHashMap.h index 831d1e83..9bed7d79 100644 --- a/lib/SynchronizedHashMap.h +++ b/lib/SynchronizedHashMap.h @@ -74,12 +74,9 @@ class SynchronizedHashMap { // clear the map and apply `f` on each removed value void clear(std::function f) { - Lock lock(mutex_); - auto it = data_.begin(); - while (it != data_.end()) { - f(it->first, it->second); - auto next = data_.erase(it); - it = next; + MapType data = move(); + for (auto&& kv : data) { + f(kv.first, kv.second); } } @@ -131,8 +128,15 @@ class SynchronizedHashMap { return data_.size(); } + MapType move() noexcept { + Lock lock(mutex_); + MapType data; + data_.swap(data); + return data; + } + private: - std::unordered_map data_; + MapType data_; // Use recursive_mutex to allow methods being called in `forEach` mutable MutexType mutex_; }; diff --git a/tests/ClientTest.cc b/tests/ClientTest.cc index 216b5482..aa48bdc4 100644 --- a/tests/ClientTest.cc +++ b/tests/ClientTest.cc @@ -20,7 +20,6 @@ #include "HttpHelper.h" #include "PulsarFriend.h" -#include "WaitUtils.h" #include #include @@ -198,37 +197,34 @@ TEST(ClientTest, testReferenceCount) { Producer producer; ASSERT_EQ(ResultOk, client.createProducer(topic, producer)); ASSERT_EQ(producers.size(), 1); - ASSERT_TRUE(producers[0].use_count() > 0); - LOG_INFO("Reference count of the producer: " << producers[0].use_count()); + + producers.forEachValue([](const ProducerImplBaseWeakPtr &weakProducer) { + LOG_INFO("Reference count of producer: " << weakProducer.use_count()); + ASSERT_FALSE(weakProducer.expired()); + }); Consumer consumer; ASSERT_EQ(ResultOk, client.subscribe(topic, "my-sub", consumer)); ASSERT_EQ(consumers.size(), 1); - ASSERT_TRUE(consumers[0].use_count() > 0); - LOG_INFO("Reference count of the consumer: " << consumers[0].use_count()); ReaderConfiguration readerConf; Reader reader; ASSERT_EQ(ResultOk, client.createReader(topic + "-reader", MessageId::earliest(), readerConf, reader)); ASSERT_EQ(consumers.size(), 2); - ASSERT_TRUE(consumers[1].use_count() > 0); - LOG_INFO("Reference count of the reader's underlying consumer: " << consumers[1].use_count()); + + consumers.forEachValue([](const ConsumerImplBaseWeakPtr &weakConsumer) { + LOG_INFO("Reference count of consumer: " << weakConsumer.use_count()); + ASSERT_FALSE(weakConsumer.expired()); + }); readerWeakPtr = PulsarFriend::getReaderImplWeakPtr(reader); ASSERT_TRUE(readerWeakPtr.use_count() > 0); LOG_INFO("Reference count of the reader: " << readerWeakPtr.use_count()); } - ASSERT_EQ(producers.size(), 1); - ASSERT_EQ(producers[0].use_count(), 0); - ASSERT_EQ(consumers.size(), 2); - - waitUntil(std::chrono::seconds(1), [&consumers, &readerWeakPtr] { - return consumers[0].use_count() == 0 && consumers[1].use_count() == 0 && readerWeakPtr.expired(); - }); - EXPECT_EQ(consumers[0].use_count(), 0); - EXPECT_EQ(consumers[1].use_count(), 0); + EXPECT_EQ(producers.size(), 0); + EXPECT_EQ(consumers.size(), 0); EXPECT_EQ(readerWeakPtr.use_count(), 0); client.close(); } diff --git a/tests/PulsarFriend.h b/tests/PulsarFriend.h index d9f9923c..df8e3dc0 100644 --- a/tests/PulsarFriend.h +++ b/tests/PulsarFriend.h @@ -98,14 +98,45 @@ class PulsarFriend { static std::shared_ptr getClientImplPtr(Client client) { return client.impl_; } - static ClientImpl::ProducersList& getProducers(const Client& client) { + static auto getProducers(const Client& client) -> decltype(ClientImpl::producers_)& { return getClientImplPtr(client)->producers_; } - static ClientImpl::ConsumersList& getConsumers(const Client& client) { + static auto getConsumers(const Client& client) -> decltype(ClientImpl::consumers_)& { return getClientImplPtr(client)->consumers_; } + static std::vector getConnections(const Client& client) { + auto& pool = client.impl_->pool_; + std::vector connections; + std::lock_guard lock(pool.mutex_); + for (const auto& kv : pool.pool_) { + auto cnx = kv.second.lock(); + if (cnx) { + connections.emplace_back(cnx); + } + } + return connections; + } + + static std::vector getProducers(const ClientConnection& cnx) { + std::vector producers; + std::lock_guard lock(cnx.mutex_); + for (const auto& kv : cnx.producers_) { + producers.emplace_back(kv.second.lock()); + } + return producers; + } + + static std::vector getConsumers(const ClientConnection& cnx) { + std::vector consumers; + std::lock_guard lock(cnx.mutex_); + for (const auto& kv : cnx.consumers_) { + consumers.emplace_back(kv.second.lock()); + } + return consumers; + } + static void setNegativeAckEnabled(Consumer consumer, bool enabled) { consumer.impl_->setNegativeAcknowledgeEnabledForTesting(enabled); } diff --git a/tests/ShutdownTest.cc b/tests/ShutdownTest.cc new file mode 100644 index 00000000..e32a95c4 --- /dev/null +++ b/tests/ShutdownTest.cc @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include +#include +#include "lib/ClientImpl.h" +#include "HttpHelper.h" +#include "PulsarFriend.h" + +using namespace pulsar; + +static const std::string lookupUrl = "pulsar://localhost:6650"; + +enum class EndToEndType : uint8_t +{ + SINGLE_TOPIC, + MULTI_TOPICS, + REGEX_TOPICS +}; + +class ShutdownTest : public ::testing::TestWithParam { + public: + void SetUp() override { + topic_ = topic_ + std::to_string(id_++) + "-" + std::to_string(time(nullptr)); + if (GetParam() != EndToEndType::SINGLE_TOPIC) { + int res = makePutRequest( + "http://localhost:8080/admin/v2/persistent/public/default/" + topic_ + "/partitions", "2"); + ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; + } + } + + protected: + Client client_{lookupUrl}; + decltype(PulsarFriend::getProducers(client_)) producers_{PulsarFriend::getProducers(client_)}; + decltype(PulsarFriend::getConsumers(client_)) consumers_{PulsarFriend::getConsumers(client_)}; + std::string topic_ = "shutdown-test-"; + + static std::atomic_int id_; + + Result subscribe(Consumer &consumer) { + if (GetParam() == EndToEndType::REGEX_TOPICS) { + // NOTE: Currently the regex subscription requires the complete namespace prefix + return client_.subscribeWithRegex("persistent://public/default/" + topic_ + ".*", "sub", + consumer); + } else { + return client_.subscribe(topic_, "sub", consumer); + } + } + + void assertConnectionsEmpty() { + auto connections = PulsarFriend::getConnections(client_); + for (const auto &cnx : PulsarFriend::getConnections(client_)) { + EXPECT_TRUE(PulsarFriend::getProducers(*cnx).empty()); + EXPECT_TRUE(PulsarFriend::getConsumers(*cnx).empty()); + } + } +}; + +std::atomic_int ShutdownTest::id_{0}; + +TEST_P(ShutdownTest, testClose) { + Producer producer; + ASSERT_EQ(ResultOk, client_.createProducer(topic_, producer)); + EXPECT_EQ(producers_.size(), 1); + ASSERT_EQ(ResultOk, producer.close()); + EXPECT_EQ(producers_.size(), 0); + + Consumer consumer; + ASSERT_EQ(ResultOk, subscribe(consumer)); + EXPECT_EQ(consumers_.size(), 1); + ASSERT_EQ(ResultOk, consumer.close()); + EXPECT_EQ(consumers_.size(), 0); + + ASSERT_EQ(ResultOk, subscribe(consumer)); + EXPECT_EQ(consumers_.size(), 1); + ASSERT_EQ(ResultOk, consumer.unsubscribe()); + EXPECT_EQ(consumers_.size(), 0); + + assertConnectionsEmpty(); + ASSERT_EQ(ResultOk, client_.close()); +} + +TEST_P(ShutdownTest, testDestructor) { + { + Producer producer; + ASSERT_EQ(ResultOk, client_.createProducer(topic_, producer)); + EXPECT_EQ(producers_.size(), 1); + } + EXPECT_EQ(producers_.size(), 0); + + { + Consumer consumer; + ASSERT_EQ(ResultOk, subscribe(consumer)); + EXPECT_EQ(consumers_.size(), 1); + } + EXPECT_EQ(consumers_.size(), 0); + + assertConnectionsEmpty(); + client_.close(); +} + +INSTANTIATE_TEST_SUITE_P(Pulsar, ShutdownTest, + ::testing::Values(EndToEndType::SINGLE_TOPIC, EndToEndType::MULTI_TOPICS, + EndToEndType::REGEX_TOPICS)); diff --git a/tests/WaitUtils.h b/tests/WaitUtils.h deleted file mode 100644 index abe3efcc..00000000 --- a/tests/WaitUtils.h +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#pragma once - -#include -#include -#include - -namespace pulsar { - -template -inline void waitUntil(std::chrono::duration timeout, std::function condition) { - auto timeoutMs = std::chrono::duration_cast(timeout).count(); - while (timeoutMs > 0) { - auto now = std::chrono::high_resolution_clock::now(); - if (condition()) { - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - auto elapsed = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - now) - .count(); - timeoutMs -= elapsed; - } -} - -} // namespace pulsar From dd1481bd7e60cb451cc3cd277f99f4d0de7ef9d8 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 21 Oct 2022 11:55:13 +0800 Subject: [PATCH 13/19] [flaky-test] Fix very flaky tests for TEST_P (#59) Fixes #58 #24 ### Motivation gtest-parallel runs tests in different processes, not threads. So the topic name could be the same even if it has the timestamp suffix. Then `ConsumerBusy` error would occur. ### Modifications In each `TEST_P` method, convert `GetParam()` to a unique string to avoid topic conflict. --- tests/ProducerTest.cc | 6 +++-- tests/ShutdownTest.cc | 54 +++++++++++++++++++++++++------------------ 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/tests/ProducerTest.cc b/tests/ProducerTest.cc index d351ee9c..36b23eed 100644 --- a/tests/ProducerTest.cc +++ b/tests/ProducerTest.cc @@ -219,7 +219,8 @@ class ProducerTest : public ::testing::TestWithParam {}; TEST_P(ProducerTest, testMaxMessageSize) { Client client(serviceUrl); - const std::string topic = "ProducerTest-NoBatchMaxMessageSize-" + std::to_string(time(nullptr)); + const auto topic = std::string("ProducerTest-NoBatchMaxMessageSize-") + + (GetParam() ? "batch-" : "-no-batch-") + std::to_string(time(nullptr)); Consumer consumer; ASSERT_EQ(ResultOk, client.subscribe(topic, "sub", consumer)); @@ -247,7 +248,8 @@ TEST_P(ProducerTest, testMaxMessageSize) { TEST_P(ProducerTest, testChunkingMaxMessageSize) { Client client(serviceUrl); - const std::string topic = "ProducerTest-ChunkingMaxMessageSize-" + std::to_string(time(nullptr)); + const auto topic = std::string("ProducerTest-ChunkingMaxMessageSize-") + + (GetParam() ? "batch-" : "no-batch-") + std::to_string(time(nullptr)); Consumer consumer; ASSERT_EQ(ResultOk, client.subscribe(topic, "sub", consumer)); diff --git a/tests/ShutdownTest.cc b/tests/ShutdownTest.cc index e32a95c4..d9a9c232 100644 --- a/tests/ShutdownTest.cc +++ b/tests/ShutdownTest.cc @@ -35,60 +35,66 @@ enum class EndToEndType : uint8_t REGEX_TOPICS }; -class ShutdownTest : public ::testing::TestWithParam { - public: - void SetUp() override { - topic_ = topic_ + std::to_string(id_++) + "-" + std::to_string(time(nullptr)); - if (GetParam() != EndToEndType::SINGLE_TOPIC) { - int res = makePutRequest( - "http://localhost:8080/admin/v2/persistent/public/default/" + topic_ + "/partitions", "2"); - ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; - } +static std::string toString(EndToEndType endToEndType) { + switch (endToEndType) { + case EndToEndType::SINGLE_TOPIC: + return "single-topic"; + case EndToEndType::MULTI_TOPICS: + return "multi-topics"; + case EndToEndType::REGEX_TOPICS: + return "regex-topics"; + default: + return "???"; } +} +class ShutdownTest : public ::testing::TestWithParam { protected: Client client_{lookupUrl}; decltype(PulsarFriend::getProducers(client_)) producers_{PulsarFriend::getProducers(client_)}; decltype(PulsarFriend::getConsumers(client_)) consumers_{PulsarFriend::getConsumers(client_)}; - std::string topic_ = "shutdown-test-"; - static std::atomic_int id_; + void createPartitionedTopic(const std::string& topic) { + if (GetParam() != EndToEndType::SINGLE_TOPIC) { + int res = makePutRequest( + "http://localhost:8080/admin/v2/persistent/public/default/" + topic + "/partitions", "2"); + ASSERT_TRUE(res == 204 || res == 409) << "res: " << res; + } + } - Result subscribe(Consumer &consumer) { + Result subscribe(Consumer& consumer, const std::string& topic) { if (GetParam() == EndToEndType::REGEX_TOPICS) { // NOTE: Currently the regex subscription requires the complete namespace prefix - return client_.subscribeWithRegex("persistent://public/default/" + topic_ + ".*", "sub", - consumer); + return client_.subscribeWithRegex("persistent://public/default/" + topic + ".*", "sub", consumer); } else { - return client_.subscribe(topic_, "sub", consumer); + return client_.subscribe(topic, "sub", consumer); } } void assertConnectionsEmpty() { auto connections = PulsarFriend::getConnections(client_); - for (const auto &cnx : PulsarFriend::getConnections(client_)) { + for (const auto& cnx : PulsarFriend::getConnections(client_)) { EXPECT_TRUE(PulsarFriend::getProducers(*cnx).empty()); EXPECT_TRUE(PulsarFriend::getConsumers(*cnx).empty()); } } }; -std::atomic_int ShutdownTest::id_{0}; - TEST_P(ShutdownTest, testClose) { + std::string topic = "shutdown-test-close-" + toString(GetParam()) + "-" + std::to_string(time(nullptr)); Producer producer; - ASSERT_EQ(ResultOk, client_.createProducer(topic_, producer)); + ASSERT_EQ(ResultOk, client_.createProducer(topic, producer)); EXPECT_EQ(producers_.size(), 1); ASSERT_EQ(ResultOk, producer.close()); EXPECT_EQ(producers_.size(), 0); Consumer consumer; - ASSERT_EQ(ResultOk, subscribe(consumer)); + ASSERT_EQ(ResultOk, subscribe(consumer, topic)); EXPECT_EQ(consumers_.size(), 1); ASSERT_EQ(ResultOk, consumer.close()); EXPECT_EQ(consumers_.size(), 0); - ASSERT_EQ(ResultOk, subscribe(consumer)); + ASSERT_EQ(ResultOk, subscribe(consumer, topic)); EXPECT_EQ(consumers_.size(), 1); ASSERT_EQ(ResultOk, consumer.unsubscribe()); EXPECT_EQ(consumers_.size(), 0); @@ -98,16 +104,18 @@ TEST_P(ShutdownTest, testClose) { } TEST_P(ShutdownTest, testDestructor) { + std::string topic = + "shutdown-test-destructor-" + toString(GetParam()) + "-" + std::to_string(time(nullptr)); { Producer producer; - ASSERT_EQ(ResultOk, client_.createProducer(topic_, producer)); + ASSERT_EQ(ResultOk, client_.createProducer(topic, producer)); EXPECT_EQ(producers_.size(), 1); } EXPECT_EQ(producers_.size(), 0); { Consumer consumer; - ASSERT_EQ(ResultOk, subscribe(consumer)); + ASSERT_EQ(ResultOk, subscribe(consumer, topic)); EXPECT_EQ(consumers_.size(), 1); } EXPECT_EQ(consumers_.size(), 0); From 1b701e335772b8e7133aaa8524e6c15ab4828ab4 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 20 Oct 2022 23:55:47 -0700 Subject: [PATCH 14/19] Fixed the release artifacts package file name (#57) ### Motivation Because of the typo in the variable name, the packages end up with the same name for both x86_64 and arm64 and GH action is merging them. It's mainly a problem for Deb packages since the files don't have the arch in the file names. --- .github/workflows/ci-build-binary-artifacts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index f2a931ef..7c0a0017 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -75,5 +75,5 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v3 with: - name: ${{matrix.pkg.type}}-${{matrix.pkg.platform}} + name: ${{matrix.pkg.type}}-${{matrix.cpu.platform}} path: ${{matrix.pkg.path}} From 7bb6402bb15dbbf51d42cf8e54793d835db52a55 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 21 Oct 2022 21:40:22 +0800 Subject: [PATCH 15/19] [flaky tests] Fix flaky ShutdownTest::testDestructor (#62) Fixes #61 ### Motivation `testDestructor` is flaky because the destructor might not be called immediately after the `shared_ptr` object goes out of the scope. It's similar like the flaky `testReferenceCount` before in https://github.com/apache/pulsar/pull/17645. ### Modifications Add back `WaitUtils.h`, which was removed in #55, add use `waitUntil` to wait until the assertion. ### Verifications Run the reproduce script in #61. Even if the loop count was increased to 100, it still never failed. --- tests/ShutdownTest.cc | 3 +++ tests/WaitUtils.h | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/WaitUtils.h diff --git a/tests/ShutdownTest.cc b/tests/ShutdownTest.cc index d9a9c232..39513474 100644 --- a/tests/ShutdownTest.cc +++ b/tests/ShutdownTest.cc @@ -23,6 +23,7 @@ #include "lib/ClientImpl.h" #include "HttpHelper.h" #include "PulsarFriend.h" +#include "WaitUtils.h" using namespace pulsar; @@ -111,6 +112,7 @@ TEST_P(ShutdownTest, testDestructor) { ASSERT_EQ(ResultOk, client_.createProducer(topic, producer)); EXPECT_EQ(producers_.size(), 1); } + waitUntil(std::chrono::seconds(2), [this] { return producers_.size() == 0; }); EXPECT_EQ(producers_.size(), 0); { @@ -118,6 +120,7 @@ TEST_P(ShutdownTest, testDestructor) { ASSERT_EQ(ResultOk, subscribe(consumer, topic)); EXPECT_EQ(consumers_.size(), 1); } + waitUntil(std::chrono::seconds(2), [this] { return consumers_.size() == 0; }); EXPECT_EQ(consumers_.size(), 0); assertConnectionsEmpty(); diff --git a/tests/WaitUtils.h b/tests/WaitUtils.h new file mode 100644 index 00000000..abe3efcc --- /dev/null +++ b/tests/WaitUtils.h @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include + +namespace pulsar { + +template +inline void waitUntil(std::chrono::duration timeout, std::function condition) { + auto timeoutMs = std::chrono::duration_cast(timeout).count(); + while (timeoutMs > 0) { + auto now = std::chrono::high_resolution_clock::now(); + if (condition()) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + auto elapsed = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - now) + .count(); + timeoutMs -= elapsed; + } +} + +} // namespace pulsar From 872f8abaade7ecd346d3f59e2f6b3901c65ef7de Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 25 Oct 2022 02:29:00 +0800 Subject: [PATCH 16/19] [refactor] Apply forward declaration as much as possible (#64) * [refactor] Apply forward declaration as much as possible Fixes https://github.com/apache/pulsar-client-cpp/issues/60 ### Motivation The includes in pulsar-client-cpp is very casual. There are a lot of implicit includes and the forward declaration is not used much. For example, if `lib/ClientConnection.h` was modified, 27 files would be recompiled. The other problem is the `SortIncludes` attribute in `.clang-format` file is false. It might be okay in early days. However, as the project grows, including many headers without ordering brings a very bad experience. Combining with the very few usages of the forward declaration, it's hard to determine whether a header is still required after a change. ### Modifications Apply forward declarations as much as possible and change the `SortIncludes` to true in `.clang-format` file. It's special for the `PulsarApi.pb.h` file because the size of this header is over 1 MiB. For classes we can use forward declaration, but for enumerations we have to include this header. To solve this problem, `ProtoApiEnums.h` is added to define some constant integers that can be cast implicitly from the enumerations. If we want to use enumerations from `PulsarApi.pb.h`, we can include `ProtoApiEnums.h` instead. Finally, to unify the include rules, in `lib/*.h`, if we want to include a header (e.g. `xxx.h`) from the same directory, just use `#include "xxx.h"`. Don't use `#include "lib/xxx.h"` or `#include ` in headers of `lib/` directory. ### Improvements Since forward declaration is applied everywhere now, take `lib/ClientConnection.h` for example. After this patch, only 10 files needs to be recompiled, while 27 files would be recompiled before. This patch also reduces the binary size and speeds up the compilation time. Binary size: - `libpulsar.a`: 319234696 (305 MiB) -> 286716564 (274 MiB) - `libpulsar.so`: 110162496 (106 MiB) -> 102428456 (98 MiB) Compilation time with the following commands: ```bash cmake -B build -DBUILD_TESTS=OFF cmake --build build -j8 ``` * Fix MSVC build --- .clang-format | 2 +- examples/SampleAsyncProducer.cc | 6 +- examples/SampleConsumer.cc | 6 +- examples/SampleConsumerListener.cc | 6 +- examples/SampleProducer.cc | 6 +- include/pulsar/Authentication.h | 9 +- include/pulsar/BatchReceivePolicy.h | 1 + include/pulsar/BrokerConsumerStats.h | 8 +- include/pulsar/Client.h | 13 +- include/pulsar/ClientConfiguration.h | 2 +- include/pulsar/Consumer.h | 5 +- include/pulsar/ConsumerConfiguration.h | 18 +-- include/pulsar/CryptoKeyReader.h | 4 +- include/pulsar/DeprecatedException.h | 3 +- include/pulsar/EncryptionKeyInfo.h | 5 +- include/pulsar/KeySharedPolicy.h | 1 - include/pulsar/Logger.h | 3 +- include/pulsar/Message.h | 6 +- include/pulsar/MessageBatch.h | 6 +- include/pulsar/MessageBuilder.h | 6 +- include/pulsar/MessageId.h | 5 +- include/pulsar/MessageRoutingPolicy.h | 3 +- include/pulsar/Producer.h | 5 +- include/pulsar/ProducerConfiguration.h | 10 +- include/pulsar/ProtobufNativeSchema.h | 2 +- include/pulsar/Reader.h | 2 +- include/pulsar/ReaderConfiguration.h | 13 +- include/pulsar/Result.h | 3 +- include/pulsar/Schema.h | 4 +- include/pulsar/c/client.h | 8 +- include/pulsar/c/consumer.h | 3 +- include/pulsar/c/consumer_configuration.h | 1 + include/pulsar/c/message.h | 2 +- include/pulsar/c/message_id.h | 2 +- include/pulsar/c/message_router.h | 2 +- include/pulsar/c/producer.h | 5 +- include/pulsar/c/producer_configuration.h | 5 +- include/pulsar/c/reader.h | 4 +- include/pulsar/c/reader_configuration.h | 2 +- lib/AckGroupingTracker.cc | 22 +-- lib/AckGroupingTracker.h | 13 +- lib/AckGroupingTrackerDisabled.cc | 8 +- lib/AckGroupingTrackerDisabled.h | 4 +- lib/AckGroupingTrackerEnabled.cc | 11 +- lib/AckGroupingTrackerEnabled.h | 18 ++- lib/Authentication.cc | 23 ++-- lib/Backoff.cc | 6 +- lib/Backoff.h | 5 +- lib/BatchAcknowledgementTracker.cc | 15 ++- lib/BatchAcknowledgementTracker.h | 18 +-- lib/BatchMessageContainer.cc | 9 +- lib/BatchMessageContainerBase.cc | 32 ++++- lib/BatchMessageContainerBase.h | 35 +---- lib/BatchMessageKeyBasedContainer.cc | 8 +- lib/BatchReceivePolicy.cc | 1 + lib/BinaryProtoLookupService.cc | 10 +- lib/BinaryProtoLookupService.h | 14 +- lib/BlockingQueue.h | 5 +- lib/BoostHash.h | 5 +- lib/BrokerConsumerStats.cc | 5 +- lib/BrokerConsumerStatsImpl.cc | 3 +- lib/BrokerConsumerStatsImpl.h | 12 +- lib/BrokerConsumerStatsImplBase.h | 1 - lib/Client.cc | 7 +- lib/ClientConfiguration.cc | 2 +- lib/ClientConnection.cc | 121 ++++++++--------- lib/ClientConnection.h | 54 ++++---- lib/ClientImpl.cc | 32 ++--- lib/ClientImpl.h | 30 +++-- lib/Commands.cc | 81 +++++++---- lib/Commands.h | 28 ++-- lib/CompressionCodec.cc | 37 +---- lib/CompressionCodec.h | 9 +- lib/CompressionCodecLZ4.cc | 3 +- lib/CompressionCodecSnappy.cc | 2 +- lib/CompressionCodecZLib.cc | 4 +- lib/CompressionCodecZLib.h | 2 - lib/ConnectionPool.cc | 9 +- lib/ConnectionPool.h | 14 +- lib/ConsoleLoggerFactory.cc | 3 +- lib/ConsoleLoggerFactoryImpl.h | 3 +- lib/Consumer.cc | 6 +- lib/ConsumerConfiguration.cc | 5 +- lib/ConsumerImpl.cc | 76 ++++++----- lib/ConsumerImpl.h | 61 +++++---- lib/ConsumerImplBase.cc | 13 +- lib/ConsumerImplBase.h | 11 +- lib/CryptoKeyReader.cc | 9 +- lib/DeprecatedException.cc | 2 +- lib/EncryptionKeyInfoImpl.h | 5 +- lib/ExecutorService.cc | 6 +- lib/ExecutorService.h | 15 ++- lib/FileLoggerFactory.cc | 3 +- lib/FileLoggerFactoryImpl.h | 5 +- lib/Future.h | 9 +- lib/GetLastMessageIdResponse.h | 1 + lib/HTTPLookupService.cc | 9 +- lib/HTTPLookupService.h | 13 +- lib/HandlerBase.cc | 7 +- lib/HandlerBase.h | 20 ++- lib/JavaStringHash.cc | 1 + lib/JavaStringHash.h | 3 +- lib/KeySharedPolicy.cc | 4 +- lib/Latch.h | 5 +- lib/Log4CxxLogger.h | 2 +- lib/Log4cxxLogger.cc | 5 +- lib/LogUtils.cc | 3 +- lib/LogUtils.h | 10 +- lib/LookupDataResult.h | 7 +- lib/LookupService.h | 18 +-- lib/MemoryLimitController.h | 2 +- lib/Message.cc | 7 +- lib/MessageAndCallbackBatch.cc | 1 + lib/MessageAndCallbackBatch.h | 5 +- lib/MessageBuilder.cc | 5 +- lib/MessageCrypto.cc | 20 +-- lib/MessageCrypto.h | 33 ++--- lib/MessageId.cc | 11 +- lib/MessageIdImpl.h | 1 + lib/MessageIdUtil.h | 6 - lib/MessageImpl.h | 3 +- lib/MessageRouterBase.cc | 2 +- lib/MessageRouterBase.h | 7 +- lib/MessagesImpl.cc | 3 +- lib/MessagesImpl.h | 3 +- lib/MultiTopicsBrokerConsumerStatsImpl.cc | 5 +- lib/MultiTopicsBrokerConsumerStatsImpl.h | 9 +- lib/MultiTopicsConsumerImpl.cc | 25 +++- lib/MultiTopicsConsumerImpl.h | 46 ++++--- lib/Murmur3_32Hash.h | 3 +- lib/NamespaceName.cc | 10 +- lib/NamespaceName.h | 3 +- lib/NegativeAcksTracker.cc | 7 +- lib/NegativeAcksTracker.h | 16 ++- lib/ObjectPool.h | 3 +- lib/OpSendMsg.h | 4 +- lib/PartitionedProducerImpl.cc | 14 +- lib/PartitionedProducerImpl.h | 30 ++++- lib/PatternMultiTopicsConsumerImpl.cc | 5 + lib/PatternMultiTopicsConsumerImpl.h | 16 ++- lib/PeriodicTask.cc | 3 +- lib/PeriodicTask.h | 4 +- lib/Producer.cc | 5 +- lib/ProducerConfiguration.cc | 4 +- lib/ProducerConfigurationImpl.h | 1 + lib/ProducerImpl.cc | 27 ++-- lib/ProducerImpl.h | 42 +++--- lib/ProducerImplBase.h | 2 + lib/ProtoApiEnums.h | 156 ++++++++++++++++++++++ lib/ProtobufNativeSchema.cc | 6 +- lib/Reader.cc | 2 +- lib/ReaderConfiguration.cc | 2 +- lib/ReaderImpl.cc | 6 +- lib/ReaderImpl.h | 23 +++- lib/Result.cc | 2 +- lib/RetryableLookupService.h | 13 +- lib/RoundRobinMessageRouter.cc | 5 +- lib/RoundRobinMessageRouter.h | 8 +- lib/Schema.cc | 2 +- lib/Semaphore.h | 2 +- lib/ServiceNameResolver.h | 2 + lib/ServiceURI.cc | 1 + lib/ServiceURI.h | 1 + lib/SharedBuffer.h | 4 +- lib/SimpleLogger.h | 6 +- lib/SinglePartitionMessageRouter.cc | 2 + lib/SinglePartitionMessageRouter.h | 6 +- lib/SynchronizedHashMap.h | 1 + lib/TimeUtils.h | 6 +- lib/TopicMetadataImpl.cc | 2 +- lib/TopicMetadataImpl.h | 1 - lib/TopicName.cc | 20 +-- lib/TopicName.h | 13 +- lib/UnAckedMessageTrackerDisabled.h | 2 +- lib/UnAckedMessageTrackerEnabled.cc | 5 + lib/UnAckedMessageTrackerEnabled.h | 20 ++- lib/UnAckedMessageTrackerInterface.h | 14 +- lib/UnboundedBlockingQueue.h | 4 +- lib/Url.cc | 1 - lib/Url.h | 3 +- lib/UtilAllocator.h | 1 + lib/Utils.h | 6 +- lib/auth/AuthAthenz.cc | 10 +- lib/auth/AuthAthenz.h | 4 +- lib/auth/AuthBasic.cc | 7 +- lib/auth/AuthBasic.h | 2 - lib/auth/AuthOauth2.cc | 9 +- lib/auth/AuthOauth2.h | 2 +- lib/auth/AuthTls.cc | 2 +- lib/auth/AuthTls.h | 1 - lib/auth/AuthToken.cc | 6 +- lib/auth/AuthToken.h | 2 - lib/auth/athenz/ZTSClient.cc | 16 +-- lib/auth/athenz/ZTSClient.h | 6 +- lib/c/cStringMap.cc | 2 +- lib/c/c_Authentication.cc | 5 +- lib/c/c_Message.cc | 1 + lib/c/c_MessageId.cc | 4 +- lib/c/c_ProducerConfiguration.cc | 2 +- lib/c/c_Reader.cc | 2 +- lib/c/c_ReaderConfiguration.cc | 8 +- lib/c/c_Result.cc | 2 +- lib/c/c_structs.h | 4 +- lib/checksum/ChecksumProvider.h | 2 +- lib/checksum/crc32c_arm.cc | 1 + lib/checksum/crc32c_sse42.cc | 3 +- lib/checksum/crc32c_sw.cc | 1 + lib/lz4/lz4.h | 2 +- lib/stats/ConsumerStatsBase.h | 6 +- lib/stats/ConsumerStatsDisabled.h | 4 +- lib/stats/ConsumerStatsImpl.cc | 15 ++- lib/stats/ConsumerStatsImpl.h | 25 ++-- lib/stats/ProducerStatsBase.h | 1 + lib/stats/ProducerStatsDisabled.h | 2 +- lib/stats/ProducerStatsImpl.cc | 8 +- lib/stats/ProducerStatsImpl.h | 20 +-- perf/PerfConsumer.cc | 15 +-- perf/PerfProducer.cc | 19 +-- perf/RateLimiter.h | 2 +- tests/AuthBasicTest.cc | 3 +- tests/AuthPluginTest.cc | 11 +- tests/AuthTokenTest.cc | 11 +- tests/BackoffTest.cc | 6 +- tests/BasicEndToEndTest.cc | 57 ++++---- tests/BatchMessageTest.cc | 34 +++-- tests/BlockingQueueTest.cc | 5 +- tests/ClientDeduplicationTest.cc | 7 +- tests/ClientTest.cc | 10 +- tests/CompressionCodecSnappyTest.cc | 2 +- tests/ConsumerConfigurationTest.cc | 3 +- tests/ConsumerStatsTest.cc | 17 +-- tests/ConsumerTest.cc | 21 +-- tests/ConsumerTest.h | 3 +- tests/CustomLoggerTest.cc | 6 +- tests/CustomRoutingPolicy.h | 5 +- tests/HashTest.cc | 11 +- tests/KeyBasedBatchingTest.cc | 7 +- tests/KeySharedConsumerTest.cc | 10 +- tests/KeySharedPolicyTest.cc | 12 +- tests/LatchTest.cc | 6 +- tests/LoggerTest.cc | 3 +- tests/LookupServiceTest.cc | 22 +-- tests/MapCacheTest.cc | 3 +- tests/MemoryLimitControllerTest.cc | 7 +- tests/MemoryLimitTest.cc | 14 +- tests/MessageChunkingTest.cc | 7 +- tests/MessageIdTest.cc | 8 +- tests/MessageTest.cc | 8 +- tests/MessagesImplTest.cc | 5 +- tests/NamespaceNameTest.cc | 4 +- tests/PartitionsUpdateTest.cc | 6 +- tests/PeriodicTaskTest.cc | 2 + tests/ProducerConfigurationTest.cc | 1 + tests/ProducerTest.cc | 6 +- tests/PromiseTest.cc | 4 +- tests/ProtobufNativeSchemaTest.cc | 4 +- tests/PulsarFriend.h | 8 +- tests/ReaderConfigurationTest.cc | 3 +- tests/ReaderTest.cc | 16 +-- tests/RoundRobinMessageRouterTest.cc | 7 +- tests/SemaphoreTest.cc | 5 +- tests/ServiceURITest.cc | 1 + tests/ShutdownTest.cc | 8 +- tests/SinglePartitionMessageRouterTest.cc | 12 +- tests/SynchronizedHashMapTest.cc | 2 + tests/TopicMetadataImplTest.cc | 4 +- tests/TopicNameTest.cc | 4 +- tests/UnboundedBlockingQueueTest.cc | 5 +- tests/UrlTest.cc | 3 +- tests/VersionTest.cc | 2 +- tests/ZLibCompressionTest.cc | 3 +- tests/ZTSClientTest.cc | 3 +- tests/ZeroQueueSizeTest.cc | 7 +- tests/c/c_BasicEndToEndTest.cc | 6 +- tests/c/c_ConsumerConfigurationTest.cc | 2 +- tests/c/c_ProducerConfigurationTest.cc | 2 +- tests/main.cc | 1 - wireshark/pulsarDissector.cc | 6 +- 278 files changed, 1557 insertions(+), 1123 deletions(-) create mode 100644 lib/ProtoApiEnums.h diff --git a/.clang-format b/.clang-format index cb40b506..85196460 100644 --- a/.clang-format +++ b/.clang-format @@ -19,7 +19,7 @@ BasedOnStyle: Google IndentWidth: 4 ColumnLimit: 110 -SortIncludes: false +SortIncludes: true BreakBeforeBraces: Custom BraceWrapping: AfterEnum: true diff --git a/examples/SampleAsyncProducer.cc b/examples/SampleAsyncProducer.cc index 9701ccbe..b1bd0f7a 100644 --- a/examples/SampleAsyncProducer.cc +++ b/examples/SampleAsyncProducer.cc @@ -16,12 +16,12 @@ * specific language governing permissions and limitations * under the License. */ +#include + #include #include -#include - -#include +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/examples/SampleConsumer.cc b/examples/SampleConsumer.cc index 1dcc550f..bbf210dd 100644 --- a/examples/SampleConsumer.cc +++ b/examples/SampleConsumer.cc @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -#include - #include -#include +#include + +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/examples/SampleConsumerListener.cc b/examples/SampleConsumerListener.cc index 9ce22916..a3a90cc1 100644 --- a/examples/SampleConsumerListener.cc +++ b/examples/SampleConsumerListener.cc @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -#include - #include -#include +#include + +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/examples/SampleProducer.cc b/examples/SampleProducer.cc index ff504879..0a3c6936 100644 --- a/examples/SampleProducer.cc +++ b/examples/SampleProducer.cc @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -#include - #include -#include +#include + +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/include/pulsar/Authentication.h b/include/pulsar/Authentication.h index 7f8f7d25..fb74df6f 100644 --- a/include/pulsar/Authentication.h +++ b/include/pulsar/Authentication.h @@ -19,13 +19,14 @@ #ifndef PULSAR_AUTHENTICATION_H_ #define PULSAR_AUTHENTICATION_H_ +#include #include -#include -#include + +#include #include #include -#include -#include +#include +#include namespace pulsar { diff --git a/include/pulsar/BatchReceivePolicy.h b/include/pulsar/BatchReceivePolicy.h index 3c66da2f..bc8b791d 100644 --- a/include/pulsar/BatchReceivePolicy.h +++ b/include/pulsar/BatchReceivePolicy.h @@ -20,6 +20,7 @@ #define BATCH_RECEIVE_POLICY_HPP_ #include + #include namespace pulsar { diff --git a/include/pulsar/BrokerConsumerStats.h b/include/pulsar/BrokerConsumerStats.h index b4fe9e08..e179e4b4 100644 --- a/include/pulsar/BrokerConsumerStats.h +++ b/include/pulsar/BrokerConsumerStats.h @@ -19,13 +19,13 @@ #ifndef PULSAR_CPP_BROKERCONSUMERSTATS_H #define PULSAR_CPP_BROKERCONSUMERSTATS_H -#include -#include -#include +#include #include +#include + #include +#include #include -#include namespace pulsar { class BrokerConsumerStatsImplBase; diff --git a/include/pulsar/Client.h b/include/pulsar/Client.h index 3edb03b5..c189a200 100644 --- a/include/pulsar/Client.h +++ b/include/pulsar/Client.h @@ -19,17 +19,18 @@ #ifndef PULSAR_CLIENT_HPP_ #define PULSAR_CLIENT_HPP_ -#include +#include +#include #include +#include +#include +#include #include #include #include -#include -#include -#include #include -#include -#include +#include + #include namespace pulsar { diff --git a/include/pulsar/ClientConfiguration.h b/include/pulsar/ClientConfiguration.h index 32ad32bc..9df92acb 100644 --- a/include/pulsar/ClientConfiguration.h +++ b/include/pulsar/ClientConfiguration.h @@ -19,9 +19,9 @@ #ifndef PULSAR_CLIENTCONFIGURATION_H_ #define PULSAR_CLIENTCONFIGURATION_H_ -#include #include #include +#include namespace pulsar { class PulsarWrapper; diff --git a/include/pulsar/Consumer.h b/include/pulsar/Consumer.h index c7911b98..d37c7f91 100644 --- a/include/pulsar/Consumer.h +++ b/include/pulsar/Consumer.h @@ -19,10 +19,11 @@ #ifndef CONSUMER_HPP_ #define CONSUMER_HPP_ -#include -#include #include #include +#include + +#include namespace pulsar { class PulsarWrapper; diff --git a/include/pulsar/ConsumerConfiguration.h b/include/pulsar/ConsumerConfiguration.h index 13d5cc02..0418cfaf 100644 --- a/include/pulsar/ConsumerConfiguration.h +++ b/include/pulsar/ConsumerConfiguration.h @@ -19,18 +19,20 @@ #ifndef PULSAR_CONSUMERCONFIGURATION_H_ #define PULSAR_CONSUMERCONFIGURATION_H_ -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include #include #include -#include +#include +#include +#include +#include + +#include +#include + #include "BatchReceivePolicy.h" namespace pulsar { diff --git a/include/pulsar/CryptoKeyReader.h b/include/pulsar/CryptoKeyReader.h index e0b2a778..d11e1f89 100644 --- a/include/pulsar/CryptoKeyReader.h +++ b/include/pulsar/CryptoKeyReader.h @@ -19,9 +19,9 @@ #ifndef CRYPTOKEYREADER_H_ #define CRYPTOKEYREADER_H_ -#include -#include #include +#include +#include namespace pulsar { diff --git a/include/pulsar/DeprecatedException.h b/include/pulsar/DeprecatedException.h index 9680591b..affdf6fa 100644 --- a/include/pulsar/DeprecatedException.h +++ b/include/pulsar/DeprecatedException.h @@ -19,9 +19,10 @@ #ifndef DEPRECATED_EXCEPTION_HPP_ #define DEPRECATED_EXCEPTION_HPP_ +#include + #include #include -#include namespace pulsar { class PULSAR_PUBLIC DeprecatedException : public std::runtime_error { diff --git a/include/pulsar/EncryptionKeyInfo.h b/include/pulsar/EncryptionKeyInfo.h index 0357622f..401d7beb 100644 --- a/include/pulsar/EncryptionKeyInfo.h +++ b/include/pulsar/EncryptionKeyInfo.h @@ -19,10 +19,11 @@ #ifndef ENCRYPTIONKEYINFO_H_ #define ENCRYPTIONKEYINFO_H_ -#include +#include + #include #include -#include +#include namespace pulsar { diff --git a/include/pulsar/KeySharedPolicy.h b/include/pulsar/KeySharedPolicy.h index 53efc4c7..08eee77d 100644 --- a/include/pulsar/KeySharedPolicy.h +++ b/include/pulsar/KeySharedPolicy.h @@ -21,7 +21,6 @@ #include #include - #include #include diff --git a/include/pulsar/Logger.h b/include/pulsar/Logger.h index e9487a71..710cdc22 100644 --- a/include/pulsar/Logger.h +++ b/include/pulsar/Logger.h @@ -18,9 +18,10 @@ */ #pragma once +#include + #include #include -#include namespace pulsar { diff --git a/include/pulsar/Message.h b/include/pulsar/Message.h index 935236bd..0c4afc28 100644 --- a/include/pulsar/Message.h +++ b/include/pulsar/Message.h @@ -19,12 +19,12 @@ #ifndef MESSAGE_HPP_ #define MESSAGE_HPP_ -#include -#include +#include +#include #include +#include -#include #include "MessageId.h" namespace pulsar { diff --git a/include/pulsar/MessageBatch.h b/include/pulsar/MessageBatch.h index be943588..952feb77 100644 --- a/include/pulsar/MessageBatch.h +++ b/include/pulsar/MessageBatch.h @@ -19,10 +19,10 @@ #ifndef LIB_MESSAGE_BATCH_H #define LIB_MESSAGE_BATCH_H -#include - -#include #include +#include + +#include namespace pulsar { diff --git a/include/pulsar/MessageBuilder.h b/include/pulsar/MessageBuilder.h index 71dafaae..2b84d208 100644 --- a/include/pulsar/MessageBuilder.h +++ b/include/pulsar/MessageBuilder.h @@ -19,13 +19,13 @@ #ifndef MESSAGE_BUILDER_H #define MESSAGE_BUILDER_H +#include +#include + #include #include #include -#include -#include - namespace pulsar { class PulsarWrapper; diff --git a/include/pulsar/MessageId.h b/include/pulsar/MessageId.h index 06be790c..fd17df6a 100644 --- a/include/pulsar/MessageId.h +++ b/include/pulsar/MessageId.h @@ -19,11 +19,12 @@ #ifndef MESSAGE_ID_H #define MESSAGE_ID_H -#include +#include #include + +#include #include #include -#include namespace pulsar { diff --git a/include/pulsar/MessageRoutingPolicy.h b/include/pulsar/MessageRoutingPolicy.h index bc76259b..26ab75e2 100644 --- a/include/pulsar/MessageRoutingPolicy.h +++ b/include/pulsar/MessageRoutingPolicy.h @@ -19,10 +19,11 @@ #ifndef PULSAR_MESSAGE_ROUTING_POLICY_HEADER_ #define PULSAR_MESSAGE_ROUTING_POLICY_HEADER_ -#include #include #include #include +#include + #include /* diff --git a/include/pulsar/Producer.h b/include/pulsar/Producer.h index f414b76e..955d9858 100644 --- a/include/pulsar/Producer.h +++ b/include/pulsar/Producer.h @@ -19,11 +19,12 @@ #ifndef PRODUCER_HPP_ #define PRODUCER_HPP_ -#include #include -#include +#include #include +#include + namespace pulsar { class ProducerImplBase; class PulsarWrapper; diff --git a/include/pulsar/ProducerConfiguration.h b/include/pulsar/ProducerConfiguration.h index fb331ea8..39ecbe09 100644 --- a/include/pulsar/ProducerConfiguration.h +++ b/include/pulsar/ProducerConfiguration.h @@ -18,16 +18,16 @@ */ #ifndef PULSAR_PRODUCERCONFIGURATION_H_ #define PULSAR_PRODUCERCONFIGURATION_H_ -#include #include -#include -#include +#include #include -#include +#include #include -#include +#include #include +#include +#include #include namespace pulsar { diff --git a/include/pulsar/ProtobufNativeSchema.h b/include/pulsar/ProtobufNativeSchema.h index ef9a7b1c..2feff5af 100644 --- a/include/pulsar/ProtobufNativeSchema.h +++ b/include/pulsar/ProtobufNativeSchema.h @@ -18,8 +18,8 @@ */ #pragma once -#include #include +#include namespace pulsar { diff --git a/include/pulsar/Reader.h b/include/pulsar/Reader.h index 554788e8..233da4ff 100644 --- a/include/pulsar/Reader.h +++ b/include/pulsar/Reader.h @@ -19,9 +19,9 @@ #ifndef PULSAR_READER_HPP_ #define PULSAR_READER_HPP_ -#include #include #include +#include namespace pulsar { class PulsarWrapper; diff --git a/include/pulsar/ReaderConfiguration.h b/include/pulsar/ReaderConfiguration.h index 5b885535..9ae8e1c4 100644 --- a/include/pulsar/ReaderConfiguration.h +++ b/include/pulsar/ReaderConfiguration.h @@ -19,14 +19,15 @@ #ifndef PULSAR_READER_CONFIGURATION_H_ #define PULSAR_READER_CONFIGURATION_H_ -#include -#include -#include -#include +#include +#include #include +#include #include -#include -#include +#include + +#include +#include namespace pulsar { diff --git a/include/pulsar/Result.h b/include/pulsar/Result.h index cc7b4575..0f7d8a8b 100644 --- a/include/pulsar/Result.h +++ b/include/pulsar/Result.h @@ -19,9 +19,10 @@ #ifndef ERROR_HPP_ #define ERROR_HPP_ -#include #include +#include + namespace pulsar { /** diff --git a/include/pulsar/Schema.h b/include/pulsar/Schema.h index 7e7a5aed..ec0802e9 100644 --- a/include/pulsar/Schema.h +++ b/include/pulsar/Schema.h @@ -18,12 +18,12 @@ */ #pragma once -#include +#include #include +#include #include #include -#include namespace pulsar { diff --git a/include/pulsar/c/client.h b/include/pulsar/c/client.h index f5da8266..3a53c017 100644 --- a/include/pulsar/c/client.h +++ b/include/pulsar/c/client.h @@ -19,18 +19,18 @@ #pragma once -#include #include +#include +#include #include #include #include -#include -#include -#include #include +#include #include #include #include +#include #ifdef __cplusplus extern "C" { diff --git a/include/pulsar/c/consumer.h b/include/pulsar/c/consumer.h index 52610d2d..99ff8c80 100644 --- a/include/pulsar/c/consumer.h +++ b/include/pulsar/c/consumer.h @@ -24,9 +24,8 @@ extern "C" { #endif -#include #include - +#include #include typedef struct _pulsar_consumer pulsar_consumer_t; diff --git a/include/pulsar/c/consumer_configuration.h b/include/pulsar/c/consumer_configuration.h index 128fa24f..96abc5cc 100644 --- a/include/pulsar/c/consumer_configuration.h +++ b/include/pulsar/c/consumer_configuration.h @@ -19,6 +19,7 @@ #pragma once #include + #include "consumer.h" #include "producer_configuration.h" diff --git a/include/pulsar/c/message.h b/include/pulsar/c/message.h index f54d0254..353e609f 100644 --- a/include/pulsar/c/message.h +++ b/include/pulsar/c/message.h @@ -23,10 +23,10 @@ extern "C" { #endif +#include #include #include -#include #include "string_map.h" typedef struct _pulsar_message pulsar_message_t; diff --git a/include/pulsar/c/message_id.h b/include/pulsar/c/message_id.h index 289c3bdc..13679340 100644 --- a/include/pulsar/c/message_id.h +++ b/include/pulsar/c/message_id.h @@ -23,9 +23,9 @@ extern "C" { #endif +#include #include #include -#include typedef struct _pulsar_message_id pulsar_message_id_t; diff --git a/include/pulsar/c/message_router.h b/include/pulsar/c/message_router.h index ed74f070..309a5b22 100644 --- a/include/pulsar/c/message_router.h +++ b/include/pulsar/c/message_router.h @@ -19,8 +19,8 @@ #pragma once -#include #include +#include #ifdef __cplusplus extern "C" { diff --git a/include/pulsar/c/producer.h b/include/pulsar/c/producer.h index bf51f562..1de8134f 100644 --- a/include/pulsar/c/producer.h +++ b/include/pulsar/c/producer.h @@ -23,10 +23,9 @@ extern "C" { #endif -#include -#include #include - +#include +#include #include typedef struct _pulsar_producer pulsar_producer_t; diff --git a/include/pulsar/c/producer_configuration.h b/include/pulsar/c/producer_configuration.h index 9e5e5b0d..f8f74c25 100644 --- a/include/pulsar/c/producer_configuration.h +++ b/include/pulsar/c/producer_configuration.h @@ -19,10 +19,9 @@ #pragma once -#include - -#include #include +#include +#include #ifdef __cplusplus extern "C" { diff --git a/include/pulsar/c/reader.h b/include/pulsar/c/reader.h index 4c09ff5d..4c546f80 100644 --- a/include/pulsar/c/reader.h +++ b/include/pulsar/c/reader.h @@ -18,9 +18,9 @@ */ #pragma once -#include -#include #include +#include +#include #ifdef __cplusplus extern "C" { diff --git a/include/pulsar/c/reader_configuration.h b/include/pulsar/c/reader_configuration.h index cc8436cd..66ce8ef9 100644 --- a/include/pulsar/c/reader_configuration.h +++ b/include/pulsar/c/reader_configuration.h @@ -19,9 +19,9 @@ #pragma once -#include #include #include +#include #ifdef __cplusplus extern "C" { diff --git a/lib/AckGroupingTracker.cc b/lib/AckGroupingTracker.cc index 7d1d706d..4abfcb19 100644 --- a/lib/AckGroupingTracker.cc +++ b/lib/AckGroupingTracker.cc @@ -19,33 +19,23 @@ #include "AckGroupingTracker.h" -#include - -#include - +#include "ClientConnection.h" #include "Commands.h" #include "LogUtils.h" -#include "PulsarApi.pb.h" -#include "ClientConnection.h" -#include namespace pulsar { DECLARE_LOG_OBJECT(); inline void sendAck(ClientConnectionPtr cnx, uint64_t consumerId, const MessageId& msgId, - proto::CommandAck_AckType ackType) { - proto::MessageIdData msgIdData; - msgIdData.set_ledgerid(msgId.ledgerId()); - msgIdData.set_entryid(msgId.entryId()); - auto cmd = Commands::newAck(consumerId, msgIdData, ackType, -1); + CommandAck_AckType ackType) { + auto cmd = Commands::newAck(consumerId, msgId.ledgerId(), msgId.entryId(), ackType, -1); cnx->sendCommand(cmd); - LOG_DEBUG("ACK request is sent for message - [" << msgIdData.ledgerid() << ", " << msgIdData.entryid() - << "]"); + LOG_DEBUG("ACK request is sent for message - [" << msgId.ledgerId() << ", " << msgId.entryId() << "]"); } bool AckGroupingTracker::doImmediateAck(ClientConnectionWeakPtr connWeakPtr, uint64_t consumerId, - const MessageId& msgId, proto::CommandAck_AckType ackType) { + const MessageId& msgId, CommandAck_AckType ackType) { auto cnx = connWeakPtr.lock(); if (cnx == nullptr) { LOG_DEBUG("Connection is not ready, ACK failed for message - [" << msgId.ledgerId() << ", " @@ -65,7 +55,7 @@ bool AckGroupingTracker::doImmediateAck(ClientConnectionWeakPtr connWeakPtr, uin } for (const auto& msgId : msgIds) { - sendAck(cnx, consumerId, msgId, proto::CommandAck::Individual); + sendAck(cnx, consumerId, msgId, CommandAck_AckType_Individual); } return true; } diff --git a/lib/AckGroupingTracker.h b/lib/AckGroupingTracker.h index f4410e45..d0e800db 100644 --- a/lib/AckGroupingTracker.h +++ b/lib/AckGroupingTracker.h @@ -19,17 +19,18 @@ #ifndef LIB_ACKGROUPINGTRACKER_H_ #define LIB_ACKGROUPINGTRACKER_H_ -#include +#include +#include #include -#include -#include "PulsarApi.pb.h" -#include "ClientConnection.h" -#include +#include "ProtoApiEnums.h" namespace pulsar { +class ClientConnection; +using ClientConnectionWeakPtr = std::weak_ptr; + /** * @class AckGroupingTracker * Default ACK grouping tracker, it actually neither tracks ACK requests nor sends them to brokers. @@ -93,7 +94,7 @@ class AckGroupingTracker : public std::enable_shared_from_this +#include "LogUtils.h" +#include "ProtoApiEnums.h" namespace pulsar { @@ -33,11 +33,11 @@ AckGroupingTrackerDisabled::AckGroupingTrackerDisabled(HandlerBase& handler, uin } void AckGroupingTrackerDisabled::addAcknowledge(const MessageId& msgId) { - this->doImmediateAck(this->handler_.getCnx(), this->consumerId_, msgId, proto::CommandAck::Individual); + this->doImmediateAck(this->handler_.getCnx(), this->consumerId_, msgId, CommandAck_AckType_Individual); } void AckGroupingTrackerDisabled::addAcknowledgeCumulative(const MessageId& msgId) { - this->doImmediateAck(this->handler_.getCnx(), this->consumerId_, msgId, proto::CommandAck::Cumulative); + this->doImmediateAck(this->handler_.getCnx(), this->consumerId_, msgId, CommandAck_AckType_Cumulative); } } // namespace pulsar diff --git a/lib/AckGroupingTrackerDisabled.h b/lib/AckGroupingTrackerDisabled.h index 6e66718a..ef6bfbed 100644 --- a/lib/AckGroupingTrackerDisabled.h +++ b/lib/AckGroupingTrackerDisabled.h @@ -21,12 +21,12 @@ #include -#include "HandlerBase.h" -#include #include "AckGroupingTracker.h" namespace pulsar { +class HandlerBase; + /** * @class AckGroupingTrackerDisabled * ACK grouping tracker that does not tracker or group ACK requests. The ACK requests are diretly diff --git a/lib/AckGroupingTrackerEnabled.cc b/lib/AckGroupingTrackerEnabled.cc index 5b6fe4e7..2683d281 100644 --- a/lib/AckGroupingTrackerEnabled.cc +++ b/lib/AckGroupingTrackerEnabled.cc @@ -20,14 +20,13 @@ #include "AckGroupingTrackerEnabled.h" #include -#include -#include "Commands.h" -#include "LogUtils.h" +#include "ClientConnection.h" #include "ClientImpl.h" +#include "Commands.h" +#include "ExecutorService.h" #include "HandlerBase.h" -#include "PulsarApi.pb.h" -#include +#include "LogUtils.h" namespace pulsar { @@ -111,7 +110,7 @@ void AckGroupingTrackerEnabled::flush() { std::lock_guard lock(this->mutexCumulativeAckMsgId_); if (this->requireCumulativeAck_) { if (!this->doImmediateAck(cnx, this->consumerId_, this->nextCumulativeAckMsgId_, - proto::CommandAck::Cumulative)) { + CommandAck_AckType_Cumulative)) { // Failed to send ACK. LOG_WARN("Failed to send cumulative ACK."); return; diff --git a/lib/AckGroupingTrackerEnabled.h b/lib/AckGroupingTrackerEnabled.h index c3926aa4..89b13f8e 100644 --- a/lib/AckGroupingTrackerEnabled.h +++ b/lib/AckGroupingTrackerEnabled.h @@ -19,18 +19,26 @@ #ifndef LIB_ACKGROUPINGTRACKERENABLED_H_ #define LIB_ACKGROUPINGTRACKERENABLED_H_ -#include +#include -#include +#include +#include #include +#include -#include "ClientImpl.h" -#include "HandlerBase.h" -#include #include "AckGroupingTracker.h" namespace pulsar { +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +using DeadlineTimerPtr = std::shared_ptr; +class ExecutorService; +using ExecutorServicePtr = std::shared_ptr; +class HandlerBase; +using HandlerBasePtr = std::shared_ptr; +using HandlerBaseWeakPtr = std::weak_ptr; + /** * @class AckGroupingTrackerEnabled * Ack grouping tracker for consumers of persistent topics that enabled ACK grouping. diff --git a/lib/Authentication.cc b/lib/Authentication.cc index 4695a03c..1bdac05b 100644 --- a/lib/Authentication.cc +++ b/lib/Authentication.cc @@ -16,23 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -#include - +#include #include -#include "auth/AuthTls.h" -#include "auth/AuthAthenz.h" -#include "auth/AuthToken.h" -#include "auth/AuthOauth2.h" -#include "auth/AuthBasic.h" -#include +#include +#include #include #include -#include -#include -#include -#include -#include + +#include "LogUtils.h" +#include "auth/AuthAthenz.h" +#include "auth/AuthBasic.h" +#include "auth/AuthOauth2.h" +#include "auth/AuthTls.h" +#include "auth/AuthToken.h" DECLARE_LOG_OBJECT() diff --git a/lib/Backoff.cc b/lib/Backoff.cc index 790d3f87..4d954220 100644 --- a/lib/Backoff.cc +++ b/lib/Backoff.cc @@ -17,10 +17,12 @@ * under the License. */ #include "Backoff.h" -#include -#include + #include /* time */ +#include +#include + namespace pulsar { Backoff::Backoff(const TimeDuration& initial, const TimeDuration& max, const TimeDuration& mandatoryStop) diff --git a/lib/Backoff.h b/lib/Backoff.h index 93b97adf..4bcebc75 100644 --- a/lib/Backoff.h +++ b/lib/Backoff.h @@ -18,13 +18,14 @@ */ #ifndef _PULSAR_BACKOFF_HEADER_ #define _PULSAR_BACKOFF_HEADER_ +#include + #include #include -#include namespace pulsar { -typedef boost::posix_time::time_duration TimeDuration; +using TimeDuration = boost::posix_time::time_duration; class PULSAR_PUBLIC Backoff { public: diff --git a/lib/BatchAcknowledgementTracker.cc b/lib/BatchAcknowledgementTracker.cc index 3d6d9208..1df4984f 100644 --- a/lib/BatchAcknowledgementTracker.cc +++ b/lib/BatchAcknowledgementTracker.cc @@ -18,6 +18,9 @@ */ #include "BatchAcknowledgementTracker.h" +#include "LogUtils.h" +#include "MessageImpl.h" + namespace pulsar { DECLARE_LOG_OBJECT() @@ -62,10 +65,9 @@ void BatchAcknowledgementTracker::receivedMessage(const Message& message) { TrackerPair(msgID, boost::dynamic_bitset<>(message.impl_->metadata.num_messages_in_batch()).set())); } -void BatchAcknowledgementTracker::deleteAckedMessage(const MessageId& messageId, - proto::CommandAck_AckType ackType) { +void BatchAcknowledgementTracker::deleteAckedMessage(const MessageId& messageId, CommandAck_AckType ackType) { // Not a batch message and a individual ack - if (messageId.batchIndex() == -1 && ackType == proto::CommandAck_AckType_Individual) { + if (messageId.batchIndex() == -1 && ackType == CommandAck_AckType_Individual) { return; } @@ -73,7 +75,7 @@ void BatchAcknowledgementTracker::deleteAckedMessage(const MessageId& messageId, MessageId(messageId.partition(), messageId.ledgerId(), messageId.entryId(), -1 /* Batch index */); Lock lock(mutex_); - if (ackType == proto::CommandAck_AckType_Cumulative) { + if (ackType == CommandAck_AckType_Cumulative) { // delete from trackerMap and sendList all messageIDs less than or equal to this one // equal to - since getGreatestCumulativeAckReady already gives us the exact message id to be acked @@ -110,8 +112,7 @@ void BatchAcknowledgementTracker::deleteAckedMessage(const MessageId& messageId, } } -bool BatchAcknowledgementTracker::isBatchReady(const MessageId& msgID, - const proto::CommandAck_AckType ackType) { +bool BatchAcknowledgementTracker::isBatchReady(const MessageId& msgID, CommandAck_AckType ackType) { Lock lock(mutex_); // Remove batch index MessageId batchMessageId = @@ -130,7 +131,7 @@ bool BatchAcknowledgementTracker::isBatchReady(const MessageId& msgID, assert(batchIndex < pos->second.size()); pos->second.set(batchIndex, false); - if (ackType == proto::CommandAck_AckType_Cumulative) { + if (ackType == CommandAck_AckType_Cumulative) { for (int i = 0; i < batchIndex; i++) { pos->second.set(i, false); } diff --git a/lib/BatchAcknowledgementTracker.h b/lib/BatchAcknowledgementTracker.h index 6a709b3e..6cbe7531 100644 --- a/lib/BatchAcknowledgementTracker.h +++ b/lib/BatchAcknowledgementTracker.h @@ -19,15 +19,17 @@ #ifndef LIB_BATCHACKNOWLEDGEMENTTRACKER_H_ #define LIB_BATCHACKNOWLEDGEMENTTRACKER_H_ -#include "MessageImpl.h" +#include +#include + +#include #include #include -#include -#include -#include -#include "LogUtils.h" +#include #include -#include + +#include "ProtoApiEnums.h" + namespace pulsar { class ConsumerImpl; @@ -57,10 +59,10 @@ class BatchAcknowledgementTracker { BatchAcknowledgementTracker(const std::string topic, const std::string subscription, const long consumerId); - bool isBatchReady(const MessageId& msgID, const proto::CommandAck_AckType ackType); + bool isBatchReady(const MessageId& msgID, CommandAck_AckType ackType); const MessageId getGreatestCumulativeAckReady(const MessageId& messageId); - void deleteAckedMessage(const MessageId& messageId, proto::CommandAck_AckType ackType); + void deleteAckedMessage(const MessageId& messageId, CommandAck_AckType ackType); void receivedMessage(const Message& message); void clear(); diff --git a/lib/BatchMessageContainer.cc b/lib/BatchMessageContainer.cc index e25b72ee..ae0425e2 100644 --- a/lib/BatchMessageContainer.cc +++ b/lib/BatchMessageContainer.cc @@ -17,14 +17,11 @@ * under the License. */ #include "BatchMessageContainer.h" -#include "ClientConnection.h" -#include "Commands.h" -#include "LogUtils.h" -#include "MessageImpl.h" -#include "ProducerImpl.h" -#include "TimeUtils.h" + #include +#include "LogUtils.h" + DECLARE_LOG_OBJECT() namespace pulsar { diff --git a/lib/BatchMessageContainerBase.cc b/lib/BatchMessageContainerBase.cc index e9e6b987..0cf338fb 100644 --- a/lib/BatchMessageContainerBase.cc +++ b/lib/BatchMessageContainerBase.cc @@ -17,11 +17,16 @@ * under the License. */ #include "BatchMessageContainerBase.h" + +#include "ClientConnection.h" +#include "CompressionCodec.h" +#include "MessageAndCallbackBatch.h" #include "MessageCrypto.h" #include "MessageImpl.h" +#include "OpSendMsg.h" #include "ProducerImpl.h" -#include "SharedBuffer.h" #include "PulsarApi.pb.h" +#include "SharedBuffer.h" namespace pulsar { @@ -55,7 +60,7 @@ Result BatchMessageContainerBase::createOpSendMsgHelper(OpSendMsg& opSendMsg, impl->metadata.set_num_messages_in_batch(batch.size()); auto compressionType = producerConfig_.getCompressionType(); if (compressionType != CompressionNone) { - impl->metadata.set_compression(CompressionCodecProvider::convertType(compressionType)); + impl->metadata.set_compression(static_cast(compressionType)); impl->metadata.set_uncompressed_size(impl->payload.readableBytes()); } impl->payload = CompressionCodecProvider::getCodec(compressionType).encode(impl->payload); @@ -83,4 +88,27 @@ Result BatchMessageContainerBase::createOpSendMsgHelper(OpSendMsg& opSendMsg, return ResultOk; } +void BatchMessageContainerBase::processAndClear( + std::function opSendMsgCallback, FlushCallback flushCallback) { + if (isEmpty()) { + if (flushCallback) { + flushCallback(ResultOk); + } + } else { + const auto numBatches = getNumBatches(); + if (numBatches == 1) { + OpSendMsg opSendMsg; + Result result = createOpSendMsg(opSendMsg, flushCallback); + opSendMsgCallback(result, opSendMsg); + } else if (numBatches > 1) { + std::vector opSendMsgs; + std::vector results = createOpSendMsgs(opSendMsgs, flushCallback); + for (size_t i = 0; i < results.size(); i++) { + opSendMsgCallback(results[i], opSendMsgs[i]); + } + } // else numBatches is 0, do nothing + } + clear(); +} + } // namespace pulsar diff --git a/lib/BatchMessageContainerBase.h b/lib/BatchMessageContainerBase.h index 71eef5fa..e9cf7ef5 100644 --- a/lib/BatchMessageContainerBase.h +++ b/lib/BatchMessageContainerBase.h @@ -19,24 +19,22 @@ #ifndef LIB_BATCHMESSAGECONTAINERBASE_H_ #define LIB_BATCHMESSAGECONTAINERBASE_H_ -#include #include -#include #include +#include +#include +#include #include #include -#include - -#include "MessageAndCallbackBatch.h" -#include "OpSendMsg.h" - namespace pulsar { class MessageCrypto; class ProducerImpl; class SharedBuffer; +struct OpSendMsg; +class MessageAndCallbackBatch; namespace proto { class MessageMetadata; @@ -160,29 +158,6 @@ inline void BatchMessageContainerBase::resetStats() { sizeInBytes_ = 0; } -inline void BatchMessageContainerBase::processAndClear( - std::function opSendMsgCallback, FlushCallback flushCallback) { - if (isEmpty()) { - if (flushCallback) { - flushCallback(ResultOk); - } - } else { - const auto numBatches = getNumBatches(); - if (numBatches == 1) { - OpSendMsg opSendMsg; - Result result = createOpSendMsg(opSendMsg, flushCallback); - opSendMsgCallback(result, opSendMsg); - } else if (numBatches > 1) { - std::vector opSendMsgs; - std::vector results = createOpSendMsgs(opSendMsgs, flushCallback); - for (size_t i = 0; i < results.size(); i++) { - opSendMsgCallback(results[i], opSendMsgs[i]); - } - } // else numBatches is 0, do nothing - } - clear(); -} - inline std::ostream& operator<<(std::ostream& os, const BatchMessageContainerBase& container) { container.serialize(os); return os; diff --git a/lib/BatchMessageKeyBasedContainer.cc b/lib/BatchMessageKeyBasedContainer.cc index 7441a3e4..05baf342 100644 --- a/lib/BatchMessageKeyBasedContainer.cc +++ b/lib/BatchMessageKeyBasedContainer.cc @@ -17,16 +17,18 @@ * under the License. */ #include "BatchMessageKeyBasedContainer.h" + +#include +#include + #include "ClientConnection.h" #include "Commands.h" #include "LogUtils.h" #include "MessageImpl.h" +#include "OpSendMsg.h" #include "ProducerImpl.h" #include "TimeUtils.h" -#include -#include - DECLARE_LOG_OBJECT() namespace pulsar { diff --git a/lib/BatchReceivePolicy.cc b/lib/BatchReceivePolicy.cc index 08aa3687..84876565 100644 --- a/lib/BatchReceivePolicy.cc +++ b/lib/BatchReceivePolicy.cc @@ -18,6 +18,7 @@ */ #include + #include "BatchReceivePolicyImpl.h" #include "LogUtils.h" diff --git a/lib/BinaryProtoLookupService.cc b/lib/BinaryProtoLookupService.cc index ff42b91b..b863d529 100644 --- a/lib/BinaryProtoLookupService.cc +++ b/lib/BinaryProtoLookupService.cc @@ -17,13 +17,13 @@ * under the License. */ #include "BinaryProtoLookupService.h" -#include "SharedBuffer.h" - -#include +#include "ClientConnection.h" #include "ConnectionPool.h" - -#include +#include "LogUtils.h" +#include "NamespaceName.h" +#include "ServiceNameResolver.h" +#include "TopicName.h" DECLARE_LOG_OBJECT() diff --git a/lib/BinaryProtoLookupService.h b/lib/BinaryProtoLookupService.h index d068c3d0..9adb6483 100644 --- a/lib/BinaryProtoLookupService.h +++ b/lib/BinaryProtoLookupService.h @@ -19,17 +19,19 @@ #ifndef _PULSAR_BINARY_LOOKUP_SERVICE_HEADER_ #define _PULSAR_BINARY_LOOKUP_SERVICE_HEADER_ -#include -#include #include -#include "ConnectionPool.h" -#include "Backoff.h" -#include + #include -#include "ServiceNameResolver.h" + +#include "LookupService.h" namespace pulsar { +class ClientConnection; +using ClientConnectionWeakPtr = std::weak_ptr; +class ConnectionPool; class LookupDataResult; +class ServiceNameResolver; +using NamespaceTopicsPromisePtr = std::shared_ptr>; class PULSAR_PUBLIC BinaryProtoLookupService : public LookupService { public: diff --git a/lib/BlockingQueue.h b/lib/BlockingQueue.h index d09166fd..ab1902b9 100644 --- a/lib/BlockingQueue.h +++ b/lib/BlockingQueue.h @@ -19,10 +19,9 @@ #ifndef LIB_BLOCKINGQUEUE_H_ #define LIB_BLOCKINGQUEUE_H_ -#include -#include -#include #include +#include +#include /** * Following structs are defined for holding a predicate in wait() call on condition variables. diff --git a/lib/BoostHash.h b/lib/BoostHash.h index 10d62e12..be48ce37 100644 --- a/lib/BoostHash.h +++ b/lib/BoostHash.h @@ -20,11 +20,12 @@ #define BOOST_HASH_HPP_ #include -#include "Hash.h" +#include #include #include -#include + +#include "Hash.h" namespace pulsar { class PULSAR_PUBLIC BoostHash : public Hash { diff --git a/lib/BrokerConsumerStats.cc b/lib/BrokerConsumerStats.cc index e3e1dea6..a403772b 100644 --- a/lib/BrokerConsumerStats.cc +++ b/lib/BrokerConsumerStats.cc @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include #include -#include +#include + +#include "BrokerConsumerStatsImplBase.h" namespace pulsar { BrokerConsumerStats::BrokerConsumerStats(std::shared_ptr impl) : impl_(impl) {} diff --git a/lib/BrokerConsumerStatsImpl.cc b/lib/BrokerConsumerStatsImpl.cc index 220415ad..2f7b6ada 100644 --- a/lib/BrokerConsumerStatsImpl.cc +++ b/lib/BrokerConsumerStatsImpl.cc @@ -16,7 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -#include +#include "BrokerConsumerStatsImpl.h" + #include namespace pulsar { diff --git a/lib/BrokerConsumerStatsImpl.h b/lib/BrokerConsumerStatsImpl.h index eb238c61..721af597 100644 --- a/lib/BrokerConsumerStatsImpl.h +++ b/lib/BrokerConsumerStatsImpl.h @@ -19,14 +19,14 @@ #ifndef PULSAR_CPP_BROKERCONSUMERSTATSIMPL_H #define PULSAR_CPP_BROKERCONSUMERSTATSIMPL_H -#include -#include +#include #include -#include + +#include #include -#include -#include -#include +#include + +#include "BrokerConsumerStatsImplBase.h" namespace pulsar { class PULSAR_PUBLIC BrokerConsumerStatsImpl : public BrokerConsumerStatsImplBase { diff --git a/lib/BrokerConsumerStatsImplBase.h b/lib/BrokerConsumerStatsImplBase.h index 282dfc0e..f2094a65 100644 --- a/lib/BrokerConsumerStatsImplBase.h +++ b/lib/BrokerConsumerStatsImplBase.h @@ -20,7 +20,6 @@ #define PULSAR_CPP_BROKERCONSUMERSTATSIMPLBASE_H #include -#include namespace pulsar { class BrokerConsumerStatsImplBase { diff --git a/lib/Client.cc b/lib/Client.cc index c72232a3..03823fb6 100644 --- a/lib/Client.cc +++ b/lib/Client.cc @@ -16,16 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -#include #include -#include +#include #include +#include #include "ClientImpl.h" -#include "Utils.h" -#include "ExecutorService.h" #include "LogUtils.h" +#include "Utils.h" DECLARE_LOG_OBJECT() diff --git a/lib/ClientConfiguration.cc b/lib/ClientConfiguration.cc index 1a161c35..70d85cb3 100644 --- a/lib/ClientConfiguration.cc +++ b/lib/ClientConfiguration.cc @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -#include +#include "ClientConfigurationImpl.h" namespace pulsar { diff --git a/lib/ClientConnection.cc b/lib/ClientConnection.cc index a037ff3e..b3df8310 100644 --- a/lib/ClientConnection.cc +++ b/lib/ClientConnection.cc @@ -18,38 +18,35 @@ */ #include "ClientConnection.h" -#include "PulsarApi.pb.h" - -#include -#include -#include -#include -#include +#include -#include "ExecutorService.h" #include "Commands.h" +#include "ConsumerImpl.h" +#include "ExecutorService.h" #include "LogUtils.h" -#include "Url.h" - -#include -#include - +#include "OpSendMsg.h" #include "ProducerImpl.h" -#include "ConsumerImpl.h" +#include "PulsarApi.pb.h" +#include "Url.h" #include "checksum/ChecksumProvider.h" -#include "MessageIdUtil.h" DECLARE_LOG_OBJECT() -using namespace pulsar::proto; using namespace boost::asio::ip; namespace pulsar { +using proto::BaseCommand; + static const uint32_t DefaultBufferSize = 64 * 1024; static const int KeepAliveIntervalInSeconds = 30; +static MessageId toMessageId(const proto::MessageIdData& messageIdData) { + return MessageId{messageIdData.partition(), static_cast(messageIdData.ledgerid()), + static_cast(messageIdData.entryid()), messageIdData.batch_index()}; +} + // Convert error codes from protobuf to client API Result static Result getResult(ServerError serverError, const std::string& message) { switch (serverError) { @@ -139,7 +136,7 @@ static Result getResult(ServerError serverError, const std::string& message) { return ResultUnknownError; } -inline std::ostream& operator<<(std::ostream& os, ServerError error) { +inline std::ostream& operator<<(std::ostream& os, proto::ServerError error) { os << getResult(error, ""); return os; } @@ -160,7 +157,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: const AuthenticationPtr& authentication) : operationsTimeout_(seconds(clientConfiguration.getOperationTimeoutSeconds())), authentication_(authentication), - serverProtocolVersion_(ProtocolVersion_MIN), + serverProtocolVersion_(proto::ProtocolVersion_MIN), executor_(executor), resolver_(executor_->createTcpResolver()), #if BOOST_VERSION >= 107000 @@ -268,7 +265,7 @@ ClientConnection::ClientConnection(const std::string& logicalAddress, const std: ClientConnection::~ClientConnection() { LOG_INFO(cnxString_ << "Destroyed connection"); } -void ClientConnection::handlePulsarConnected(const CommandConnected& cmdConnected) { +void ClientConnection::handlePulsarConnected(const proto::CommandConnected& cmdConnected) { if (!cmdConnected.has_server_version()) { LOG_ERROR(cnxString_ << "Server version is not set"); close(); @@ -286,7 +283,7 @@ void ClientConnection::handlePulsarConnected(const CommandConnected& cmdConnecte serverProtocolVersion_ = cmdConnected.protocol_version(); connectPromise_.setValue(shared_from_this()); - if (serverProtocolVersion_ >= v1) { + if (serverProtocolVersion_ >= proto::v1) { // Only send keep-alive probes if the broker supports it keepAliveTimer_ = executor_->createDeadlineTimer(); Lock lock(mutex_); @@ -298,7 +295,7 @@ void ClientConnection::handlePulsarConnected(const CommandConnected& cmdConnecte lock.unlock(); } - if (serverProtocolVersion_ >= v8) { + if (serverProtocolVersion_ >= proto::v8) { startConsumerStatsTimer(std::vector()); } } @@ -659,7 +656,8 @@ void ClientConnection::processIncomingBuffer() { // At this point, we have at least one complete frame available in the buffer uint32_t cmdSize = incomingBuffer_.readUnsignedInt(); - if (!incomingCmd_.ParseFromArray(incomingBuffer_.data(), cmdSize)) { + proto::BaseCommand incomingCmd; + if (!incomingCmd.ParseFromArray(incomingBuffer_.data(), cmdSize)) { LOG_ERROR(cnxString_ << "Error parsing protocol buffer command"); close(); return; @@ -667,20 +665,20 @@ void ClientConnection::processIncomingBuffer() { incomingBuffer_.consume(cmdSize); - if (incomingCmd_.type() == BaseCommand::MESSAGE) { + if (incomingCmd.type() == BaseCommand::MESSAGE) { // Parse message metadata and extract payload - MessageMetadata msgMetadata; + proto::MessageMetadata msgMetadata; // read checksum uint32_t remainingBytes = frameSize - (cmdSize + 4); - bool isChecksumValid = verifyChecksum(incomingBuffer_, remainingBytes, incomingCmd_); + bool isChecksumValid = verifyChecksum(incomingBuffer_, remainingBytes, incomingCmd); uint32_t metadataSize = incomingBuffer_.readUnsignedInt(); if (!msgMetadata.ParseFromArray(incomingBuffer_.data(), metadataSize)) { - LOG_ERROR(cnxString_ << "[consumer id " << incomingCmd_.message().consumer_id() // + LOG_ERROR(cnxString_ << "[consumer id " << incomingCmd.message().consumer_id() // << ", message ledger id " - << incomingCmd_.message().message_id().ledgerid() // - << ", entry id " << incomingCmd_.message().message_id().entryid() + << incomingCmd.message().message_id().ledgerid() // + << ", entry id " << incomingCmd.message().message_id().entryid() << "] Error parsing message metadata"); close(); return; @@ -692,9 +690,9 @@ void ClientConnection::processIncomingBuffer() { uint32_t payloadSize = remainingBytes; SharedBuffer payload = SharedBuffer::copy(incomingBuffer_.data(), payloadSize); incomingBuffer_.consume(payloadSize); - handleIncomingMessage(incomingCmd_.message(), isChecksumValid, msgMetadata, payload); + handleIncomingMessage(incomingCmd.message(), isChecksumValid, msgMetadata, payload); } else { - handleIncomingCommand(); + handleIncomingCommand(incomingCmd); } } if (incomingBuffer_.readableBytes() > 0) { @@ -722,7 +720,7 @@ void ClientConnection::processIncomingBuffer() { } bool ClientConnection::verifyChecksum(SharedBuffer& incomingBuffer_, uint32_t& remainingBytes, - proto::BaseCommand& incomingCmd_) { + proto::BaseCommand& incomingCmd) { int readerIndex = incomingBuffer_.readerIndex(); bool isChecksumValid = true; @@ -738,9 +736,9 @@ bool ClientConnection::verifyChecksum(SharedBuffer& incomingBuffer_, uint32_t& r if (!isChecksumValid) { LOG_ERROR("[consumer id " - << incomingCmd_.message().consumer_id() // - << ", message ledger id " << incomingCmd_.message().message_id().ledgerid() // - << ", entry id " << incomingCmd_.message().message_id().entryid() // + << incomingCmd.message().consumer_id() // + << ", message ledger id " << incomingCmd.message().message_id().ledgerid() // + << ", entry id " << incomingCmd.message().message_id().entryid() // << "stored-checksum" << storedChecksum << "computedChecksum" << computedChecksum // << "] Checksum verification failed"); } @@ -795,8 +793,8 @@ void ClientConnection::handleIncomingMessage(const proto::CommandMessage& msg, b } } -void ClientConnection::handleIncomingCommand() { - LOG_DEBUG(cnxString_ << "Handling incoming command: " << Commands::messageType(incomingCmd_.type())); +void ClientConnection::handleIncomingCommand(BaseCommand& incomingCmd) { + LOG_DEBUG(cnxString_ << "Handling incoming command: " << Commands::messageType(incomingCmd.type())); switch (state_) { case Pending: { @@ -806,11 +804,11 @@ void ClientConnection::handleIncomingCommand() { case TcpConnected: { // Handle Pulsar Connected - if (incomingCmd_.type() != BaseCommand::CONNECTED) { + if (incomingCmd.type() != BaseCommand::CONNECTED) { // Wrong cmd close(); } else { - handlePulsarConnected(incomingCmd_.connected()); + handlePulsarConnected(incomingCmd.connected()); } break; } @@ -826,9 +824,9 @@ void ClientConnection::handleIncomingCommand() { havePendingPingRequest_ = false; // Handle normal commands - switch (incomingCmd_.type()) { + switch (incomingCmd.type()) { case BaseCommand::SEND_RECEIPT: { - const CommandSendReceipt& sendReceipt = incomingCmd_.send_receipt(); + const auto& sendReceipt = incomingCmd.send_receipt(); int producerId = sendReceipt.producer_id(); uint64_t sequenceId = sendReceipt.sequence_id(); const proto::MessageIdData& messageIdData = sendReceipt.message_id(); @@ -860,7 +858,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::SEND_ERROR: { - const CommandSendError& error = incomingCmd_.send_error(); + const auto& error = incomingCmd.send_error(); LOG_WARN(cnxString_ << "Received send error from server: " << error.message()); if (ChecksumError == error.error()) { long producerId = error.producer_id(); @@ -886,7 +884,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::SUCCESS: { - const CommandSuccess& success = incomingCmd_.success(); + const auto& success = incomingCmd.success(); LOG_DEBUG(cnxString_ << "Received success response from server. req_id: " << success.request_id()); @@ -904,8 +902,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::PARTITIONED_METADATA_RESPONSE: { - const CommandPartitionedTopicMetadataResponse& partitionMetadataResponse = - incomingCmd_.partitionmetadataresponse(); + const auto& partitionMetadataResponse = incomingCmd.partitionmetadataresponse(); LOG_DEBUG(cnxString_ << "Received partition-metadata response from server. req_id: " << partitionMetadataResponse.request_id()); @@ -921,7 +918,7 @@ void ClientConnection::handleIncomingCommand() { if (!partitionMetadataResponse.has_response() || (partitionMetadataResponse.response() == - CommandPartitionedTopicMetadataResponse::Failed)) { + proto::CommandPartitionedTopicMetadataResponse::Failed)) { if (partitionMetadataResponse.has_error()) { LOG_ERROR(cnxString_ << "Failed partition-metadata lookup req_id: " << partitionMetadataResponse.request_id() @@ -950,8 +947,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::CONSUMER_STATS_RESPONSE: { - const CommandConsumerStatsResponse& consumerStatsResponse = - incomingCmd_.consumerstatsresponse(); + const auto& consumerStatsResponse = incomingCmd.consumerstatsresponse(); LOG_DEBUG(cnxString_ << "ConsumerStatsResponse command - Received consumer stats " "response from server. req_id: " << consumerStatsResponse.request_id()); @@ -994,8 +990,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::LOOKUP_RESPONSE: { - const CommandLookupTopicResponse& lookupTopicResponse = - incomingCmd_.lookuptopicresponse(); + const auto& lookupTopicResponse = incomingCmd.lookuptopicresponse(); LOG_DEBUG(cnxString_ << "Received lookup response from server. req_id: " << lookupTopicResponse.request_id()); @@ -1010,7 +1005,7 @@ void ClientConnection::handleIncomingCommand() { lock.unlock(); if (!lookupTopicResponse.has_response() || - (lookupTopicResponse.response() == CommandLookupTopicResponse::Failed)) { + (lookupTopicResponse.response() == proto::CommandLookupTopicResponse::Failed)) { if (lookupTopicResponse.has_error()) { LOG_ERROR(cnxString_ << "Failed lookup req_id: " << lookupTopicResponse.request_id() @@ -1045,7 +1040,7 @@ void ClientConnection::handleIncomingCommand() { lookupResultPtr->setBrokerUrlTls(lookupTopicResponse.brokerserviceurltls()); lookupResultPtr->setAuthoritative(lookupTopicResponse.authoritative()); lookupResultPtr->setRedirect(lookupTopicResponse.response() == - CommandLookupTopicResponse::Redirect); + proto::CommandLookupTopicResponse::Redirect); lookupResultPtr->setShouldProxyThroughServiceUrl( lookupTopicResponse.proxy_through_service_url()); lookupDataPromise->setValue(lookupResultPtr); @@ -1059,7 +1054,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::PRODUCER_SUCCESS: { - const CommandProducerSuccess& producerSuccess = incomingCmd_.producer_success(); + const auto& producerSuccess = incomingCmd.producer_success(); LOG_DEBUG(cnxString_ << "Received success producer response from server. req_id: " << producerSuccess.request_id() // << " -- producer name: " << producerSuccess.producer_name()); @@ -1089,7 +1084,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::ERROR: { - const CommandError& error = incomingCmd_.error(); + const auto& error = incomingCmd.error(); Result result = getResult(error.error(), error.message()); LOG_WARN(cnxString_ << "Received error response from server: " << result << (error.has_message() ? (" (" + error.message() + ")") : "") @@ -1132,7 +1127,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::CLOSE_PRODUCER: { - const CommandCloseProducer& closeProducer = incomingCmd_.close_producer(); + const auto& closeProducer = incomingCmd.close_producer(); int producerId = closeProducer.producer_id(); LOG_DEBUG("Broker notification of Closed producer: " << producerId); @@ -1156,7 +1151,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::CLOSE_CONSUMER: { - const CommandCloseConsumer& closeconsumer = incomingCmd_.close_consumer(); + const auto& closeconsumer = incomingCmd.close_consumer(); int consumerId = closeconsumer.consumer_id(); LOG_DEBUG("Broker notification of Closed consumer: " << consumerId); @@ -1208,7 +1203,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::ACTIVE_CONSUMER_CHANGE: { - const CommandActiveConsumerChange& change = incomingCmd_.active_consumer_change(); + const auto& change = incomingCmd.active_consumer_change(); LOG_DEBUG(cnxString_ << "Received notification about active consumer change, consumer_id: " << change.consumer_id() << " isActive: " << change.is_active()); @@ -1217,8 +1212,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::GET_LAST_MESSAGE_ID_RESPONSE: { - const CommandGetLastMessageIdResponse& getLastMessageIdResponse = - incomingCmd_.getlastmessageidresponse(); + const auto& getLastMessageIdResponse = incomingCmd.getlastmessageidresponse(); LOG_DEBUG(cnxString_ << "Received getLastMessageIdResponse from server. req_id: " << getLastMessageIdResponse.request_id()); @@ -1249,8 +1243,7 @@ void ClientConnection::handleIncomingCommand() { } case BaseCommand::GET_TOPICS_OF_NAMESPACE_RESPONSE: { - const CommandGetTopicsOfNamespaceResponse& response = - incomingCmd_.gettopicsofnamespaceresponse(); + const auto& response = incomingCmd.gettopicsofnamespaceresponse(); LOG_DEBUG(cnxString_ << "Received GetTopicsOfNamespaceResponse from server. req_id: " << response.request_id() << " topicsSize" << response.topics_size()); @@ -1405,8 +1398,9 @@ void ClientConnection::sendMessage(const OpSendMsg& opSend) { } void ClientConnection::sendMessageInternal(const OpSendMsg& opSend) { + BaseCommand outgoingCmd; PairSharedBuffer buffer = - Commands::newSend(outgoingBuffer_, outgoingCmd_, opSend.producerId_, opSend.sequenceId_, + Commands::newSend(outgoingBuffer_, outgoingCmd, opSend.producerId_, opSend.sequenceId_, getChecksumType(), opSend.metadata_, opSend.payload_); asyncWrite(buffer, customAllocWriteHandler(std::bind(&ClientConnection::handleSendPair, @@ -1448,8 +1442,9 @@ void ClientConnection::sendPendingCommands() { assert(any.type() == typeid(OpSendMsg)); const OpSendMsg& op = boost::any_cast(any); + BaseCommand outgoingCmd; PairSharedBuffer buffer = - Commands::newSend(outgoingBuffer_, outgoingCmd_, op.producerId_, op.sequenceId_, + Commands::newSend(outgoingBuffer_, outgoingCmd, op.producerId_, op.sequenceId_, getChecksumType(), op.metadata_, op.payload_); asyncWrite(buffer, customAllocWriteHandler(std::bind(&ClientConnection::handleSendPair, @@ -1697,7 +1692,7 @@ void ClientConnection::closeSocket() { } } -void ClientConnection::checkServerError(const proto::ServerError& error) { +void ClientConnection::checkServerError(ServerError error) { switch (error) { case proto::ServerError::ServiceNotReady: closeSocket(); diff --git a/lib/ClientConnection.h b/lib/ClientConnection.h index 8a48408e..ad5c3adf 100644 --- a/lib/ClientConnection.h +++ b/lib/ClientConnection.h @@ -19,42 +19,39 @@ #ifndef _PULSAR_CLIENT_CONNECTION_HEADER_ #define _PULSAR_CLIENT_CONNECTION_HEADER_ +#include #include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include -#include -#include -#include "ExecutorService.h" -#include "Future.h" -#include "PulsarApi.pb.h" -#include -#include "SharedBuffer.h" -#include "Backoff.h" #include "Commands.h" +#include "GetLastMessageIdResponse.h" #include "LookupDataResult.h" +#include "SharedBuffer.h" #include "UtilAllocator.h" -#include -#include -#include -#include "lib/PeriodicTask.h" -#include "lib/GetLastMessageIdResponse.h" - -using namespace pulsar; +#include "Utils.h" namespace pulsar { class PulsarFriend; +using DeadlineTimerPtr = std::shared_ptr; +using TimeDuration = boost::posix_time::time_duration; +using TcpResolverPtr = std::shared_ptr; + class ExecutorService; +using ExecutorServicePtr = std::shared_ptr; class ClientConnection; typedef std::shared_ptr ClientConnectionPtr; @@ -69,9 +66,18 @@ typedef std::shared_ptr ConsumerImplPtr; typedef std::weak_ptr ConsumerImplWeakPtr; class LookupDataResult; +class BrokerConsumerStatsImpl; +class PeriodicTask; struct OpSendMsg; +namespace proto { +class BaseCommand; +class CommandActiveConsumerChange; +class CommandMessage; +class CommandConnected; +} // namespace proto + // Data returned on the request operation. Mostly used on create-producer command struct ResponseData { std::string producerName; @@ -193,10 +199,10 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this connectPromise_; std::shared_ptr connectTimeoutTask_; @@ -322,7 +327,6 @@ class PULSAR_PUBLIC ClientConnection : public std::enable_shared_from_this + +#include + +#include "BinaryProtoLookupService.h" #include "ClientConfigurationImpl.h" -#include "LogUtils.h" +#include "Commands.h" #include "ConsumerImpl.h" -#include "ProducerImpl.h" -#include "ReaderImpl.h" -#include "PartitionedProducerImpl.h" +#include "ExecutorService.h" +#include "HTTPLookupService.h" +#include "LogUtils.h" #include "MultiTopicsConsumerImpl.h" +#include "PartitionedProducerImpl.h" #include "PatternMultiTopicsConsumerImpl.h" +#include "ProducerImpl.h" +#include "ReaderImpl.h" +#include "RetryableLookupService.h" #include "TimeUtils.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "TopicName.h" + #ifdef USE_LOG4CXX #include "Log4CxxLogger.h" #endif diff --git a/lib/ClientImpl.h b/lib/ClientImpl.h index 50ddeffe..8a393960 100644 --- a/lib/ClientImpl.h +++ b/lib/ClientImpl.h @@ -20,22 +20,21 @@ #define LIB_CLIENTIMPL_H_ #include -#include "ExecutorService.h" -#include "LookupService.h" -#include "MemoryLimitController.h" + +#include +#include + #include "ConnectionPool.h" +#include "Future.h" #include "LookupDataResult.h" -#include -#include -#include "ProducerImplBase.h" -#include -#include +#include "MemoryLimitController.h" #include "ServiceNameResolver.h" #include "SynchronizedHashMap.h" namespace pulsar { class PulsarFriend; +class ClientImpl; typedef std::shared_ptr ClientImplPtr; typedef std::weak_ptr ClientImplWeakPtr; @@ -46,6 +45,21 @@ typedef std::weak_ptr ReaderImplWeakPtr; class ConsumerImplBase; typedef std::weak_ptr ConsumerImplBaseWeakPtr; +class ClientConnection; +using ClientConnectionWeakPtr = std::weak_ptr; + +class LookupService; +using LookupServicePtr = std::shared_ptr; + +class ProducerImplBase; +using ProducerImplBaseWeakPtr = std::weak_ptr; +class ConsumerImplBase; +using ConsumerImplBaseWeakPtr = std::weak_ptr; +class TopicName; +using TopicNamePtr = std::shared_ptr; + +using NamespaceTopicsPtr = std::shared_ptr>; + std::string generateRandomName(); class ClientImpl : public std::enable_shared_from_this { diff --git a/lib/Commands.cc b/lib/Commands.cc index 26288d1e..13febd09 100644 --- a/lib/Commands.cc +++ b/lib/Commands.cc @@ -17,25 +17,51 @@ * under the License. */ #include "Commands.h" -#include "MessageImpl.h" -#include "pulsar/Version.h" -#include "pulsar/MessageBuilder.h" + +#include +#include +#include + +#include +#include + #include "LogUtils.h" +#include "MessageImpl.h" #include "PulsarApi.pb.h" -#include "Utils.h" #include "Url.h" -#include #include "checksum/ChecksumProvider.h" -#include -#include using namespace pulsar; namespace pulsar { -using namespace pulsar::proto; - DECLARE_LOG_OBJECT(); +using proto::AuthData; +using proto::BaseCommand; +using proto::CommandAck; +using proto::CommandAuthResponse; +using proto::CommandCloseConsumer; +using proto::CommandCloseProducer; +using proto::CommandConnect; +using proto::CommandConsumerStats; +using proto::CommandFlow; +using proto::CommandGetLastMessageId; +using proto::CommandGetTopicsOfNamespace; +using proto::CommandLookupTopic; +using proto::CommandPartitionedTopicMetadata; +using proto::CommandProducer; +using proto::CommandRedeliverUnacknowledgedMessages; +using proto::CommandSeek; +using proto::CommandSend; +using proto::CommandSubscribe; +using proto::CommandUnsubscribe; +using proto::FeatureFlags; +using proto::IntRange; +using proto::KeySharedMeta; +using proto::MessageIdData; +using proto::ProtocolVersion_MAX; +using proto::SingleMessageMetadata; + static inline bool isBuiltInSchema(SchemaType schemaType) { switch (schemaType) { case STRING: @@ -53,19 +79,19 @@ static inline bool isBuiltInSchema(SchemaType schemaType) { static inline proto::Schema_Type getSchemaType(SchemaType type) { switch (type) { case SchemaType::NONE: - return Schema_Type_None; + return proto::Schema_Type_None; case STRING: - return Schema_Type_String; + return proto::Schema_Type_String; case JSON: - return Schema_Type_Json; + return proto::Schema_Type_Json; case PROTOBUF: - return Schema_Type_Protobuf; + return proto::Schema_Type_Protobuf; case AVRO: - return Schema_Type_Avro; + return proto::Schema_Type_Avro; case PROTOBUF_NATIVE: - return Schema_Type_ProtobufNative; + return proto::Schema_Type_ProtobufNative; default: - return Schema_Type_None; + return proto::Schema_Type_None; } } @@ -278,13 +304,14 @@ SharedBuffer Commands::newSubscribe(const std::string& topic, const std::string& CommandSubscribe* subscribe = cmd.mutable_subscribe(); subscribe->set_topic(topic); subscribe->set_subscription(subscription); - subscribe->set_subtype(subType); + subscribe->set_subtype(static_cast(subType)); subscribe->set_consumer_id(consumerId); subscribe->set_request_id(requestId); subscribe->set_consumer_name(consumerName); subscribe->set_durable(subscriptionMode == SubscriptionModeDurable); subscribe->set_read_compacted(readCompacted); - subscribe->set_initialposition(subscriptionInitialPosition); + subscribe->set_initialposition( + static_cast(subscriptionInitialPosition)); subscribe->set_replicate_subscription_state(replicateSubscriptionState); subscribe->set_priority_level(priorityLevel); @@ -362,7 +389,7 @@ SharedBuffer Commands::newProducer(const std::string& topic, uint64_t producerId producer->set_epoch(epoch); producer->set_user_provided_producer_name(userProvidedProducerName); producer->set_encrypted(encrypted); - producer->set_producer_access_mode(accessMode); + producer->set_producer_access_mode(static_cast(accessMode)); if (topicEpoch.is_present()) { producer->set_topic_epoch(topicEpoch.value()); } @@ -386,17 +413,19 @@ SharedBuffer Commands::newProducer(const std::string& topic, uint64_t producerId return writeMessageWithSize(cmd); } -SharedBuffer Commands::newAck(uint64_t consumerId, const MessageIdData& messageId, CommandAck_AckType ackType, - int validationError) { +SharedBuffer Commands::newAck(uint64_t consumerId, int64_t ledgerId, int64_t entryId, + CommandAck_AckType ackType, CommandAck_ValidationError validationError) { BaseCommand cmd; cmd.set_type(BaseCommand::ACK); CommandAck* ack = cmd.mutable_ack(); ack->set_consumer_id(consumerId); - ack->set_ack_type(ackType); - if (CommandAck_AckType_IsValid(validationError)) { - ack->set_validation_error((CommandAck_ValidationError)validationError); + ack->set_ack_type(static_cast(ackType)); + if (proto::CommandAck_AckType_IsValid(validationError)) { + ack->set_validation_error((proto::CommandAck_ValidationError)validationError); } - *(ack->add_message_id()) = messageId; + auto* msgId = ack->add_message_id(); + msgId->set_ledgerid(ledgerId); + msgId->set_entryid(entryId); return writeMessageWithSize(cmd); } @@ -405,7 +434,7 @@ SharedBuffer Commands::newMultiMessageAck(uint64_t consumerId, const std::setset_consumer_id(consumerId); - ack->set_ack_type(CommandAck_AckType_Individual); + ack->set_ack_type(proto::CommandAck_AckType_Individual); for (const auto& msgId : msgIds) { auto newMsgId = ack->add_message_id(); newMsgId->set_ledgerid(msgId.ledgerId()); diff --git a/lib/Commands.h b/lib/Commands.h index 4ff86744..09f6f8be 100644 --- a/lib/Commands.h +++ b/lib/Commands.h @@ -20,21 +20,27 @@ #define LIB_COMMANDS_H_ #include -#include +#include #include #include -#include +#include -#include "PulsarApi.pb.h" +#include + +#include "ProtoApiEnums.h" #include "SharedBuffer.h" #include "Utils.h" -#include - using namespace pulsar; namespace pulsar { +namespace proto { +class BaseCommand; +class MessageIdData; +class MessageMetadata; +} // namespace proto + typedef std::shared_ptr MessageMetadataPtr; /** @@ -85,12 +91,12 @@ class Commands { static SharedBuffer newSubscribe(const std::string& topic, const std::string& subscription, uint64_t consumerId, uint64_t requestId, - proto::CommandSubscribe_SubType subType, const std::string& consumerName, + CommandSubscribe_SubType subType, const std::string& consumerName, SubscriptionMode subscriptionMode, Optional startMessageId, bool readCompacted, const std::map& metadata, const std::map& subscriptionProperties, const SchemaInfo& schemaInfo, - proto::CommandSubscribe_InitialPosition subscriptionInitialPosition, + CommandSubscribe_InitialPosition subscriptionInitialPosition, bool replicateSubscriptionState, KeySharedPolicy keySharedPolicy, int priorityLevel = 0); @@ -101,10 +107,10 @@ class Commands { const std::map& metadata, const SchemaInfo& schemaInfo, uint64_t epoch, bool userProvidedProducerName, bool encrypted, - proto::ProducerAccessMode accessMode, Optional topicEpoch); + ProducerAccessMode accessMode, Optional topicEpoch); - static SharedBuffer newAck(uint64_t consumerId, const proto::MessageIdData& messageId, - proto::CommandAck_AckType ackType, int validationError); + static SharedBuffer newAck(uint64_t consumerId, int64_t ledgerId, int64_t entryId, + CommandAck_AckType ackType, CommandAck_ValidationError validationError); static SharedBuffer newMultiMessageAck(uint64_t consumerId, const std::set& msgIds); static SharedBuffer newFlow(uint64_t consumerId, uint32_t messagePermits); @@ -119,7 +125,7 @@ class Commands { static SharedBuffer newRedeliverUnacknowledgedMessages(uint64_t consumerId, const std::set& messageIds); - static std::string messageType(proto::BaseCommand::Type type); + static std::string messageType(BaseCommand_Type type); static void initBatchMessageMetadata(const Message& msg, pulsar::proto::MessageMetadata& batchMetadata); diff --git a/lib/CompressionCodec.cc b/lib/CompressionCodec.cc index c17b5344..991d52c0 100644 --- a/lib/CompressionCodec.cc +++ b/lib/CompressionCodec.cc @@ -17,12 +17,11 @@ * under the License. */ #include "CompressionCodec.h" + #include "CompressionCodecLZ4.h" +#include "CompressionCodecSnappy.h" #include "CompressionCodecZLib.h" #include "CompressionCodecZstd.h" -#include "CompressionCodecSnappy.h" - -#include using namespace pulsar; namespace pulsar { @@ -49,38 +48,6 @@ CompressionCodec& CompressionCodecProvider::getCodec(CompressionType compression BOOST_THROW_EXCEPTION(std::logic_error("Invalid CompressionType enumeration value")); } -CompressionType CompressionCodecProvider::convertType(proto::CompressionType type) { - switch (type) { - case proto::NONE: - return CompressionNone; - case proto::LZ4: - return CompressionLZ4; - case proto::ZLIB: - return CompressionZLib; - case proto::ZSTD: - return CompressionZSTD; - case proto::SNAPPY: - return CompressionSNAPPY; - } - BOOST_THROW_EXCEPTION(std::logic_error("Invalid proto::CompressionType enumeration value")); -} - -proto::CompressionType CompressionCodecProvider::convertType(CompressionType type) { - switch (type) { - case CompressionNone: - return proto::NONE; - case CompressionLZ4: - return proto::LZ4; - case CompressionZLib: - return proto::ZLIB; - case CompressionZSTD: - return proto::ZSTD; - case CompressionSNAPPY: - return proto::SNAPPY; - } - BOOST_THROW_EXCEPTION(std::logic_error("Invalid CompressionType enumeration value")); -} - SharedBuffer CompressionCodecNone::encode(const SharedBuffer& raw) { return raw; } bool CompressionCodecNone::decode(const SharedBuffer& encoded, uint32_t uncompressedSize, diff --git a/lib/CompressionCodec.h b/lib/CompressionCodec.h index fd65f9cd..9fe36803 100644 --- a/lib/CompressionCodec.h +++ b/lib/CompressionCodec.h @@ -19,14 +19,12 @@ #ifndef LIB_COMPRESSIONCODEC_H_ #define LIB_COMPRESSIONCODEC_H_ -#include #include -#include "SharedBuffer.h" -#include "PulsarApi.pb.h" - #include +#include "SharedBuffer.h" + using namespace pulsar; namespace pulsar { @@ -39,9 +37,6 @@ class CompressionCodecSnappy; class PULSAR_PUBLIC CompressionCodecProvider { public: - static CompressionType convertType(proto::CompressionType type); - static proto::CompressionType convertType(CompressionType type); - static CompressionCodec& getCodec(CompressionType compressionType); private: diff --git a/lib/CompressionCodecLZ4.cc b/lib/CompressionCodecLZ4.cc index 508e4f4a..587849b3 100644 --- a/lib/CompressionCodecLZ4.cc +++ b/lib/CompressionCodecLZ4.cc @@ -18,9 +18,10 @@ */ #include "CompressionCodecLZ4.h" -#include "lz4/lz4.h" #include +#include "lz4/lz4.h" + namespace pulsar { SharedBuffer CompressionCodecLZ4::encode(const SharedBuffer& raw) { diff --git a/lib/CompressionCodecSnappy.cc b/lib/CompressionCodecSnappy.cc index 04b0d973..efd9f381 100644 --- a/lib/CompressionCodecSnappy.cc +++ b/lib/CompressionCodecSnappy.cc @@ -19,8 +19,8 @@ #include "CompressionCodecSnappy.h" #if HAS_SNAPPY -#include #include +#include namespace pulsar { diff --git a/lib/CompressionCodecZLib.cc b/lib/CompressionCodecZLib.cc index 657c5488..3c42f0a9 100644 --- a/lib/CompressionCodecZLib.cc +++ b/lib/CompressionCodecZLib.cc @@ -19,9 +19,7 @@ #include "CompressionCodecZLib.h" #include -#include -#include -#include + #include "LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/lib/CompressionCodecZLib.h b/lib/CompressionCodecZLib.h index cd4380b5..d03d8800 100644 --- a/lib/CompressionCodecZLib.h +++ b/lib/CompressionCodecZLib.h @@ -19,9 +19,7 @@ #ifndef LIB_COMPRESSIONCODECZLIB_H_ #define LIB_COMPRESSIONCODECZLIB_H_ -#include #include "CompressionCodec.h" -#include // Make symbol visible to unit tests diff --git a/lib/ConnectionPool.cc b/lib/ConnectionPool.cc index e03697f9..1c246d64 100644 --- a/lib/ConnectionPool.cc +++ b/lib/ConnectionPool.cc @@ -18,12 +18,13 @@ */ #include "ConnectionPool.h" -#include "LogUtils.h" -#include "Url.h" - -#include +#include #include +#include "ClientConnection.h" +#include "ExecutorService.h" +#include "LogUtils.h" + using boost::asio::ip::tcp; namespace ssl = boost::asio::ssl; typedef ssl::stream ssl_socket; diff --git a/lib/ConnectionPool.h b/lib/ConnectionPool.h index 996df54e..cd55fc2a 100644 --- a/lib/ConnectionPool.h +++ b/lib/ConnectionPool.h @@ -19,18 +19,24 @@ #ifndef _PULSAR_CONNECTION_POOL_HEADER_ #define _PULSAR_CONNECTION_POOL_HEADER_ -#include +#include #include - -#include "ClientConnection.h" +#include #include -#include #include +#include #include +#include + +#include "Future.h" namespace pulsar { +class ClientConnection; +using ClientConnectionWeakPtr = std::weak_ptr; class ExecutorService; +class ExecutorServiceProvider; +using ExecutorServiceProviderPtr = std::shared_ptr; class PULSAR_PUBLIC ConnectionPool { public: diff --git a/lib/ConsoleLoggerFactory.cc b/lib/ConsoleLoggerFactory.cc index 397c7fee..7c5df5ef 100644 --- a/lib/ConsoleLoggerFactory.cc +++ b/lib/ConsoleLoggerFactory.cc @@ -18,7 +18,8 @@ */ #include -#include "lib/ConsoleLoggerFactoryImpl.h" + +#include "ConsoleLoggerFactoryImpl.h" namespace pulsar { diff --git a/lib/ConsoleLoggerFactoryImpl.h b/lib/ConsoleLoggerFactoryImpl.h index 61c1d90d..7808547f 100644 --- a/lib/ConsoleLoggerFactoryImpl.h +++ b/lib/ConsoleLoggerFactoryImpl.h @@ -20,7 +20,8 @@ #pragma once #include -#include "lib/SimpleLogger.h" + +#include "SimpleLogger.h" namespace pulsar { diff --git a/lib/Consumer.cc b/lib/Consumer.cc index 13fb9f4a..8e5c7ddf 100644 --- a/lib/Consumer.cc +++ b/lib/Consumer.cc @@ -16,12 +16,14 @@ * specific language governing permissions and limitations * under the License. */ +#include #include +#include #include + #include "ConsumerImpl.h" +#include "GetLastMessageIdResponse.h" #include "Utils.h" -#include -#include namespace pulsar { diff --git a/lib/ConsumerConfiguration.cc b/lib/ConsumerConfiguration.cc index 0705cca3..62982458 100644 --- a/lib/ConsumerConfiguration.cc +++ b/lib/ConsumerConfiguration.cc @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -#include +#include #include -#include + +#include "ConsumerConfigurationImpl.h" namespace pulsar { diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 7be5a6aa..155c5bf5 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -17,21 +17,30 @@ * under the License. */ #include "ConsumerImpl.h" -#include "MessageImpl.h" -#include "MessagesImpl.h" + +#include + +#include "AckGroupingTracker.h" +#include "AckGroupingTrackerDisabled.h" +#include "AckGroupingTrackerEnabled.h" +#include "ClientConnection.h" +#include "ClientImpl.h" #include "Commands.h" +#include "ExecutorService.h" +#include "GetLastMessageIdResponse.h" #include "LogUtils.h" +#include "MessageCrypto.h" +#include "MessageIdUtil.h" +#include "MessageImpl.h" +#include "MessagesImpl.h" +#include "PulsarApi.pb.h" #include "TimeUtils.h" -#include -#include "pulsar/Result.h" -#include "pulsar/MessageId.h" +#include "TopicName.h" +#include "UnAckedMessageTrackerDisabled.h" +#include "UnAckedMessageTrackerEnabled.h" #include "Utils.h" -#include "MessageIdUtil.h" -#include "AckGroupingTracker.h" -#include "AckGroupingTrackerEnabled.h" -#include "AckGroupingTrackerDisabled.h" -#include -#include +#include "stats/ConsumerStatsDisabled.h" +#include "stats/ConsumerStatsImpl.h" namespace pulsar { @@ -396,7 +405,7 @@ void ConsumerImpl::messageReceived(const ClientConnectionPtr& cnx, const proto:: if (!isChecksumValid) { // Message discarded for checksum error - discardCorruptedMessage(cnx, msg.message_id(), proto::CommandAck::ChecksumMismatch); + discardCorruptedMessage(cnx, msg.message_id(), CommandAck_ValidationError_ChecksumMismatch); return; } @@ -613,7 +622,7 @@ bool ConsumerImpl::decryptMessageIfNeeded(const ClientConnectionPtr& cnx, const } else if (config_.getCryptoFailureAction() == ConsumerCryptoFailureAction::DISCARD) { LOG_WARN(getName() << "Skipping decryption since CryptoKeyReader is not implemented and config " "is set to discard"); - discardCorruptedMessage(cnx, msg.message_id(), proto::CommandAck::DecryptionError); + discardCorruptedMessage(cnx, msg.message_id(), CommandAck_ValidationError_DecryptionError); } else { LOG_ERROR(getName() << "Message delivery failed since CryptoKeyReader is not implemented to " "consume encrypted message"); @@ -634,7 +643,7 @@ bool ConsumerImpl::decryptMessageIfNeeded(const ClientConnectionPtr& cnx, const return true; } else if (config_.getCryptoFailureAction() == ConsumerCryptoFailureAction::DISCARD) { LOG_WARN(getName() << "Discarding message since decryption failed and config is set to discard"); - discardCorruptedMessage(cnx, msg.message_id(), proto::CommandAck::DecryptionError); + discardCorruptedMessage(cnx, msg.message_id(), CommandAck_ValidationError_DecryptionError); } else { LOG_ERROR(getName() << "Message delivery failed since unable to decrypt incoming message"); } @@ -649,7 +658,7 @@ bool ConsumerImpl::uncompressMessageIfNeeded(const ClientConnectionPtr& cnx, return true; } - CompressionType compressionType = CompressionCodecProvider::convertType(metadata.compression()); + CompressionType compressionType = static_cast(metadata.compression()); uint32_t uncompressedSize = metadata.uncompressed_size(); uint32_t payloadSize = payload.readableBytes(); @@ -658,7 +667,8 @@ bool ConsumerImpl::uncompressMessageIfNeeded(const ClientConnectionPtr& cnx, // Uncompressed size is itself corrupted since it cannot be bigger than the MaxMessageSize LOG_ERROR(getName() << "Got corrupted payload message size " << payloadSize // << " at " << messageIdData.ledgerid() << ":" << messageIdData.entryid()); - discardCorruptedMessage(cnx, messageIdData, proto::CommandAck::UncompressedSizeCorruption); + discardCorruptedMessage(cnx, messageIdData, + CommandAck_ValidationError_UncompressedSizeCorruption); return false; } } else { @@ -669,7 +679,7 @@ bool ConsumerImpl::uncompressMessageIfNeeded(const ClientConnectionPtr& cnx, if (!CompressionCodecProvider::getCodec(compressionType).decode(payload, uncompressedSize, payload)) { LOG_ERROR(getName() << "Failed to decompress message with " << uncompressedSize // << " at " << messageIdData.ledgerid() << ":" << messageIdData.entryid()); - discardCorruptedMessage(cnx, messageIdData, proto::CommandAck::DecompressionError); + discardCorruptedMessage(cnx, messageIdData, CommandAck_ValidationError_DecompressionError); return false; } @@ -678,12 +688,12 @@ bool ConsumerImpl::uncompressMessageIfNeeded(const ClientConnectionPtr& cnx, void ConsumerImpl::discardCorruptedMessage(const ClientConnectionPtr& cnx, const proto::MessageIdData& messageId, - proto::CommandAck::ValidationError validationError) { + CommandAck_ValidationError validationError) { LOG_ERROR(getName() << "Discarding corrupted message at " << messageId.ledgerid() << ":" << messageId.entryid()); - SharedBuffer cmd = - Commands::newAck(consumerId_, messageId, proto::CommandAck::Individual, validationError); + SharedBuffer cmd = Commands::newAck(consumerId_, messageId.ledgerid(), messageId.entryid(), + CommandAck_AckType_Individual, validationError); cnx->sendCommand(cmd); increaseAvailablePermits(cnx); @@ -899,37 +909,37 @@ void ConsumerImpl::increaseAvailablePermits(const ClientConnectionPtr& currentCn } } -inline proto::CommandSubscribe_SubType ConsumerImpl::getSubType() { +inline CommandSubscribe_SubType ConsumerImpl::getSubType() { ConsumerType type = config_.getConsumerType(); switch (type) { case ConsumerExclusive: - return proto::CommandSubscribe::Exclusive; + return CommandSubscribe_SubType_Exclusive; case ConsumerShared: - return proto::CommandSubscribe::Shared; + return CommandSubscribe_SubType_Shared; case ConsumerFailover: - return proto::CommandSubscribe::Failover; + return CommandSubscribe_SubType_Failover; case ConsumerKeyShared: - return proto::CommandSubscribe_SubType_Key_Shared; + return CommandSubscribe_SubType_Key_Shared; } BOOST_THROW_EXCEPTION(std::logic_error("Invalid ConsumerType enumeration value")); } -inline proto::CommandSubscribe_InitialPosition ConsumerImpl::getInitialPosition() { +inline CommandSubscribe_InitialPosition ConsumerImpl::getInitialPosition() { InitialPosition initialPosition = config_.getSubscriptionInitialPosition(); switch (initialPosition) { case InitialPositionLatest: - return proto::CommandSubscribe_InitialPosition::CommandSubscribe_InitialPosition_Latest; + return CommandSubscribe_InitialPosition_Latest; case InitialPositionEarliest: - return proto::CommandSubscribe_InitialPosition::CommandSubscribe_InitialPosition_Earliest; + return CommandSubscribe_InitialPosition_Earliest; } BOOST_THROW_EXCEPTION(std::logic_error("Invalid InitialPosition enumeration value")); } -void ConsumerImpl::statsCallback(Result res, ResultCallback callback, proto::CommandAck_AckType ackType) { +void ConsumerImpl::statsCallback(Result res, ResultCallback callback, CommandAck_AckType ackType) { consumerStatsBasePtr_->messageAcknowledged(res, ackType); if (callback) { callback(res); @@ -938,9 +948,9 @@ void ConsumerImpl::statsCallback(Result res, ResultCallback callback, proto::Com void ConsumerImpl::acknowledgeAsync(const MessageId& msgId, ResultCallback callback) { ResultCallback cb = std::bind(&ConsumerImpl::statsCallback, get_shared_this_ptr(), std::placeholders::_1, - callback, proto::CommandAck_AckType_Individual); + callback, CommandAck_AckType_Individual); if (msgId.batchIndex() != -1 && - !batchAcknowledgementTracker_.isBatchReady(msgId, proto::CommandAck_AckType_Individual)) { + !batchAcknowledgementTracker_.isBatchReady(msgId, CommandAck_AckType_Individual)) { cb(ResultOk); return; } @@ -949,13 +959,13 @@ void ConsumerImpl::acknowledgeAsync(const MessageId& msgId, ResultCallback callb void ConsumerImpl::acknowledgeCumulativeAsync(const MessageId& msgId, ResultCallback callback) { ResultCallback cb = std::bind(&ConsumerImpl::statsCallback, get_shared_this_ptr(), std::placeholders::_1, - callback, proto::CommandAck_AckType_Cumulative); + callback, CommandAck_AckType_Cumulative); if (!isCumulativeAcknowledgementAllowed(config_.getConsumerType())) { cb(ResultCumulativeAcknowledgementNotAllowedError); return; } if (msgId.batchIndex() != -1 && - !batchAcknowledgementTracker_.isBatchReady(msgId, proto::CommandAck_AckType_Cumulative)) { + !batchAcknowledgementTracker_.isBatchReady(msgId, CommandAck_AckType_Cumulative)) { MessageId messageId = batchAcknowledgementTracker_.getGreatestCumulativeAckReady(msgId); if (messageId == MessageId()) { // Nothing to ACK, because the batch that msgId belongs to is NOT completely consumed. diff --git a/lib/ConsumerImpl.h b/lib/ConsumerImpl.h index 3aa632a7..d65676ba 100644 --- a/lib/ConsumerImpl.h +++ b/lib/ConsumerImpl.h @@ -19,46 +19,45 @@ #ifndef LIB_CONSUMERIMPL_H_ #define LIB_CONSUMERIMPL_H_ -#include +#include -#include "pulsar/Result.h" -#include "UnboundedBlockingQueue.h" -#include "HandlerBase.h" -#include "ClientConnection.h" -#include "lib/UnAckedMessageTrackerEnabled.h" -#include "NegativeAcksTracker.h" -#include "Commands.h" -#include "ExecutorService.h" -#include "ConsumerImplBase.h" -#include "lib/UnAckedMessageTrackerDisabled.h" -#include "MessageCrypto.h" -#include "AckGroupingTracker.h" -#include "GetLastMessageIdResponse.h" +#include +#include -#include "CompressionCodec.h" -#include -#include #include "BatchAcknowledgementTracker.h" -#include -#include -#include -#include -#include -#include -#include +#include "BrokerConsumerStatsImpl.h" +#include "Commands.h" +#include "CompressionCodec.h" +#include "ConsumerImplBase.h" +#include "MapCache.h" +#include "NegativeAcksTracker.h" #include "Synchronized.h" - -using namespace pulsar; +#include "TestUtil.h" +#include "UnboundedBlockingQueue.h" namespace pulsar { -class UnAckedMessageTracker; +class UnAckedMessageTrackerInterface; class ExecutorService; class ConsumerImpl; class BatchAcknowledgementTracker; +class MessageCrypto; +class GetLastMessageIdResponse; typedef std::shared_ptr MessageCryptoPtr; typedef std::function BrokerGetLastMessageIdCallback; typedef std::shared_ptr BackoffPtr; +class AckGroupingTracker; +using AckGroupingTrackerPtr = std::shared_ptr; +class ConsumerStatsBase; +using ConsumerStatsBasePtr = std::shared_ptr; +class UnAckedMessageTracker; +using UnAckedMessageTrackerPtr = std::shared_ptr; + +namespace proto { +class CommandMessage; +class MessageMetadata; +} // namespace proto + enum ConsumerTopicType { NonPartitioned, @@ -82,8 +81,8 @@ class ConsumerImpl : public ConsumerImplBase { bool& isChecksumValid, proto::MessageMetadata& msgMetadata, SharedBuffer& payload); void messageProcessed(Message& msg, bool track = true); void activeConsumerChanged(bool isActive); - inline proto::CommandSubscribe_SubType getSubType(); - inline proto::CommandSubscribe_InitialPosition getInitialPosition(); + inline CommandSubscribe_SubType getSubType(); + inline CommandSubscribe_InitialPosition getInitialPosition(); /** * Send individual ACK request of given message ID to broker. @@ -167,7 +166,7 @@ class ConsumerImpl : public ConsumerImplBase { const proto::MessageMetadata& metadata, SharedBuffer& payload, bool checkMaxMessageSize); void discardCorruptedMessage(const ClientConnectionPtr& cnx, const proto::MessageIdData& messageId, - proto::CommandAck::ValidationError validationError); + CommandAck_ValidationError validationError); void increaseAvailablePermits(const ClientConnectionPtr& currentCnx, int delta = 1); void drainIncomingMessageQueue(size_t count); uint32_t receiveIndividualMessagesFromBatch(const ClientConnectionPtr& cnx, Message& batchedMessage, @@ -182,7 +181,7 @@ class ConsumerImpl : public ConsumerImplBase { // TODO - Convert these functions to lambda when we move to C++11 Result receiveHelper(Message& msg); Result receiveHelper(Message& msg, int timeout); - void statsCallback(Result, ResultCallback, proto::CommandAck_AckType); + void statsCallback(Result, ResultCallback, CommandAck_AckType); void executeNotifyCallback(Message& msg); void notifyPendingReceivedCallback(Result result, Message& message, const ReceiveCallback& callback); void failPendingReceiveCallback(); diff --git a/lib/ConsumerImplBase.cc b/lib/ConsumerImplBase.cc index 4a8c0276..6c86aa44 100644 --- a/lib/ConsumerImplBase.cc +++ b/lib/ConsumerImplBase.cc @@ -16,18 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -#include "ConsumerImpl.h" -#include "MessageImpl.h" -#include "MessagesImpl.h" -#include "LogUtils.h" -#include "TimeUtils.h" -#include "pulsar/Result.h" -#include "MessageIdUtil.h" -#include "AckGroupingTracker.h" #include "ConsumerImplBase.h" #include +#include "ConsumerImpl.h" +#include "ExecutorService.h" +#include "LogUtils.h" +#include "TimeUtils.h" + DECLARE_LOG_OBJECT() namespace pulsar { diff --git a/lib/ConsumerImplBase.h b/lib/ConsumerImplBase.h index 18b8bc1c..37b66462 100644 --- a/lib/ConsumerImplBase.h +++ b/lib/ConsumerImplBase.h @@ -18,17 +18,18 @@ */ #ifndef PULSAR_CONSUMER_IMPL_BASE_HEADER #define PULSAR_CONSUMER_IMPL_BASE_HEADER -#include #include -#include "HandlerBase.h" +#include + #include #include +#include "Future.h" +#include "HandlerBase.h" + namespace pulsar { class ConsumerImplBase; -class HandlerBase; - -typedef std::weak_ptr ConsumerImplBaseWeakPtr; +using ConsumerImplBaseWeakPtr = std::weak_ptr; class OpBatchReceive { public: diff --git a/lib/CryptoKeyReader.cc b/lib/CryptoKeyReader.cc index 1eb73e8f..b64e8654 100644 --- a/lib/CryptoKeyReader.cc +++ b/lib/CryptoKeyReader.cc @@ -16,12 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include -#include #include +#include #include +#include +#include + using namespace pulsar; CryptoKeyReader::CryptoKeyReader() {} @@ -77,4 +78,4 @@ Result DefaultCryptoKeyReader::getPrivateKey(const std::string& keyName, CryptoKeyReaderPtr DefaultCryptoKeyReader::create(const std::string& publicKeyPath, const std::string& privateKeyPath) { return CryptoKeyReaderPtr(new DefaultCryptoKeyReader(publicKeyPath, privateKeyPath)); -} \ No newline at end of file +} diff --git a/lib/DeprecatedException.cc b/lib/DeprecatedException.cc index 283d8bb6..4a5b7bd4 100644 --- a/lib/DeprecatedException.cc +++ b/lib/DeprecatedException.cc @@ -23,4 +23,4 @@ const std::string DeprecatedException::message_prefix = "Deprecated: "; DeprecatedException::DeprecatedException(const std::string& __arg) : std::runtime_error(message_prefix + __arg) {} -} // namespace pulsar \ No newline at end of file +} // namespace pulsar diff --git a/lib/EncryptionKeyInfoImpl.h b/lib/EncryptionKeyInfoImpl.h index 0470d1cb..5ff4ceb0 100644 --- a/lib/EncryptionKeyInfoImpl.h +++ b/lib/EncryptionKeyInfoImpl.h @@ -19,15 +19,16 @@ #ifndef LIB_ENCRYPTIONKEYINFOIMPL_H_ #define LIB_ENCRYPTIONKEYINFOIMPL_H_ +#include + #include #include -#include namespace pulsar { class PULSAR_PUBLIC EncryptionKeyInfoImpl { public: - typedef std::map StringMap; + using StringMap = std::map; EncryptionKeyInfoImpl() = default; diff --git a/lib/ExecutorService.cc b/lib/ExecutorService.cc index a7390f19..3a1d35d8 100644 --- a/lib/ExecutorService.cc +++ b/lib/ExecutorService.cc @@ -18,12 +18,8 @@ */ #include "ExecutorService.h" -#include -#include -#include -#include "TimeUtils.h" - #include "LogUtils.h" +#include "TimeUtils.h" DECLARE_LOG_OBJECT() namespace pulsar { diff --git a/lib/ExecutorService.h b/lib/ExecutorService.h index e4cbb3ce..5a32c1b1 100644 --- a/lib/ExecutorService.h +++ b/lib/ExecutorService.h @@ -19,16 +19,19 @@ #ifndef _PULSAR_EXECUTOR_SERVICE_HEADER_ #define _PULSAR_EXECUTOR_SERVICE_HEADER_ +#include + #include -#include -#include -#include -#include +#include +#include +#include #include +#include +#include #include -#include +#include #include -#include +#include namespace pulsar { typedef std::shared_ptr SocketPtr; diff --git a/lib/FileLoggerFactory.cc b/lib/FileLoggerFactory.cc index a82613f0..41ea24a8 100644 --- a/lib/FileLoggerFactory.cc +++ b/lib/FileLoggerFactory.cc @@ -17,7 +17,8 @@ * under the License. */ #include -#include "lib/FileLoggerFactoryImpl.h" + +#include "FileLoggerFactoryImpl.h" namespace pulsar { diff --git a/lib/FileLoggerFactoryImpl.h b/lib/FileLoggerFactoryImpl.h index 75329c65..2b877a26 100644 --- a/lib/FileLoggerFactoryImpl.h +++ b/lib/FileLoggerFactoryImpl.h @@ -18,12 +18,13 @@ */ #pragma once +#include + #include #include #include -#include -#include "lib/SimpleLogger.h" +#include "SimpleLogger.h" namespace pulsar { diff --git a/lib/Future.h b/lib/Future.h index 6754c890..35930576 100644 --- a/lib/Future.h +++ b/lib/Future.h @@ -19,14 +19,13 @@ #ifndef LIB_FUTURE_H_ #define LIB_FUTURE_H_ -#include -#include -#include #include - +#include #include +#include +#include -typedef std::unique_lock Lock; +using Lock = std::unique_lock; namespace pulsar { diff --git a/lib/GetLastMessageIdResponse.h b/lib/GetLastMessageIdResponse.h index 0acb7839..1ff7933e 100644 --- a/lib/GetLastMessageIdResponse.h +++ b/lib/GetLastMessageIdResponse.h @@ -19,6 +19,7 @@ #pragma once #include + #include namespace pulsar { diff --git a/lib/HTTPLookupService.cc b/lib/HTTPLookupService.cc index 91f5d795..8167b641 100644 --- a/lib/HTTPLookupService.cc +++ b/lib/HTTPLookupService.cc @@ -16,12 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -#include +#include "HTTPLookupService.h" #include +#include #include #include + +#include "ExecutorService.h" +#include "LogUtils.h" +#include "NamespaceName.h" +#include "ServiceNameResolver.h" +#include "TopicName.h" namespace ptree = boost::property_tree; DECLARE_LOG_OBJECT() diff --git a/lib/HTTPLookupService.h b/lib/HTTPLookupService.h index c9dfc57a..929d7ab1 100644 --- a/lib/HTTPLookupService.h +++ b/lib/HTTPLookupService.h @@ -19,13 +19,16 @@ #ifndef PULSAR_CPP_HTTPLOOKUPSERVICE_H #define PULSAR_CPP_HTTPLOOKUPSERVICE_H -#include -#include -#include -#include -#include +#include "ClientImpl.h" +#include "LookupService.h" +#include "Url.h" namespace pulsar { + +class ServiceNameResolver; +using NamespaceTopicsPromise = Promise; +using NamespaceTopicsPromisePtr = std::shared_ptr; + class HTTPLookupService : public LookupService, public std::enable_shared_from_this { class CurlInitializer { public: diff --git a/lib/HandlerBase.cc b/lib/HandlerBase.cc index 1f4ce6e9..0989eacc 100644 --- a/lib/HandlerBase.cc +++ b/lib/HandlerBase.cc @@ -17,11 +17,12 @@ * under the License. */ #include "HandlerBase.h" -#include "TimeUtils.h" - -#include +#include "ClientConnection.h" +#include "ClientImpl.h" +#include "ExecutorService.h" #include "LogUtils.h" +#include "TimeUtils.h" DECLARE_LOG_OBJECT() diff --git a/lib/HandlerBase.h b/lib/HandlerBase.h index 6616ec40..4a5df5c7 100644 --- a/lib/HandlerBase.h +++ b/lib/HandlerBase.h @@ -18,13 +18,14 @@ */ #ifndef _PULSAR_HANDLER_BASE_HEADER_ #define _PULSAR_HANDLER_BASE_HEADER_ -#include "Backoff.h" -#include "ClientImpl.h" -#include "ClientConnection.h" +#include + +#include #include -#include +#include #include -#include + +#include "Backoff.h" namespace pulsar { @@ -35,6 +36,15 @@ using boost::posix_time::seconds; class HandlerBase; typedef std::weak_ptr HandlerBaseWeakPtr; typedef std::shared_ptr HandlerBasePtr; +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +using ClientImplWeakPtr = std::weak_ptr; +class ClientConnection; +using ClientConnectionPtr = std::shared_ptr; +using ClientConnectionWeakPtr = std::weak_ptr; +class ExecutorService; +using ExecutorServicePtr = std::shared_ptr; +using DeadlineTimerPtr = std::shared_ptr; class HandlerBase { public: diff --git a/lib/JavaStringHash.cc b/lib/JavaStringHash.cc index bf809bf4..2dcbfc2c 100644 --- a/lib/JavaStringHash.cc +++ b/lib/JavaStringHash.cc @@ -17,6 +17,7 @@ * under the License. */ #include "JavaStringHash.h" + #include namespace pulsar { diff --git a/lib/JavaStringHash.h b/lib/JavaStringHash.h index 6059b1a0..1a66110e 100644 --- a/lib/JavaStringHash.h +++ b/lib/JavaStringHash.h @@ -20,11 +20,12 @@ #define JAVA_DEFAULT_HASH_HPP_ #include -#include "Hash.h" #include #include +#include "Hash.h" + namespace pulsar { class PULSAR_PUBLIC JavaStringHash : public Hash { public: diff --git a/lib/KeySharedPolicy.cc b/lib/KeySharedPolicy.cc index e23a942c..6c3e36ae 100644 --- a/lib/KeySharedPolicy.cc +++ b/lib/KeySharedPolicy.cc @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -#include - #include #include +#include "KeySharedPolicyImpl.h" + namespace pulsar { static const int DefaultHashRangeSize = 2 << 15; diff --git a/lib/Latch.h b/lib/Latch.h index f5b711bc..749ce296 100644 --- a/lib/Latch.h +++ b/lib/Latch.h @@ -19,10 +19,11 @@ #ifndef LIB_LATCH_H_ #define LIB_LATCH_H_ +#include + +#include #include #include -#include -#include namespace pulsar { diff --git a/lib/Log4CxxLogger.h b/lib/Log4CxxLogger.h index cd2fe9e7..2e3f819a 100644 --- a/lib/Log4CxxLogger.h +++ b/lib/Log4CxxLogger.h @@ -19,8 +19,8 @@ #pragma once -#include #include +#include #ifdef USE_LOG4CXX diff --git a/lib/Log4cxxLogger.cc b/lib/Log4cxxLogger.cc index fdd0395d..fc5ae5bd 100644 --- a/lib/Log4cxxLogger.cc +++ b/lib/Log4cxxLogger.cc @@ -18,15 +18,16 @@ */ #include "Log4CxxLogger.h" + #include #ifdef USE_LOG4CXX +#include #include #include -#include -#include #include +#include using namespace log4cxx; diff --git a/lib/LogUtils.cc b/lib/LogUtils.cc index 31746087..6e8a866c 100644 --- a/lib/LogUtils.cc +++ b/lib/LogUtils.cc @@ -18,9 +18,10 @@ */ #include "LogUtils.h" +#include + #include #include -#include #include "Log4CxxLogger.h" diff --git a/lib/LogUtils.h b/lib/LogUtils.h index 67ddf431..7cfad5b3 100644 --- a/lib/LogUtils.h +++ b/lib/LogUtils.h @@ -19,12 +19,12 @@ #pragma once -#include -#include -#include - -#include #include +#include + +#include +#include +#include namespace pulsar { diff --git a/lib/LookupDataResult.h b/lib/LookupDataResult.h index b48b8545..81e50ccd 100644 --- a/lib/LookupDataResult.h +++ b/lib/LookupDataResult.h @@ -18,12 +18,13 @@ */ #ifndef _PULSAR_LOOKUP_DATA_RESULT_HEADER_ #define _PULSAR_LOOKUP_DATA_RESULT_HEADER_ -#include -#include #include #include #include +#include + +#include "Future.h" namespace pulsar { class LookupDataResult; @@ -67,7 +68,7 @@ class LookupDataResult { bool proxyThroughServiceUrl_; }; -std::ostream& operator<<(std::ostream& os, const LookupDataResult& b) { +inline std::ostream& operator<<(std::ostream& os, const LookupDataResult& b) { os << "{ LookupDataResult [brokerUrl_ = " << b.brokerUrl_ << "] [brokerUrlTls_ = " << b.brokerUrlTls_ << "] [partitions = " << b.partitions << "] [authoritative = " << b.authoritative << "] [redirect = " << b.redirect << "] proxyThroughServiceUrl = " << b.proxyThroughServiceUrl_ diff --git a/lib/LookupService.h b/lib/LookupService.h index 50f2d84f..6af290ce 100644 --- a/lib/LookupService.h +++ b/lib/LookupService.h @@ -19,19 +19,21 @@ #ifndef PULSAR_CPP_LOOKUPSERVICE_H #define PULSAR_CPP_LOOKUPSERVICE_H -#include #include -#include -#include -#include -#include +#include +#include #include +#include "Future.h" +#include "LookupDataResult.h" + namespace pulsar { -typedef std::shared_ptr> NamespaceTopicsPtr; -typedef Promise NamespaceTopicsPromise; -typedef std::shared_ptr> NamespaceTopicsPromisePtr; +using NamespaceTopicsPtr = std::shared_ptr>; +class TopicName; +using TopicNamePtr = std::shared_ptr; +class NamespaceName; +using NamespaceNamePtr = std::shared_ptr; class LookupService { public: diff --git a/lib/MemoryLimitController.h b/lib/MemoryLimitController.h index 38987ea0..30a18365 100644 --- a/lib/MemoryLimitController.h +++ b/lib/MemoryLimitController.h @@ -44,4 +44,4 @@ class MemoryLimitController { bool isClosed_ = false; }; -} // namespace pulsar \ No newline at end of file +} // namespace pulsar diff --git a/lib/Message.cc b/lib/Message.cc index b928945c..cb7a75e3 100644 --- a/lib/Message.cc +++ b/lib/Message.cc @@ -16,17 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -#include #include #include +#include -#include "PulsarApi.pb.h" +#include #include "MessageImpl.h" +#include "PulsarApi.pb.h" #include "SharedBuffer.h" -#include - using namespace pulsar; namespace pulsar { diff --git a/lib/MessageAndCallbackBatch.cc b/lib/MessageAndCallbackBatch.cc index 3e229b0f..3f50dc02 100644 --- a/lib/MessageAndCallbackBatch.cc +++ b/lib/MessageAndCallbackBatch.cc @@ -17,6 +17,7 @@ * under the License. */ #include "MessageAndCallbackBatch.h" + #include "ClientConnection.h" #include "Commands.h" #include "LogUtils.h" diff --git a/lib/MessageAndCallbackBatch.h b/lib/MessageAndCallbackBatch.h index 38c0d12e..3d107c63 100644 --- a/lib/MessageAndCallbackBatch.h +++ b/lib/MessageAndCallbackBatch.h @@ -19,13 +19,12 @@ #ifndef LIB_MESSAGEANDCALLBACK_BATCH_H_ #define LIB_MESSAGEANDCALLBACK_BATCH_H_ -#include -#include - #include #include +#include #include +#include namespace pulsar { diff --git a/lib/MessageBuilder.cc b/lib/MessageBuilder.cc index 977331b9..7d8d8cb7 100644 --- a/lib/MessageBuilder.cc +++ b/lib/MessageBuilder.cc @@ -25,14 +25,13 @@ #include "LogUtils.h" #include "MessageImpl.h" +#include "ObjectPool.h" #include "PulsarApi.pb.h" #include "SharedBuffer.h" +#include "TimeUtils.h" DECLARE_LOG_OBJECT() -#include "ObjectPool.h" -#include "TimeUtils.h" - using namespace pulsar; namespace pulsar { diff --git a/lib/MessageCrypto.cc b/lib/MessageCrypto.cc index 8798dbf8..bab96d1e 100644 --- a/lib/MessageCrypto.cc +++ b/lib/MessageCrypto.cc @@ -17,9 +17,13 @@ * under the License. */ -#include "LogUtils.h" #include "MessageCrypto.h" +#include + +#include "LogUtils.h" +#include "PulsarApi.pb.h" + namespace pulsar { DECLARE_LOG_OBJECT() @@ -335,9 +339,10 @@ bool MessageCrypto::encrypt(const std::set& encKeys, const CryptoKe return true; } -bool MessageCrypto::decryptDataKey(const std::string& keyName, const std::string& encryptedDataKey, - const google::protobuf::RepeatedPtrField& encKeyMeta, - const CryptoKeyReaderPtr keyReader) { +bool MessageCrypto::decryptDataKey(const proto::EncryptionKeys& encKeys, const CryptoKeyReader& keyReader) { + const auto& keyName = encKeys.key(); + const auto& encryptedDataKey = encKeys.value(); + const auto& encKeyMeta = encKeys.metadata(); StringMap keyMeta; for (auto iter = encKeyMeta.begin(); iter != encKeyMeta.end(); iter++) { keyMeta[iter->key()] = iter->value(); @@ -345,7 +350,7 @@ bool MessageCrypto::decryptDataKey(const std::string& keyName, const std::string // Read the private key info using callback EncryptionKeyInfo keyInfo; - keyReader->getPrivateKey(keyName, keyMeta, keyInfo); + keyReader.getPrivateKey(keyName, keyMeta, keyInfo); // Convert key from string to RSA key RSA* privKey = loadPrivateKey(keyInfo.getKey()); @@ -498,10 +503,7 @@ bool MessageCrypto::decrypt(const proto::MessageMetadata& msgMetadata, SharedBuf bool isDataKeyDecrypted = false; for (int index = 0; index < msgMetadata.encryption_keys_size(); index++) { const proto::EncryptionKeys& encKeys = msgMetadata.encryption_keys(index); - - const std::string& encDataKey = encKeys.value(); - const google::protobuf::RepeatedPtrField& encKeyMeta = encKeys.metadata(); - if (decryptDataKey(encKeys.key(), encDataKey, encKeyMeta, keyReader)) { + if (decryptDataKey(encKeys, *keyReader)) { isDataKeyDecrypted = true; break; } diff --git a/lib/MessageCrypto.h b/lib/MessageCrypto.h index 21720665..fd139c1e 100644 --- a/lib/MessageCrypto.h +++ b/lib/MessageCrypto.h @@ -19,26 +19,31 @@ #ifndef LIB_MESSAGECRYPTO_H_ #define LIB_MESSAGECRYPTO_H_ -#include -#include -#include -#include -#include - -#include -#include #include +#include #include +#include #include -#include +#include +#include + +#include +#include +#include +#include +#include +#include #include "SharedBuffer.h" -#include "ExecutorService.h" -#include "pulsar/CryptoKeyReader.h" -#include "PulsarApi.pb.h" namespace pulsar { +namespace proto { +class EncryptionKeys; +class MessageMetadata; +class KeyValue; +} // namespace proto + class MessageCrypto { public: typedef std::map StringMap; @@ -128,9 +133,7 @@ class MessageCrypto { Result addPublicKeyCipher(const std::string& keyName, const CryptoKeyReaderPtr keyReader); - bool decryptDataKey(const std::string& keyName, const std::string& encryptedDataKey, - const google::protobuf::RepeatedPtrField& encKeyMeta, - const CryptoKeyReaderPtr keyReader); + bool decryptDataKey(const proto::EncryptionKeys& encKeys, const CryptoKeyReader& keyReader); bool decryptData(const std::string& dataKeySecret, const proto::MessageMetadata& msgMetadata, SharedBuffer& payload, SharedBuffer& decPayload); bool getKeyAndDecryptData(const proto::MessageMetadata& msgMetadata, SharedBuffer& payload, diff --git a/lib/MessageId.cc b/lib/MessageId.cc index 31b01548..5b133282 100644 --- a/lib/MessageId.cc +++ b/lib/MessageId.cc @@ -17,18 +17,15 @@ * under the License. */ -#include #include -#include "PulsarApi.pb.h" -#include "MessageIdImpl.h" - #include #include -#include -#include -#include #include +#include + +#include "MessageIdImpl.h" +#include "PulsarApi.pb.h" namespace pulsar { diff --git a/lib/MessageIdImpl.h b/lib/MessageIdImpl.h index ae33da4c..9db758c5 100644 --- a/lib/MessageIdImpl.h +++ b/lib/MessageIdImpl.h @@ -20,6 +20,7 @@ #pragma once #include +#include namespace pulsar { diff --git a/lib/MessageIdUtil.h b/lib/MessageIdUtil.h index d6f80a10..1f4ffd36 100644 --- a/lib/MessageIdUtil.h +++ b/lib/MessageIdUtil.h @@ -17,15 +17,9 @@ * under the License. */ #include -#include "PulsarApi.pb.h" namespace pulsar { -inline MessageId toMessageId(const proto::MessageIdData& messageIdData) { - return MessageId{messageIdData.partition(), static_cast(messageIdData.ledgerid()), - static_cast(messageIdData.entryid()), messageIdData.batch_index()}; -} - namespace internal { template static int compare(T lhs, T rhs) { diff --git a/lib/MessageImpl.h b/lib/MessageImpl.h index c9a37f43..587b6638 100644 --- a/lib/MessageImpl.h +++ b/lib/MessageImpl.h @@ -21,8 +21,9 @@ #include #include -#include "SharedBuffer.h" + #include "PulsarApi.pb.h" +#include "SharedBuffer.h" using namespace pulsar; namespace pulsar { diff --git a/lib/MessageRouterBase.cc b/lib/MessageRouterBase.cc index c0824f9e..8338c57d 100644 --- a/lib/MessageRouterBase.cc +++ b/lib/MessageRouterBase.cc @@ -37,4 +37,4 @@ MessageRouterBase::MessageRouterBase(ProducerConfiguration::HashingScheme hashin break; } } -} // namespace pulsar \ No newline at end of file +} // namespace pulsar diff --git a/lib/MessageRouterBase.h b/lib/MessageRouterBase.h index 39374a18..0b290fdf 100644 --- a/lib/MessageRouterBase.h +++ b/lib/MessageRouterBase.h @@ -19,13 +19,14 @@ #ifndef PULSAR_CPP_MESSAGEROUTERBASE_H #define PULSAR_CPP_MESSAGEROUTERBASE_H -#include - #include #include -#include "Hash.h" + +#include namespace pulsar { +class Hash; +using HashPtr = std::unique_ptr; typedef std::unique_ptr HashPtr; class MessageRouterBase : public MessageRoutingPolicy { diff --git a/lib/MessagesImpl.cc b/lib/MessagesImpl.cc index 7d45cddc..022f25fc 100644 --- a/lib/MessagesImpl.cc +++ b/lib/MessagesImpl.cc @@ -17,7 +17,8 @@ * under the License. */ #include "MessagesImpl.h" -#include "stdexcept" + +#include MessagesImpl::MessagesImpl(int maxNumberOfMessages, long maxSizeOfMessages) : maxNumberOfMessages_(maxNumberOfMessages), diff --git a/lib/MessagesImpl.h b/lib/MessagesImpl.h index 0c12768f..3ffaf6b1 100644 --- a/lib/MessagesImpl.h +++ b/lib/MessagesImpl.h @@ -19,9 +19,10 @@ #ifndef PULSAR_CPP_MESSAGESIMPL_H #define PULSAR_CPP_MESSAGESIMPL_H -#include #include +#include + using namespace pulsar; namespace pulsar { diff --git a/lib/MultiTopicsBrokerConsumerStatsImpl.cc b/lib/MultiTopicsBrokerConsumerStatsImpl.cc index 5220307b..4f969222 100644 --- a/lib/MultiTopicsBrokerConsumerStatsImpl.cc +++ b/lib/MultiTopicsBrokerConsumerStatsImpl.cc @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include +#include "MultiTopicsBrokerConsumerStatsImpl.h" + #include #include +#include using namespace pulsar; diff --git a/lib/MultiTopicsBrokerConsumerStatsImpl.h b/lib/MultiTopicsBrokerConsumerStatsImpl.h index a76ecdce..481318e5 100644 --- a/lib/MultiTopicsBrokerConsumerStatsImpl.h +++ b/lib/MultiTopicsBrokerConsumerStatsImpl.h @@ -19,14 +19,11 @@ #ifndef PULSAR_CPP_MULTITOPICSBROKERCONSUMERSTATSIMPL_H #define PULSAR_CPP_MULTITOPICSBROKERCONSUMERSTATSIMPL_H -#include +#include #include #include -#include -#include -#include -#include -#include + +#include "BrokerConsumerStatsImplBase.h" namespace pulsar { class PULSAR_PUBLIC MultiTopicsBrokerConsumerStatsImpl : public BrokerConsumerStatsImplBase { diff --git a/lib/MultiTopicsConsumerImpl.cc b/lib/MultiTopicsConsumerImpl.cc index c54f8e8e..a170faf6 100644 --- a/lib/MultiTopicsConsumerImpl.cc +++ b/lib/MultiTopicsConsumerImpl.cc @@ -17,14 +17,35 @@ * under the License. */ #include "MultiTopicsConsumerImpl.h" -#include "MultiResultCallback.h" -#include "MessagesImpl.h" + #include +#include "ClientImpl.h" +#include "ConsumerImpl.h" +#include "ExecutorService.h" +#include "LogUtils.h" +#include "LookupService.h" +#include "MessageImpl.h" +#include "MessagesImpl.h" +#include "MultiResultCallback.h" +#include "MultiTopicsBrokerConsumerStatsImpl.h" +#include "TopicName.h" +#include "UnAckedMessageTrackerDisabled.h" +#include "UnAckedMessageTrackerEnabled.h" + DECLARE_LOG_OBJECT() using namespace pulsar; +MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, TopicNamePtr topicName, + int numPartitions, const std::string& subscriptionName, + const ConsumerConfiguration& conf, + LookupServicePtr lookupServicePtr) + : MultiTopicsConsumerImpl(client, {topicName->toString()}, subscriptionName, topicName, conf, + lookupServicePtr) { + topicsPartitions_[topicName->toString()] = numPartitions; +} + MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std::vector& topics, const std::string& subscriptionName, TopicNamePtr topicName, const ConsumerConfiguration& conf, diff --git a/lib/MultiTopicsConsumerImpl.h b/lib/MultiTopicsConsumerImpl.h index 7c83da9d..2b4f83df 100644 --- a/lib/MultiTopicsConsumerImpl.h +++ b/lib/MultiTopicsConsumerImpl.h @@ -18,38 +18,46 @@ */ #ifndef PULSAR_MULTI_TOPICS_CONSUMER_HEADER #define PULSAR_MULTI_TOPICS_CONSUMER_HEADER -#include "lib/TestUtil.h" -#include "ConsumerImpl.h" -#include "ClientImpl.h" -#include "BlockingQueue.h" + +#include + +#include #include -#include -#include +#include "BlockingQueue.h" #include "ConsumerImplBase.h" -#include "lib/UnAckedMessageTrackerDisabled.h" -#include -#include -#include -#include -#include +#include "Future.h" +#include "Latch.h" +#include "LookupDataResult.h" +#include "SynchronizedHashMap.h" +#include "TestUtil.h" namespace pulsar { typedef std::shared_ptr> ConsumerSubResultPromisePtr; +class ConsumerImpl; +using ConsumerImplPtr = std::shared_ptr; +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +class TopicName; +using TopicNamePtr = std::shared_ptr; +class MultiTopicsBrokerConsumerStatsImpl; +using MultiTopicsBrokerConsumerStatsPtr = std::shared_ptr; +class UnAckedMessageTrackerInterface; +using UnAckedMessageTrackerPtr = std::shared_ptr; +class LookupService; +using LookupServicePtr = std::shared_ptr; + class MultiTopicsConsumerImpl; class MultiTopicsConsumerImpl : public ConsumerImplBase { public: + MultiTopicsConsumerImpl(ClientImplPtr client, TopicNamePtr topicName, int numPartitions, + const std::string& subscriptionName, const ConsumerConfiguration& conf, + LookupServicePtr lookupServicePtr); MultiTopicsConsumerImpl(ClientImplPtr client, const std::vector& topics, const std::string& subscriptionName, TopicNamePtr topicName, const ConsumerConfiguration& conf, LookupServicePtr lookupServicePtr_); - MultiTopicsConsumerImpl(ClientImplPtr client, TopicNamePtr topicName, int numPartitions, - const std::string& subscriptionName, const ConsumerConfiguration& conf, - LookupServicePtr lookupServicePtr) - : MultiTopicsConsumerImpl(client, {topicName->toString()}, subscriptionName, topicName, conf, - lookupServicePtr) { - topicsPartitions_[topicName->toString()] = numPartitions; - } + ~MultiTopicsConsumerImpl(); // overrided methods from ConsumerImplBase Future getConsumerCreatedFuture() override; diff --git a/lib/Murmur3_32Hash.h b/lib/Murmur3_32Hash.h index 50e6f164..d83635d2 100644 --- a/lib/Murmur3_32Hash.h +++ b/lib/Murmur3_32Hash.h @@ -25,11 +25,12 @@ #define MURMUR3_32_HASH_HPP_ #include -#include "Hash.h" #include #include +#include "Hash.h" + namespace pulsar { class PULSAR_PUBLIC Murmur3_32Hash : public Hash { diff --git a/lib/NamespaceName.cc b/lib/NamespaceName.cc index 02bde00b..f493db25 100644 --- a/lib/NamespaceName.cc +++ b/lib/NamespaceName.cc @@ -17,14 +17,14 @@ * under the License. */ #include "NamespaceName.h" -#include "NamedEntity.h" -#include "LogUtils.h" -#include -#include -#include #include +#include #include +#include + +#include "LogUtils.h" +#include "NamedEntity.h" DECLARE_LOG_OBJECT() namespace pulsar { diff --git a/lib/NamespaceName.h b/lib/NamespaceName.h index 86ffc2f4..ce451a20 100644 --- a/lib/NamespaceName.h +++ b/lib/NamespaceName.h @@ -20,11 +20,12 @@ #define _PULSAR_NAMESPACE_NAME_HEADER_ #include -#include "ServiceUnitId.h" #include #include +#include "ServiceUnitId.h" + namespace pulsar { class PULSAR_PUBLIC NamespaceName : public ServiceUnitId { diff --git a/lib/NegativeAcksTracker.cc b/lib/NegativeAcksTracker.cc index 8e501dc4..3ccf0bea 100644 --- a/lib/NegativeAcksTracker.cc +++ b/lib/NegativeAcksTracker.cc @@ -19,11 +19,12 @@ #include "NegativeAcksTracker.h" -#include "ConsumerImpl.h" - -#include #include +#include +#include "ClientImpl.h" +#include "ConsumerImpl.h" +#include "ExecutorService.h" #include "LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/lib/NegativeAcksTracker.h b/lib/NegativeAcksTracker.h index 14762754..c5a945b2 100644 --- a/lib/NegativeAcksTracker.h +++ b/lib/NegativeAcksTracker.h @@ -19,16 +19,24 @@ #pragma once +#include #include -#include "ExecutorService.h" -#include "ClientImpl.h" - -#include +#include +#include #include +#include +#include namespace pulsar { +class ConsumerImpl; +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +using DeadlineTimerPtr = std::shared_ptr; +class ExecutorService; +using ExecutorServicePtr = std::shared_ptr; + class NegativeAcksTracker { public: NegativeAcksTracker(ClientImplPtr client, ConsumerImpl &consumer, const ConsumerConfiguration &conf); diff --git a/lib/ObjectPool.h b/lib/ObjectPool.h index 87507a74..883e0809 100644 --- a/lib/ObjectPool.h +++ b/lib/ObjectPool.h @@ -19,9 +19,8 @@ #ifndef LIB_OBJECTPOOL_H_ #define LIB_OBJECTPOOL_H_ -#include -#include #include +#include namespace pulsar { diff --git a/lib/OpSendMsg.h b/lib/OpSendMsg.h index 365301be..c94bcbe7 100644 --- a/lib/OpSendMsg.h +++ b/lib/OpSendMsg.h @@ -21,10 +21,12 @@ #include #include + #include +#include "PulsarApi.pb.h" +#include "SharedBuffer.h" #include "TimeUtils.h" -#include "MessageImpl.h" namespace pulsar { diff --git a/lib/PartitionedProducerImpl.cc b/lib/PartitionedProducerImpl.cc index 3d383ffb..26d5796a 100644 --- a/lib/PartitionedProducerImpl.cc +++ b/lib/PartitionedProducerImpl.cc @@ -17,12 +17,18 @@ * under the License. */ #include "PartitionedProducerImpl.h" -#include "LogUtils.h" -#include + #include + +#include "ClientImpl.h" +#include "ExecutorService.h" +#include "LogUtils.h" +#include "LookupService.h" +#include "ProducerImpl.h" #include "RoundRobinMessageRouter.h" #include "SinglePartitionMessageRouter.h" #include "TopicMetadataImpl.h" +#include "TopicName.h" DECLARE_LOG_OBJECT() @@ -352,10 +358,10 @@ void PartitionedProducerImpl::triggerFlush() { void PartitionedProducerImpl::flushAsync(FlushCallback callback) { if (!flushPromise_ || flushPromise_->isComplete()) { - flushPromise_ = std::make_shared>(); + flushPromise_ = std::make_shared>(); } else { // already in flushing, register a listener callback - auto listenerCallback = [callback](Result result, bool_type v) { + auto listenerCallback = [callback](Result result, bool v) { if (v) { callback(ResultOk); } else { diff --git a/lib/PartitionedProducerImpl.h b/lib/PartitionedProducerImpl.h index cc7a4e00..b9a4b01a 100644 --- a/lib/PartitionedProducerImpl.h +++ b/lib/PartitionedProducerImpl.h @@ -16,17 +16,33 @@ * specific language governing permissions and limitations * under the License. */ -#include "ProducerImpl.h" -#include "ClientImpl.h" -#include - -#include #include #include -#include + +#include +#include +#include +#include +#include + +#include "LookupDataResult.h" +#include "ProducerImplBase.h" namespace pulsar { +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +using ClientImplWeakPtr = std::weak_ptr; +using DeadlineTimerPtr = std::shared_ptr; +class ExecutorService; +using ExecutorServicePtr = std::shared_ptr; +class LookupService; +using LookupServicePtr = std::shared_ptr; +class ProducerImpl; +using ProducerImplPtr = std::shared_ptr; +class TopicName; +using TopicNamePtr = std::shared_ptr; + class PartitionedProducerImpl : public ProducerImplBase, public std::enable_shared_from_this { public: @@ -107,7 +123,7 @@ class PartitionedProducerImpl : public ProducerImplBase, std::unique_ptr topicMetadata_; std::atomic flushedPartitions_; - std::shared_ptr> flushPromise_; + std::shared_ptr> flushPromise_; ExecutorServicePtr listenerExecutor_; DeadlineTimerPtr partitionsUpdateTimer_; diff --git a/lib/PatternMultiTopicsConsumerImpl.cc b/lib/PatternMultiTopicsConsumerImpl.cc index 8014078d..657f869c 100644 --- a/lib/PatternMultiTopicsConsumerImpl.cc +++ b/lib/PatternMultiTopicsConsumerImpl.cc @@ -18,6 +18,11 @@ */ #include "PatternMultiTopicsConsumerImpl.h" +#include "ClientImpl.h" +#include "ExecutorService.h" +#include "LogUtils.h" +#include "LookupService.h" + DECLARE_LOG_OBJECT() using namespace pulsar; diff --git a/lib/PatternMultiTopicsConsumerImpl.h b/lib/PatternMultiTopicsConsumerImpl.h index 448f2e39..28ad23ef 100644 --- a/lib/PatternMultiTopicsConsumerImpl.h +++ b/lib/PatternMultiTopicsConsumerImpl.h @@ -18,12 +18,14 @@ */ #ifndef PULSAR_PATTERN_MULTI_TOPICS_CONSUMER_HEADER #define PULSAR_PATTERN_MULTI_TOPICS_CONSUMER_HEADER -#include "ConsumerImpl.h" -#include "ClientImpl.h" -#include -#include -#include "MultiTopicsConsumerImpl.h" #include +#include +#include + +#include "LookupDataResult.h" +#include "MultiTopicsConsumerImpl.h" +#include "NamespaceName.h" +#include "TopicName.h" #ifdef PULSAR_USE_BOOST_REGEX #include @@ -35,7 +37,9 @@ namespace pulsar { -class PatternMultiTopicsConsumerImpl; +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +using NamespaceTopicsPtr = std::shared_ptr>; class PatternMultiTopicsConsumerImpl : public MultiTopicsConsumerImpl { public: diff --git a/lib/PeriodicTask.cc b/lib/PeriodicTask.cc index 65bdf234..a196da84 100644 --- a/lib/PeriodicTask.cc +++ b/lib/PeriodicTask.cc @@ -16,7 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -#include "lib/PeriodicTask.h" +#include "PeriodicTask.h" + #include namespace pulsar { diff --git a/lib/PeriodicTask.h b/lib/PeriodicTask.h index 159c86a8..76f90398 100644 --- a/lib/PeriodicTask.h +++ b/lib/PeriodicTask.h @@ -19,12 +19,12 @@ #pragma once #include +#include +#include #include #include #include -#include - namespace pulsar { /** diff --git a/lib/Producer.cc b/lib/Producer.cc index ad60828a..fc588ba4 100644 --- a/lib/Producer.cc +++ b/lib/Producer.cc @@ -16,12 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include "SharedBuffer.h" #include +#include -#include "Utils.h" #include "ProducerImpl.h" +#include "Utils.h" namespace pulsar { diff --git a/lib/ProducerConfiguration.cc b/lib/ProducerConfiguration.cc index 4f64870c..9b3fdfbd 100644 --- a/lib/ProducerConfiguration.cc +++ b/lib/ProducerConfiguration.cc @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include - #include +#include "ProducerConfigurationImpl.h" + namespace pulsar { const static std::string emptyString; diff --git a/lib/ProducerConfigurationImpl.h b/lib/ProducerConfigurationImpl.h index 80c6432c..6c2b19da 100644 --- a/lib/ProducerConfigurationImpl.h +++ b/lib/ProducerConfigurationImpl.h @@ -20,6 +20,7 @@ #define LIB_PRODUCERCONFIGURATIONIMPL_H_ #include + #include #include "Utils.h" diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index e228c836..1213fcef 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -17,17 +17,26 @@ * under the License. */ #include "ProducerImpl.h" + +#include + +#include "BatchMessageContainer.h" +#include "BatchMessageKeyBasedContainer.h" +#include "ClientConnection.h" +#include "ClientImpl.h" +#include "Commands.h" +#include "CompressionCodec.h" +#include "ExecutorService.h" #include "LogUtils.h" +#include "MemoryLimitController.h" +#include "MessageCrypto.h" #include "MessageImpl.h" -#include "TimeUtils.h" +#include "OpSendMsg.h" #include "PulsarApi.pb.h" -#include "Commands.h" -#include "BatchMessageContainerBase.h" -#include "BatchMessageContainer.h" -#include "BatchMessageKeyBasedContainer.h" -#include -#include -#include "MessageAndCallbackBatch.h" +#include "TimeUtils.h" +#include "TopicName.h" +#include "stats/ProducerStatsDisabled.h" +#include "stats/ProducerStatsImpl.h" namespace pulsar { DECLARE_LOG_OBJECT() @@ -314,7 +323,7 @@ void ProducerImpl::setMessageMetadata(const Message& msg, const uint64_t& sequen msgMetadata.set_publish_time(TimeUtils::currentTimeMillis()); msgMetadata.set_sequence_id(sequenceId); if (conf_.getCompressionType() != CompressionNone) { - msgMetadata.set_compression(CompressionCodecProvider::convertType(conf_.getCompressionType())); + msgMetadata.set_compression(static_cast(conf_.getCompressionType())); msgMetadata.set_uncompressed_size(uncompressedSize); } if (!this->getSchemaVersion().empty()) { diff --git a/lib/ProducerImpl.h b/lib/ProducerImpl.h index 05595154..51bf1bb6 100644 --- a/lib/ProducerImpl.h +++ b/lib/ProducerImpl.h @@ -19,36 +19,45 @@ #ifndef LIB_PRODUCERIMPL_H_ #define LIB_PRODUCERIMPL_H_ -#include -#include +#include -#include "ClientImpl.h" -#include "BlockingQueue.h" +#include "Future.h" #include "HandlerBase.h" -#include "SharedBuffer.h" -#include "CompressionCodec.h" -#include "MessageCrypto.h" -#include "stats/ProducerStatsDisabled.h" -#include "stats/ProducerStatsImpl.h" -#include "PulsarApi.pb.h" +// In MSVC, the value type of a STL container cannot be forward declared +#if defined(_MSC_VER) #include "OpSendMsg.h" -#include "BatchMessageContainerBase.h" +#endif #include "PendingFailures.h" -#include "Semaphore.h" #include "PeriodicTask.h" - -using namespace pulsar; +#include "ProducerImplBase.h" +#include "Semaphore.h" +#include "Utils.h" namespace pulsar { -typedef bool bool_type; -typedef std::shared_ptr MessageCryptoPtr; +class BatchMessageContainerBase; +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +class MessageCrypto; +using MessageCryptoPtr = std::shared_ptr; +class ProducerImpl; +using ProducerImplWeakPtr = std::weak_ptr; +class ProducerStatsBase; +using ProducerStatsBasePtr = std::shared_ptr; +struct ResponseData; +class ProducerImpl; +using ProducerImplPtr = std::shared_ptr; class PulsarFriend; class Producer; class MemoryLimitController; class TopicName; +struct OpSendMsg; + +namespace proto { +class MessageMetadata; +} // namespace proto class ProducerImpl : public HandlerBase, public std::enable_shared_from_this, @@ -160,7 +169,6 @@ class ProducerImpl : public HandlerBase, std::string producerStr_; uint64_t producerId_; int64_t msgSequenceGenerator_; - proto::BaseCommand cmd_; std::unique_ptr batchMessageContainer_; boost::asio::deadline_timer batchTimer_; diff --git a/lib/ProducerImplBase.h b/lib/ProducerImplBase.h index 15a6e1d5..0b1622c4 100644 --- a/lib/ProducerImplBase.h +++ b/lib/ProducerImplBase.h @@ -21,6 +21,8 @@ #include #include +#include "Future.h" + namespace pulsar { class ProducerImplBase; diff --git a/lib/ProtoApiEnums.h b/lib/ProtoApiEnums.h new file mode 100644 index 00000000..1f1a79fd --- /dev/null +++ b/lib/ProtoApiEnums.h @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +// This file contains some constants with the same names of the enum constants in PulsarApi.pb.h to avoid +// including the huge PulsarApi.pb.h in *.h and *.cc files. +// Since it's safe to convert an enum constant to int, this file converts a enum from: +// +// ```c++ +// // in PulsarApi.pb.h +// enum MyEnum { +// A = 0, +// B = 1 +// }; +// ``` +// +// to +// +// ```c++ +// using MyEnum = int; +// constexpr MyEnum A = 0; +// constexpr MyEnum B = 1; +// ``` +#pragma once + +namespace pulsar { + +using CommandAck_AckType = int; +constexpr CommandAck_AckType CommandAck_AckType_Individual = 0; +constexpr CommandAck_AckType CommandAck_AckType_Cumulative = 1; + +using CommandSubscribe_SubType = int; +constexpr int CommandSubscribe_SubType_Exclusive = 0; +constexpr int CommandSubscribe_SubType_Shared = 1; +constexpr int CommandSubscribe_SubType_Failover = 2; +constexpr int CommandSubscribe_SubType_Key_Shared = 3; + +using CommandAck_ValidationError = int; +constexpr CommandAck_ValidationError CommandAck_ValidationError_UncompressedSizeCorruption = 0; +constexpr CommandAck_ValidationError CommandAck_ValidationError_DecompressionError = 1; +constexpr CommandAck_ValidationError CommandAck_ValidationError_ChecksumMismatch = 2; +constexpr CommandAck_ValidationError CommandAck_ValidationError_BatchDeSerializeError = 3; +constexpr CommandAck_ValidationError CommandAck_ValidationError_DecryptionError = 4; + +using CommandSubscribe_InitialPosition = int; +constexpr CommandSubscribe_InitialPosition CommandSubscribe_InitialPosition_Latest = 0; +constexpr CommandSubscribe_InitialPosition CommandSubscribe_InitialPosition_Earliest = 1; + +using ProducerAccessMode = int; +constexpr ProducerAccessMode Shared = 0; +constexpr ProducerAccessMode Exclusive = 1; +constexpr ProducerAccessMode WaitForExclusive = 2; +constexpr ProducerAccessMode ExclusiveWithFencing = 3; + +using ServerError = int; +constexpr ServerError UnknownError = 0; +constexpr ServerError MetadataError = 1; +constexpr ServerError PersistenceError = 2; +constexpr ServerError AuthenticationError = 3; +constexpr ServerError AuthorizationError = 4; +constexpr ServerError ConsumerBusy = 5; +constexpr ServerError ServiceNotReady = 6; +constexpr ServerError ProducerBlockedQuotaExceededError = 7; +constexpr ServerError ProducerBlockedQuotaExceededException = 8; +constexpr ServerError ChecksumError = 9; +constexpr ServerError UnsupportedVersionError = 10; +constexpr ServerError TopicNotFound = 11; +constexpr ServerError SubscriptionNotFound = 12; +constexpr ServerError ConsumerNotFound = 13; +constexpr ServerError TooManyRequests = 14; +constexpr ServerError TopicTerminatedError = 15; +constexpr ServerError ProducerBusy = 16; +constexpr ServerError InvalidTopicName = 17; +constexpr ServerError IncompatibleSchema = 18; +constexpr ServerError ConsumerAssignError = 19; +constexpr ServerError TransactionCoordinatorNotFound = 20; +constexpr ServerError InvalidTxnStatus = 21; +constexpr ServerError NotAllowedError = 22; +constexpr ServerError TransactionConflict = 23; +constexpr ServerError TransactionNotFound = 24; +constexpr ServerError ProducerFenced = 25; + +using BaseCommand_Type = int; +constexpr BaseCommand_Type BaseCommand_Type_CONNECT = 2; +constexpr BaseCommand_Type BaseCommand_Type_CONNECTED = 3; +constexpr BaseCommand_Type BaseCommand_Type_SUBSCRIBE = 4; +constexpr BaseCommand_Type BaseCommand_Type_PRODUCER = 5; +constexpr BaseCommand_Type BaseCommand_Type_SEND = 6; +constexpr BaseCommand_Type BaseCommand_Type_SEND_RECEIPT = 7; +constexpr BaseCommand_Type BaseCommand_Type_SEND_ERROR = 8; +constexpr BaseCommand_Type BaseCommand_Type_MESSAGE = 9; +constexpr BaseCommand_Type BaseCommand_Type_ACK = 10; +constexpr BaseCommand_Type BaseCommand_Type_FLOW = 11; +constexpr BaseCommand_Type BaseCommand_Type_UNSUBSCRIBE = 12; +constexpr BaseCommand_Type BaseCommand_Type_SUCCESS = 13; +constexpr BaseCommand_Type BaseCommand_Type_ERROR = 14; +constexpr BaseCommand_Type BaseCommand_Type_CLOSE_PRODUCER = 15; +constexpr BaseCommand_Type BaseCommand_Type_CLOSE_CONSUMER = 16; +constexpr BaseCommand_Type BaseCommand_Type_PRODUCER_SUCCESS = 17; +constexpr BaseCommand_Type BaseCommand_Type_PING = 18; +constexpr BaseCommand_Type BaseCommand_Type_PONG = 19; +constexpr BaseCommand_Type BaseCommand_Type_REDELIVER_UNACKNOWLEDGED_MESSAGES = 20; +constexpr BaseCommand_Type BaseCommand_Type_PARTITIONED_METADATA = 21; +constexpr BaseCommand_Type BaseCommand_Type_PARTITIONED_METADATA_RESPONSE = 22; +constexpr BaseCommand_Type BaseCommand_Type_LOOKUP = 23; +constexpr BaseCommand_Type BaseCommand_Type_LOOKUP_RESPONSE = 24; +constexpr BaseCommand_Type BaseCommand_Type_CONSUMER_STATS = 25; +constexpr BaseCommand_Type BaseCommand_Type_CONSUMER_STATS_RESPONSE = 26; +constexpr BaseCommand_Type BaseCommand_Type_REACHED_END_OF_TOPIC = 27; +constexpr BaseCommand_Type BaseCommand_Type_SEEK = 28; +constexpr BaseCommand_Type BaseCommand_Type_GET_LAST_MESSAGE_ID = 29; +constexpr BaseCommand_Type BaseCommand_Type_GET_LAST_MESSAGE_ID_RESPONSE = 30; +constexpr BaseCommand_Type BaseCommand_Type_ACTIVE_CONSUMER_CHANGE = 31; +constexpr BaseCommand_Type BaseCommand_Type_GET_TOPICS_OF_NAMESPACE = 32; +constexpr BaseCommand_Type BaseCommand_Type_GET_TOPICS_OF_NAMESPACE_RESPONSE = 33; +constexpr BaseCommand_Type BaseCommand_Type_GET_SCHEMA = 34; +constexpr BaseCommand_Type BaseCommand_Type_GET_SCHEMA_RESPONSE = 35; +constexpr BaseCommand_Type BaseCommand_Type_AUTH_CHALLENGE = 36; +constexpr BaseCommand_Type BaseCommand_Type_AUTH_RESPONSE = 37; +constexpr BaseCommand_Type BaseCommand_Type_ACK_RESPONSE = 38; +constexpr BaseCommand_Type BaseCommand_Type_GET_OR_CREATE_SCHEMA = 39; +constexpr BaseCommand_Type BaseCommand_Type_GET_OR_CREATE_SCHEMA_RESPONSE = 40; +constexpr BaseCommand_Type BaseCommand_Type_NEW_TXN = 50; +constexpr BaseCommand_Type BaseCommand_Type_NEW_TXN_RESPONSE = 51; +constexpr BaseCommand_Type BaseCommand_Type_ADD_PARTITION_TO_TXN = 52; +constexpr BaseCommand_Type BaseCommand_Type_ADD_PARTITION_TO_TXN_RESPONSE = 53; +constexpr BaseCommand_Type BaseCommand_Type_ADD_SUBSCRIPTION_TO_TXN = 54; +constexpr BaseCommand_Type BaseCommand_Type_ADD_SUBSCRIPTION_TO_TXN_RESPONSE = 55; +constexpr BaseCommand_Type BaseCommand_Type_END_TXN = 56; +constexpr BaseCommand_Type BaseCommand_Type_END_TXN_RESPONSE = 57; +constexpr BaseCommand_Type BaseCommand_Type_END_TXN_ON_PARTITION = 58; +constexpr BaseCommand_Type BaseCommand_Type_END_TXN_ON_PARTITION_RESPONSE = 59; +constexpr BaseCommand_Type BaseCommand_Type_END_TXN_ON_SUBSCRIPTION = 60; +constexpr BaseCommand_Type BaseCommand_Type_END_TXN_ON_SUBSCRIPTION_RESPONSE = 61; +constexpr BaseCommand_Type BaseCommand_Type_TC_CLIENT_CONNECT_REQUEST = 62; +constexpr BaseCommand_Type BaseCommand_Type_TC_CLIENT_CONNECT_RESPONSE = 63; +constexpr BaseCommand_Type BaseCommand_Type_WATCH_TOPIC_LIST = 64; +constexpr BaseCommand_Type BaseCommand_Type_WATCH_TOPIC_LIST_SUCCESS = 65; +constexpr BaseCommand_Type BaseCommand_Type_WATCH_TOPIC_UPDATE = 66; +constexpr BaseCommand_Type BaseCommand_Type_WATCH_TOPIC_LIST_CLOSE = 67; + +} // namespace pulsar diff --git a/lib/ProtobufNativeSchema.cc b/lib/ProtobufNativeSchema.cc index 3b8a404b..edae2ec2 100644 --- a/lib/ProtobufNativeSchema.cc +++ b/lib/ProtobufNativeSchema.cc @@ -18,12 +18,12 @@ */ #include "pulsar/ProtobufNativeSchema.h" -#include -#include +#include #include #include -#include +#include +#include using google::protobuf::FileDescriptor; using google::protobuf::FileDescriptorSet; diff --git a/lib/Reader.cc b/lib/Reader.cc index fa485362..261c0fac 100644 --- a/lib/Reader.cc +++ b/lib/Reader.cc @@ -20,8 +20,8 @@ #include #include "Future.h" -#include "Utils.h" #include "ReaderImpl.h" +#include "Utils.h" namespace pulsar { diff --git a/lib/ReaderConfiguration.cc b/lib/ReaderConfiguration.cc index 0dfdbedc..3ba7fedd 100644 --- a/lib/ReaderConfiguration.cc +++ b/lib/ReaderConfiguration.cc @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -#include +#include "ReaderConfigurationImpl.h" namespace pulsar { diff --git a/lib/ReaderImpl.cc b/lib/ReaderImpl.cc index 83fa6a57..d1b25b58 100644 --- a/lib/ReaderImpl.cc +++ b/lib/ReaderImpl.cc @@ -17,8 +17,12 @@ * under the License. */ -#include "ClientImpl.h" #include "ReaderImpl.h" + +#include "ClientImpl.h" +#include "ConsumerImpl.h" +#include "ExecutorService.h" +#include "GetLastMessageIdResponse.h" #include "TopicName.h" namespace pulsar { diff --git a/lib/ReaderImpl.h b/lib/ReaderImpl.h index b0d8a6bc..ed16c7d3 100644 --- a/lib/ReaderImpl.h +++ b/lib/ReaderImpl.h @@ -20,7 +20,15 @@ #ifndef LIB_READERIMPL_H_ #define LIB_READERIMPL_H_ -#include "ConsumerImpl.h" +#include +#include +#include + +#include +#include +#include + +#include "Future.h" namespace pulsar { @@ -29,6 +37,17 @@ class ReaderImpl; typedef std::shared_ptr ReaderImplPtr; typedef std::weak_ptr ReaderImplWeakPtr; +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +using ClientImplWeakPtr = std::weak_ptr; +class ConsumerImplBase; +using ConsumerImplBaseWeakPtr = std::weak_ptr; +class ConsumerImpl; +using ConsumerImplPtr = std::shared_ptr; +using ConsumerImplWeakPtr = std::weak_ptr; +class ExecutorService; +using ExecutorServicePtr = std::shared_ptr; + namespace test { extern PULSAR_PUBLIC std::mutex readerConfigTestMutex; @@ -53,7 +72,7 @@ class PULSAR_PUBLIC ReaderImpl : public std::enable_shared_from_this Future getReaderCreatedFuture(); - ConsumerImplBaseWeakPtr getConsumer() const noexcept { return consumer_; } + ConsumerImplWeakPtr getConsumer() const noexcept { return consumer_; } void hasMessageAvailableAsync(HasMessageAvailableCallback callback); diff --git a/lib/Result.cc b/lib/Result.cc index 6682341b..3533b1ec 100644 --- a/lib/Result.cc +++ b/lib/Result.cc @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include #include diff --git a/lib/RetryableLookupService.h b/lib/RetryableLookupService.h index a8f7bfce..7d704ec5 100644 --- a/lib/RetryableLookupService.h +++ b/lib/RetryableLookupService.h @@ -20,11 +20,14 @@ #include #include -#include "lib/Backoff.h" -#include "lib/ExecutorService.h" -#include "lib/LookupService.h" -#include "lib/SynchronizedHashMap.h" -#include "lib/LogUtils.h" + +#include "Backoff.h" +#include "ExecutorService.h" +#include "LogUtils.h" +#include "LookupDataResult.h" +#include "LookupService.h" +#include "SynchronizedHashMap.h" +#include "TopicName.h" namespace pulsar { diff --git a/lib/RoundRobinMessageRouter.cc b/lib/RoundRobinMessageRouter.cc index 51d10e2c..e33f8d95 100644 --- a/lib/RoundRobinMessageRouter.cc +++ b/lib/RoundRobinMessageRouter.cc @@ -18,11 +18,12 @@ */ #include "RoundRobinMessageRouter.h" -#include "TimeUtils.h" - #include #include +#include "Hash.h" +#include "TimeUtils.h" + namespace pulsar { RoundRobinMessageRouter::RoundRobinMessageRouter(ProducerConfiguration::HashingScheme hashingScheme, bool batchingEnabled, uint32_t maxBatchingMessages, diff --git a/lib/RoundRobinMessageRouter.h b/lib/RoundRobinMessageRouter.h index be172a0e..753573a1 100644 --- a/lib/RoundRobinMessageRouter.h +++ b/lib/RoundRobinMessageRouter.h @@ -19,15 +19,13 @@ #pragma once -#include -#include #include #include -#include "Hash.h" -#include "MessageRouterBase.h" #include -#include +#include + +#include "MessageRouterBase.h" namespace pulsar { class PULSAR_PUBLIC RoundRobinMessageRouter : public MessageRouterBase { diff --git a/lib/Schema.cc b/lib/Schema.cc index af452f4c..17a301e6 100644 --- a/lib/Schema.cc +++ b/lib/Schema.cc @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include #include #include diff --git a/lib/Semaphore.h b/lib/Semaphore.h index dcef2ad5..14ebb876 100644 --- a/lib/Semaphore.h +++ b/lib/Semaphore.h @@ -45,4 +45,4 @@ class Semaphore { bool isClosed_ = false; }; -} // namespace pulsar \ No newline at end of file +} // namespace pulsar diff --git a/lib/ServiceNameResolver.h b/lib/ServiceNameResolver.h index cf7a5832..8457d0e1 100644 --- a/lib/ServiceNameResolver.h +++ b/lib/ServiceNameResolver.h @@ -19,7 +19,9 @@ #pragma once #include + #include + #include "ServiceURI.h" namespace pulsar { diff --git a/lib/ServiceURI.cc b/lib/ServiceURI.cc index ec515b24..d95ec4ab 100644 --- a/lib/ServiceURI.cc +++ b/lib/ServiceURI.cc @@ -17,6 +17,7 @@ * under the License. */ #include "ServiceURI.h" + #include namespace pulsar { diff --git a/lib/ServiceURI.h b/lib/ServiceURI.h index 4f459d98..6ee26a78 100644 --- a/lib/ServiceURI.h +++ b/lib/ServiceURI.h @@ -21,6 +21,7 @@ #include #include #include + #include "PulsarScheme.h" namespace pulsar { diff --git a/lib/SharedBuffer.h b/lib/SharedBuffer.h index be889a7e..7ee26184 100644 --- a/lib/SharedBuffer.h +++ b/lib/SharedBuffer.h @@ -19,9 +19,11 @@ #ifndef LIB_SHARED_BUFFER_H_ #define LIB_SHARED_BUFFER_H_ -#include +#include #include +#include +#include #include #include #include diff --git a/lib/SimpleLogger.h b/lib/SimpleLogger.h index b750336a..d1e44c76 100644 --- a/lib/SimpleLogger.h +++ b/lib/SimpleLogger.h @@ -19,11 +19,13 @@ #pragma once +#include + +#include +#include #include #include #include -#include -#include namespace pulsar { diff --git a/lib/SinglePartitionMessageRouter.cc b/lib/SinglePartitionMessageRouter.cc index 5ebe4c87..1658ba6b 100644 --- a/lib/SinglePartitionMessageRouter.cc +++ b/lib/SinglePartitionMessageRouter.cc @@ -21,6 +21,8 @@ #include #include +#include "Hash.h" + namespace pulsar { SinglePartitionMessageRouter::~SinglePartitionMessageRouter() {} diff --git a/lib/SinglePartitionMessageRouter.h b/lib/SinglePartitionMessageRouter.h index 4407bd42..4e5d24dd 100644 --- a/lib/SinglePartitionMessageRouter.h +++ b/lib/SinglePartitionMessageRouter.h @@ -19,11 +19,11 @@ #ifndef PULSAR_SINGLE_PARTITION_MESSAGE_ROUTER_HEADER_ #define PULSAR_SINGLE_PARTITION_MESSAGE_ROUTER_HEADER_ -#include #include -#include -#include "Hash.h" +#include #include +#include + #include "MessageRouterBase.h" namespace pulsar { diff --git a/lib/SynchronizedHashMap.h b/lib/SynchronizedHashMap.h index 9bed7d79..b8a8c91e 100644 --- a/lib/SynchronizedHashMap.h +++ b/lib/SynchronizedHashMap.h @@ -23,6 +23,7 @@ #include #include #include + #include "Utils.h" namespace pulsar { diff --git a/lib/TimeUtils.h b/lib/TimeUtils.h index 45157ae8..a55773d9 100644 --- a/lib/TimeUtils.h +++ b/lib/TimeUtils.h @@ -18,12 +18,12 @@ */ #pragma once -#include +#include + #include +#include #include -#include - namespace pulsar { using namespace boost::posix_time; diff --git a/lib/TopicMetadataImpl.cc b/lib/TopicMetadataImpl.cc index e29cd970..2f622b2c 100644 --- a/lib/TopicMetadataImpl.cc +++ b/lib/TopicMetadataImpl.cc @@ -23,4 +23,4 @@ namespace pulsar { TopicMetadataImpl::TopicMetadataImpl(const int numPartitions) : numPartitions_(numPartitions) {} int TopicMetadataImpl::getNumPartitions() const { return numPartitions_; } -} // namespace pulsar \ No newline at end of file +} // namespace pulsar diff --git a/lib/TopicMetadataImpl.h b/lib/TopicMetadataImpl.h index 76393c43..aff763de 100644 --- a/lib/TopicMetadataImpl.h +++ b/lib/TopicMetadataImpl.h @@ -19,7 +19,6 @@ #ifndef TOPIC_METADATA_IMPL_HPP_ #define TOPIC_METADATA_IMPL_HPP_ -#include #include namespace pulsar { diff --git a/lib/TopicName.cc b/lib/TopicName.cc index 70b7b7e5..48c52c4a 100644 --- a/lib/TopicName.cc +++ b/lib/TopicName.cc @@ -16,26 +16,26 @@ * specific language governing permissions and limitations * under the License. */ -#include "NamedEntity.h" -#include "LogUtils.h" -#include "PartitionedProducerImpl.h" #include "TopicName.h" #include -#include +#include +#include #include +#include #include #include -#include -#include -#include -#include + +#include "LogUtils.h" +#include "NamedEntity.h" +#include "NamespaceName.h" DECLARE_LOG_OBJECT() namespace pulsar { const std::string TopicDomain::Persistent = "persistent"; const std::string TopicDomain::NonPersistent = "non-persistent"; +static const std::string PARTITION_NAME_SUFFIX = "-partition-"; typedef std::unique_lock Lock; // static members @@ -233,12 +233,12 @@ bool TopicName::isPersistent() const { return this->domain_ == TopicDomain::Pers std::string TopicName::getTopicPartitionName(unsigned int partition) const { std::stringstream topicPartitionName; // make this topic name as well - topicPartitionName << toString() << PartitionedProducerImpl::PARTITION_NAME_SUFFIX << partition; + topicPartitionName << toString() << PARTITION_NAME_SUFFIX << partition; return topicPartitionName.str(); } int TopicName::getPartitionIndex(const std::string& topic) { - const auto& suffix = PartitionedProducerImpl::PARTITION_NAME_SUFFIX; + const auto& suffix = PARTITION_NAME_SUFFIX; const size_t pos = topic.rfind(suffix); if (pos == std::string::npos) { return -1; diff --git a/lib/TopicName.h b/lib/TopicName.h index d8620ea1..51f701fb 100644 --- a/lib/TopicName.h +++ b/lib/TopicName.h @@ -19,15 +19,20 @@ #ifndef _PULSAR_TOPIC_NAME_HEADER_ #define _PULSAR_TOPIC_NAME_HEADER_ +#include #include -#include "NamespaceName.h" -#include "ServiceUnitId.h" -#include -#include +#include #include +#include + +#include "ServiceUnitId.h" namespace pulsar { + +class NamespaceName; +using NamespaceNamePtr = std::shared_ptr; + class PULSAR_PUBLIC TopicDomain { public: static const std::string Persistent; diff --git a/lib/UnAckedMessageTrackerDisabled.h b/lib/UnAckedMessageTrackerDisabled.h index c25c1a5b..bd12b3ef 100644 --- a/lib/UnAckedMessageTrackerDisabled.h +++ b/lib/UnAckedMessageTrackerDisabled.h @@ -18,7 +18,7 @@ */ #ifndef LIB_UNACKEDMESSAGETRACKERDISABLED_H_ #define LIB_UNACKEDMESSAGETRACKERDISABLED_H_ -#include "lib/UnAckedMessageTrackerInterface.h" +#include "UnAckedMessageTrackerInterface.h" namespace pulsar { class UnAckedMessageTrackerDisabled : public UnAckedMessageTrackerInterface { diff --git a/lib/UnAckedMessageTrackerEnabled.cc b/lib/UnAckedMessageTrackerEnabled.cc index 9d0160f3..1bc878b8 100644 --- a/lib/UnAckedMessageTrackerEnabled.cc +++ b/lib/UnAckedMessageTrackerEnabled.cc @@ -20,6 +20,11 @@ #include +#include "ClientImpl.h" +#include "ConsumerImplBase.h" +#include "ExecutorService.h" +#include "LogUtils.h" + DECLARE_LOG_OBJECT(); namespace pulsar { diff --git a/lib/UnAckedMessageTrackerEnabled.h b/lib/UnAckedMessageTrackerEnabled.h index 7ed7b038..13dee211 100644 --- a/lib/UnAckedMessageTrackerEnabled.h +++ b/lib/UnAckedMessageTrackerEnabled.h @@ -18,17 +18,27 @@ */ #ifndef LIB_UNACKEDMESSAGETRACKERENABLED_H_ #define LIB_UNACKEDMESSAGETRACKERENABLED_H_ -#include "lib/TestUtil.h" -#include "lib/UnAckedMessageTrackerInterface.h" - +#include +#include +#include #include +#include + +#include "TestUtil.h" +#include "UnAckedMessageTrackerInterface.h" namespace pulsar { + +class ClientImpl; +class ConsumerImplBase; +using ClientImplPtr = std::shared_ptr; +using DeadlineTimerPtr = std::shared_ptr; + class UnAckedMessageTrackerEnabled : public UnAckedMessageTrackerInterface { public: ~UnAckedMessageTrackerEnabled(); - UnAckedMessageTrackerEnabled(long timeoutMs, const ClientImplPtr, ConsumerImplBase&); - UnAckedMessageTrackerEnabled(long timeoutMs, long tickDuration, const ClientImplPtr, ConsumerImplBase&); + UnAckedMessageTrackerEnabled(long timeoutMs, ClientImplPtr, ConsumerImplBase&); + UnAckedMessageTrackerEnabled(long timeoutMs, long tickDuration, ClientImplPtr, ConsumerImplBase&); bool add(const MessageId& msgId); bool remove(const MessageId& msgId); void removeMessagesTill(const MessageId& msgId); diff --git a/lib/UnAckedMessageTrackerInterface.h b/lib/UnAckedMessageTrackerInterface.h index 50fa72c5..3bcaaa5f 100644 --- a/lib/UnAckedMessageTrackerInterface.h +++ b/lib/UnAckedMessageTrackerInterface.h @@ -18,18 +18,10 @@ */ #ifndef LIB_UNACKEDMESSAGETRACKERINTERFACE_H_ #define LIB_UNACKEDMESSAGETRACKERINTERFACE_H_ -#include +#include + #include -#include -#include -#include -#include "pulsar/MessageId.h" -#include "lib/ClientImpl.h" -#include "lib/ConsumerImplBase.h" -#include -#include -#include "lib/PulsarApi.pb.h" -#include +#include namespace pulsar { class UnAckedMessageTrackerInterface { diff --git a/lib/UnboundedBlockingQueue.h b/lib/UnboundedBlockingQueue.h index 0f7fc2a3..01ffc0c3 100644 --- a/lib/UnboundedBlockingQueue.h +++ b/lib/UnboundedBlockingQueue.h @@ -19,9 +19,9 @@ #ifndef LIB_UNBOUNDEDBLOCKINGQUEUE_H_ #define LIB_UNBOUNDEDBLOCKINGQUEUE_H_ -#include -#include #include +#include +#include // For struct QueueNotEmpty #include "BlockingQueue.h" diff --git a/lib/Url.cc b/lib/Url.cc index f31e1fcc..b52b7231 100644 --- a/lib/Url.cc +++ b/lib/Url.cc @@ -19,7 +19,6 @@ #include "Url.h" #include - #include #ifdef PULSAR_USE_BOOST_REGEX diff --git a/lib/Url.h b/lib/Url.h index f5596c53..5cf76efc 100644 --- a/lib/Url.h +++ b/lib/Url.h @@ -19,9 +19,10 @@ #ifndef LIB_URL_H_ #define LIB_URL_H_ -#include #include +#include + namespace pulsar { /** diff --git a/lib/UtilAllocator.h b/lib/UtilAllocator.h index acd1414b..7096e35d 100644 --- a/lib/UtilAllocator.h +++ b/lib/UtilAllocator.h @@ -20,6 +20,7 @@ #define LIB_UTILALLOCATOR_H_ #include +#include class HandlerAllocator : private boost::noncopyable { public: diff --git a/lib/Utils.h b/lib/Utils.h index b0f500ef..016f09f2 100644 --- a/lib/Utils.h +++ b/lib/Utils.h @@ -21,10 +21,10 @@ #include -#include "Future.h" - -#include #include +#include + +#include "Future.h" namespace pulsar { diff --git a/lib/auth/AuthAthenz.cc b/lib/auth/AuthAthenz.cc index 82d12761..2360ab8c 100644 --- a/lib/auth/AuthAthenz.cc +++ b/lib/auth/AuthAthenz.cc @@ -16,15 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -#include +#include "AuthAthenz.h" #include #include -namespace ptree = boost::property_tree; - +#include #include -#include +#include "athenz/ZTSClient.h" +#include "lib/LogUtils.h" + +namespace ptree = boost::property_tree; DECLARE_LOG_OBJECT() diff --git a/lib/auth/AuthAthenz.h b/lib/auth/AuthAthenz.h index e58a4bc8..7d2048f0 100644 --- a/lib/auth/AuthAthenz.h +++ b/lib/auth/AuthAthenz.h @@ -20,11 +20,11 @@ #define PULSAR_AUTH_ATHENZ_H_ #include -#include -#include namespace pulsar { +class ZTSClient; + const std::string ATHENZ_PLUGIN_NAME = "athenz"; const std::string ATHENZ_JAVA_PLUGIN_NAME = "org.apache.pulsar.client.impl.auth.AuthenticationAthenz"; diff --git a/lib/auth/AuthBasic.cc b/lib/auth/AuthBasic.cc index ca74803a..8d42061c 100644 --- a/lib/auth/AuthBasic.cc +++ b/lib/auth/AuthBasic.cc @@ -19,15 +19,14 @@ #include "AuthBasic.h" -#include #include #include #include #include -namespace ptree = boost::property_tree; - -#include #include +#include +#include +namespace ptree = boost::property_tree; namespace pulsar { diff --git a/lib/auth/AuthBasic.h b/lib/auth/AuthBasic.h index 2bd9e11e..65c104fc 100644 --- a/lib/auth/AuthBasic.h +++ b/lib/auth/AuthBasic.h @@ -20,8 +20,6 @@ #pragma once #include -#include -#include namespace pulsar { diff --git a/lib/auth/AuthOauth2.cc b/lib/auth/AuthOauth2.cc index 2fce8047..66c1b053 100644 --- a/lib/auth/AuthOauth2.cc +++ b/lib/auth/AuthOauth2.cc @@ -16,15 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -#include +#include "AuthOauth2.h" #include -#include -#include + #include #include +#include +#include -#include +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() namespace pulsar { diff --git a/lib/auth/AuthOauth2.h b/lib/auth/AuthOauth2.h index c940cf96..565af069 100644 --- a/lib/auth/AuthOauth2.h +++ b/lib/auth/AuthOauth2.h @@ -20,9 +20,9 @@ #pragma once #include + #include #include -#include namespace pulsar { diff --git a/lib/auth/AuthTls.cc b/lib/auth/AuthTls.cc index fdf7f210..e0287cd0 100644 --- a/lib/auth/AuthTls.cc +++ b/lib/auth/AuthTls.cc @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -#include +#include "AuthTls.h" namespace pulsar { AuthDataTls::AuthDataTls(const std::string& certificatePath, const std::string& privateKeyPath) { diff --git a/lib/auth/AuthTls.h b/lib/auth/AuthTls.h index 510aea06..8b243e03 100644 --- a/lib/auth/AuthTls.h +++ b/lib/auth/AuthTls.h @@ -20,7 +20,6 @@ #pragma once #include -#include namespace pulsar { diff --git a/lib/auth/AuthToken.cc b/lib/auth/AuthToken.cc index e8ebc722..367a26d0 100644 --- a/lib/auth/AuthToken.cc +++ b/lib/auth/AuthToken.cc @@ -19,11 +19,9 @@ #include "AuthToken.h" #include -#include -#include - -#include #include +#include +#include namespace pulsar { diff --git a/lib/auth/AuthToken.h b/lib/auth/AuthToken.h index 8473fe31..eed7b305 100644 --- a/lib/auth/AuthToken.h +++ b/lib/auth/AuthToken.h @@ -20,8 +20,6 @@ #pragma once #include -#include -#include namespace pulsar { diff --git a/lib/auth/athenz/ZTSClient.cc b/lib/auth/athenz/ZTSClient.cc index 919536fe..65bfd21b 100644 --- a/lib/auth/athenz/ZTSClient.cc +++ b/lib/auth/athenz/ZTSClient.cc @@ -17,22 +17,23 @@ * under the License. */ #include "ZTSClient.h" + #include +#include "lib/LogUtils.h" + #ifndef _MSC_VER #include #else #include #endif -#include -#include - -#include -#include +#include #include #include - -#include +#include +#include +#include +#include #include #include @@ -51,7 +52,6 @@ namespace ptree = boost::property_tree; #include #include - #include #ifdef PULSAR_USE_BOOST_REGEX diff --git a/lib/auth/athenz/ZTSClient.h b/lib/auth/athenz/ZTSClient.h index fdc690c4..429087e7 100644 --- a/lib/auth/athenz/ZTSClient.h +++ b/lib/auth/athenz/ZTSClient.h @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include #include -#include + +#include +#include namespace pulsar { diff --git a/lib/c/cStringMap.cc b/lib/c/cStringMap.cc index 221dce4b..a24c2f6f 100644 --- a/lib/c/cStringMap.cc +++ b/lib/c/cStringMap.cc @@ -57,4 +57,4 @@ const char *pulsar_string_map_get_value(pulsar_string_map_t *map, int idx) { } return it->second.c_str(); -} \ No newline at end of file +} diff --git a/lib/c/c_Authentication.cc b/lib/c/c_Authentication.cc index 8384fac5..e4b2fbda 100644 --- a/lib/c/c_Authentication.cc +++ b/lib/c/c_Authentication.cc @@ -17,14 +17,13 @@ * under the License. */ +#include #include -#include +#include #include "c_structs.h" -#include - pulsar_authentication_t *pulsar_authentication_create(const char *dynamicLibPath, const char *authParamsString) { pulsar_authentication_t *authentication = new pulsar_authentication_t; diff --git a/lib/c/c_Message.cc b/lib/c/c_Message.cc index 4fe4c391..552fa081 100644 --- a/lib/c/c_Message.cc +++ b/lib/c/c_Message.cc @@ -18,6 +18,7 @@ */ #include + #include "c_structs.h" pulsar_message_t *pulsar_message_create() { return new pulsar_message_t; } diff --git a/lib/c/c_MessageId.cc b/lib/c/c_MessageId.cc index 537bb709..ca7739e9 100644 --- a/lib/c/c_MessageId.cc +++ b/lib/c/c_MessageId.cc @@ -18,11 +18,13 @@ */ #include -#include "c_structs.h" +#include #include #include +#include "c_structs.h" + std::once_flag initialized; static pulsar_message_id_t earliest; diff --git a/lib/c/c_ProducerConfiguration.cc b/lib/c/c_ProducerConfiguration.cc index fbc5714d..868eddf3 100644 --- a/lib/c/c_ProducerConfiguration.cc +++ b/lib/c/c_ProducerConfiguration.cc @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ +#include #include -#include #include "c_structs.h" diff --git a/lib/c/c_Reader.cc b/lib/c/c_Reader.cc index a28d9c23..3490b540 100644 --- a/lib/c/c_Reader.cc +++ b/lib/c/c_Reader.cc @@ -17,8 +17,8 @@ * under the License. */ -#include #include +#include #include "c_structs.h" diff --git a/lib/c/c_ReaderConfiguration.cc b/lib/c/c_ReaderConfiguration.cc index 9d3a1a08..fe05cf22 100644 --- a/lib/c/c_ReaderConfiguration.cc +++ b/lib/c/c_ReaderConfiguration.cc @@ -17,11 +17,11 @@ * under the License. */ -#include +#include +#include #include +#include #include -#include -#include #include "c_structs.h" @@ -85,4 +85,4 @@ void pulsar_reader_configuration_set_read_compacted(pulsar_reader_configuration_ int pulsar_reader_configuration_is_read_compacted(pulsar_reader_configuration_t *configuration) { return configuration->conf.isReadCompacted(); -} \ No newline at end of file +} diff --git a/lib/c/c_Result.cc b/lib/c/c_Result.cc index 157a91f3..e0624b4e 100644 --- a/lib/c/c_Result.cc +++ b/lib/c/c_Result.cc @@ -17,7 +17,7 @@ * under the License. */ -#include #include +#include const char *pulsar_result_str(pulsar_result result) { return pulsar::strResult((pulsar::Result)result); } diff --git a/lib/c/c_structs.h b/lib/c/c_structs.h index eb8889a9..0ad134fa 100644 --- a/lib/c/c_structs.h +++ b/lib/c/c_structs.h @@ -18,11 +18,11 @@ */ #pragma once -#include #include +#include -#include #include +#include struct _pulsar_client { std::unique_ptr client; diff --git a/lib/checksum/ChecksumProvider.h b/lib/checksum/ChecksumProvider.h index 378b3217..a1e5ef77 100644 --- a/lib/checksum/ChecksumProvider.h +++ b/lib/checksum/ChecksumProvider.h @@ -19,8 +19,8 @@ #ifndef _CHECKSUM_PROVIDER_H_ #define _CHECKSUM_PROVIDER_H_ -#include #include +#include namespace pulsar { diff --git a/lib/checksum/crc32c_arm.cc b/lib/checksum/crc32c_arm.cc index d937a167..266ac044 100644 --- a/lib/checksum/crc32c_arm.cc +++ b/lib/checksum/crc32c_arm.cc @@ -23,6 +23,7 @@ // (found in the LICENSE.Apache file in the root directory). #include "crc32c_arm.h" + #include "lib/checksum/crc32c_sw.h" #if defined(HAVE_ARM64_CRC) diff --git a/lib/checksum/crc32c_sse42.cc b/lib/checksum/crc32c_sse42.cc index c5dba04b..8c52a27e 100644 --- a/lib/checksum/crc32c_sse42.cc +++ b/lib/checksum/crc32c_sse42.cc @@ -28,8 +28,9 @@ #include #include -#include "lib/checksum/crc32c_sw.h" + #include "gf2.hpp" +#include "lib/checksum/crc32c_sw.h" #if BOOST_ARCH_X86_64 && !defined(__arm64__) #define PULSAR_X86_64 diff --git a/lib/checksum/crc32c_sw.cc b/lib/checksum/crc32c_sw.cc index d03802ce..4186964d 100644 --- a/lib/checksum/crc32c_sw.cc +++ b/lib/checksum/crc32c_sw.cc @@ -34,6 +34,7 @@ */ #include "crc32c_sw.h" + #include namespace pulsar { diff --git a/lib/lz4/lz4.h b/lib/lz4/lz4.h index e5fb5a47..a428b8b3 100644 --- a/lib/lz4/lz4.h +++ b/lib/lz4/lz4.h @@ -402,4 +402,4 @@ int LZ4_decompress_safe_withPrefix64k(const char *src, char *dst, int compressed LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k(const char *src, char *dst, int originalSize); -} // namespace pulsar \ No newline at end of file +} // namespace pulsar diff --git a/lib/stats/ConsumerStatsBase.h b/lib/stats/ConsumerStatsBase.h index de7e07bf..13ca1549 100644 --- a/lib/stats/ConsumerStatsBase.h +++ b/lib/stats/ConsumerStatsBase.h @@ -20,15 +20,15 @@ #ifndef PULSAR_CONSUMER_STATS_BASE_HEADER #define PULSAR_CONSUMER_STATS_BASE_HEADER #include -#include #include -#include + +#include "lib/ProtoApiEnums.h" namespace pulsar { class ConsumerStatsBase { public: virtual void receivedMessage(Message&, Result) = 0; - virtual void messageAcknowledged(Result, proto::CommandAck_AckType) = 0; + virtual void messageAcknowledged(Result, CommandAck_AckType) = 0; virtual ~ConsumerStatsBase() {} }; diff --git a/lib/stats/ConsumerStatsDisabled.h b/lib/stats/ConsumerStatsDisabled.h index e2233d55..f32d0262 100644 --- a/lib/stats/ConsumerStatsDisabled.h +++ b/lib/stats/ConsumerStatsDisabled.h @@ -20,14 +20,14 @@ #ifndef PULSAR_CONSUMER_STATS_DISABLED_H_ #define PULSAR_CONSUMER_STATS_DISABLED_H_ -#include +#include "ConsumerStatsBase.h" namespace pulsar { class ConsumerStatsDisabled : public ConsumerStatsBase { public: virtual void receivedMessage(Message&, Result) {} - virtual void messageAcknowledged(Result, proto::CommandAck_AckType) {} + virtual void messageAcknowledged(Result, CommandAck_AckType) {} }; } /* namespace pulsar */ diff --git a/lib/stats/ConsumerStatsImpl.cc b/lib/stats/ConsumerStatsImpl.cc index 38534a60..833dcd19 100644 --- a/lib/stats/ConsumerStatsImpl.cc +++ b/lib/stats/ConsumerStatsImpl.cc @@ -17,14 +17,19 @@ * under the License. */ -#include -#include +#include "ConsumerStatsImpl.h" #include +#include "lib/ExecutorService.h" +#include "lib/LogUtils.h" +#include "lib/Utils.h" + namespace pulsar { DECLARE_LOG_OBJECT(); +using Lock = std::unique_lock; + ConsumerStatsImpl::ConsumerStatsImpl(std::string consumerStr, ExecutorServicePtr executor, unsigned int statsIntervalInSeconds) : consumerStr_(consumerStr), @@ -80,16 +85,16 @@ void ConsumerStatsImpl::receivedMessage(Message& msg, Result res) { totalReceivedMsgMap_[res] += 1; } -void ConsumerStatsImpl::messageAcknowledged(Result res, proto::CommandAck_AckType ackType) { +void ConsumerStatsImpl::messageAcknowledged(Result res, CommandAck_AckType ackType) { Lock lock(mutex_); ackedMsgMap_[std::make_pair(res, ackType)] += 1; totalAckedMsgMap_[std::make_pair(res, ackType)] += 1; } std::ostream& operator<<(std::ostream& os, - const std::map, unsigned long>& m) { + const std::map, unsigned long>& m) { os << "{"; - for (std::map, unsigned long>::const_iterator it = m.begin(); + for (std::map, unsigned long>::const_iterator it = m.begin(); it != m.end(); it++) { os << "[Key: {" << "Result: " << strResult((it->first).first) << ", ackType: " << (it->first).second diff --git a/lib/stats/ConsumerStatsImpl.h b/lib/stats/ConsumerStatsImpl.h index 5607cceb..a301e12b 100644 --- a/lib/stats/ConsumerStatsImpl.h +++ b/lib/stats/ConsumerStatsImpl.h @@ -20,23 +20,30 @@ #ifndef PULSAR_CONSUMER_STATS_IMPL_H_ #define PULSAR_CONSUMER_STATS_IMPL_H_ -#include -#include -#include +#include +#include +#include #include + +#include "ConsumerStatsBase.h" +#include "lib/ExecutorService.h" namespace pulsar { +using DeadlineTimerPtr = std::shared_ptr; +class ExecutorService; +using ExecutorServicePtr = std::shared_ptr; + class ConsumerStatsImpl : public ConsumerStatsBase { private: std::string consumerStr_; unsigned long numBytesRecieved_ = 0; std::map receivedMsgMap_; - std::map, unsigned long> ackedMsgMap_; + std::map, unsigned long> ackedMsgMap_; unsigned long totalNumBytesRecieved_ = 0; std::map totalReceivedMsgMap_; - std::map, unsigned long> totalAckedMsgMap_; + std::map, unsigned long> totalAckedMsgMap_; ExecutorServicePtr executor_; DeadlineTimerPtr timer_; @@ -52,11 +59,10 @@ class ConsumerStatsImpl : public ConsumerStatsBase { ConsumerStatsImpl(const ConsumerStatsImpl& stats); void flushAndReset(const boost::system::error_code&); virtual void receivedMessage(Message&, Result); - virtual void messageAcknowledged(Result, proto::CommandAck_AckType); + virtual void messageAcknowledged(Result, CommandAck_AckType); virtual ~ConsumerStatsImpl(); - const inline std::map, unsigned long>& getAckedMsgMap() - const { + const inline std::map, unsigned long>& getAckedMsgMap() const { return ackedMsgMap_; } @@ -64,8 +70,7 @@ class ConsumerStatsImpl : public ConsumerStatsBase { const inline std::map& getReceivedMsgMap() const { return receivedMsgMap_; } - inline const std::map, unsigned long>& getTotalAckedMsgMap() - const { + inline const std::map, unsigned long>& getTotalAckedMsgMap() const { return totalAckedMsgMap_; } diff --git a/lib/stats/ProducerStatsBase.h b/lib/stats/ProducerStatsBase.h index 0ae16d17..aafc8774 100644 --- a/lib/stats/ProducerStatsBase.h +++ b/lib/stats/ProducerStatsBase.h @@ -21,6 +21,7 @@ #define PULSAR_PRODUCER_STATS_BASE_HEADER #include #include + #include namespace pulsar { diff --git a/lib/stats/ProducerStatsDisabled.h b/lib/stats/ProducerStatsDisabled.h index 6568c074..df1df0f8 100644 --- a/lib/stats/ProducerStatsDisabled.h +++ b/lib/stats/ProducerStatsDisabled.h @@ -19,7 +19,7 @@ #ifndef PULSAR_PRODUCER_STATS_DISABLED_HEADER #define PULSAR_PRODUCER_STATS_DISABLED_HEADER -#include +#include "ProducerStatsBase.h" namespace pulsar { class ProducerStatsDisabled : public ProducerStatsBase { diff --git a/lib/stats/ProducerStatsImpl.cc b/lib/stats/ProducerStatsImpl.cc index e6f0221c..f30aebb4 100644 --- a/lib/stats/ProducerStatsImpl.cc +++ b/lib/stats/ProducerStatsImpl.cc @@ -17,12 +17,14 @@ * under the License. */ -#include - -#include +#include "ProducerStatsImpl.h" #include +#include "lib/ExecutorService.h" +#include "lib/LogUtils.h" +#include "lib/Utils.h" + namespace pulsar { DECLARE_LOG_OBJECT(); diff --git a/lib/stats/ProducerStatsImpl.h b/lib/stats/ProducerStatsImpl.h index 27ffacc8..a826ef00 100644 --- a/lib/stats/ProducerStatsImpl.h +++ b/lib/stats/ProducerStatsImpl.h @@ -20,29 +20,31 @@ #ifndef PULSAR_PRODUCER_STATS_IMPL_HEADER #define PULSAR_PRODUCER_STATS_IMPL_HEADER -#include #include -#include #if BOOST_VERSION >= 106400 #include #endif -#include - #include -#include #include +#include +#include #include - +#include #include +#include #include #include -#include #include -#include -#include + +#include "ProducerStatsBase.h" namespace pulsar { + +class ExecutorService; +using ExecutorServicePtr = std::shared_ptr; +using DeadlineTimerPtr = std::shared_ptr; + typedef boost::accumulators::accumulator_set< double, boost::accumulators::stats > diff --git a/perf/PerfConsumer.cc b/perf/PerfConsumer.cc index 6717fdeb..c4f1f008 100644 --- a/perf/PerfConsumer.cc +++ b/perf/PerfConsumer.cc @@ -20,27 +20,26 @@ DECLARE_LOG_OBJECT() #include -#include -#include #include -#include #include +#include +#include +#include using namespace std::chrono; -#include -#include #include -#include #include #include +#include +#include +#include namespace po = boost::program_options; using namespace boost::accumulators; #include - -#include #include +#include using namespace pulsar; static int64_t currentTimeMillis() { diff --git a/perf/PerfProducer.cc b/perf/PerfProducer.cc index f7d33612..04d5cbf2 100644 --- a/perf/PerfProducer.cc +++ b/perf/PerfProducer.cc @@ -19,26 +19,27 @@ #include DECLARE_LOG_OBJECT() -#include - #include -#include #include #include -#include +#include #include -#include +#include #include +#include +#include namespace po = boost::program_options; +#include +#include +#include + #include -#include #include +#include #include -#include + #include "RateLimiter.h" -#include -#include typedef std::shared_ptr RateLimiterPtr; struct Arguments { diff --git a/perf/RateLimiter.h b/perf/RateLimiter.h index eea76ec8..e65969ed 100644 --- a/perf/RateLimiter.h +++ b/perf/RateLimiter.h @@ -20,8 +20,8 @@ #define PERF_RATELIMITER_H_ #include -#include #include +#include namespace pulsar { diff --git a/tests/AuthBasicTest.cc b/tests/AuthBasicTest.cc index 29d66248..0b8b7f0c 100644 --- a/tests/AuthBasicTest.cc +++ b/tests/AuthBasicTest.cc @@ -17,9 +17,8 @@ * under the License. */ -#include - #include +#include #include #include diff --git a/tests/AuthPluginTest.cc b/tests/AuthPluginTest.cc index 8ba9563f..e2d33511 100644 --- a/tests/AuthPluginTest.cc +++ b/tests/AuthPluginTest.cc @@ -16,18 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -#include "pulsar/Authentication.h" #include +#include #include -#include + #include +#include #include -#include -#include -#include #include "lib/Future.h" +#include "lib/Latch.h" +#include "lib/LogUtils.h" #include "lib/Utils.h" +#include "lib/auth/AuthOauth2.h" DECLARE_LOG_OBJECT() using namespace pulsar; diff --git a/tests/AuthTokenTest.cc b/tests/AuthTokenTest.cc index fb14d4ce..8bdd2685 100644 --- a/tests/AuthTokenTest.cc +++ b/tests/AuthTokenTest.cc @@ -17,19 +17,18 @@ * under the License. */ -#include - #include +#include #include -#include -#include -#include -#include +#include +#include #include #include +#include #include "lib/Future.h" +#include "lib/LogUtils.h" #include "lib/Utils.h" DECLARE_LOG_OBJECT() diff --git a/tests/BackoffTest.cc b/tests/BackoffTest.cc index 8ecf5665..d066b944 100644 --- a/tests/BackoffTest.cc +++ b/tests/BackoffTest.cc @@ -17,9 +17,13 @@ * under the License. */ #include + #include -#include "Backoff.h" + #include "PulsarFriend.h" +#include "lib/Backoff.h" +#include "lib/ClientConnection.h" +#include "lib/stats/ProducerStatsImpl.h" using namespace pulsar; using boost::posix_time::milliseconds; diff --git a/tests/BasicEndToEndTest.cc b/tests/BasicEndToEndTest.cc index d3e424e6..61cf28a9 100644 --- a/tests/BasicEndToEndTest.cc +++ b/tests/BasicEndToEndTest.cc @@ -16,41 +16,38 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include +#include +#include + +#include #include -#include -#include #include +#include +#include +#include #include #include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include "CustomRoutingPolicy.h" #include "HttpHelper.h" #include "PulsarFriend.h" -#include "CustomRoutingPolicy.h" +#include "lib/AckGroupingTrackerDisabled.h" +#include "lib/AckGroupingTrackerEnabled.h" +#include "lib/ClientConnection.h" +#include "lib/ClientImpl.h" +#include "lib/Commands.h" +#include "lib/ConsumerImpl.h" +#include "lib/Future.h" +#include "lib/Latch.h" +#include "lib/LogUtils.h" +#include "lib/TimeUtils.h" +#include "lib/TopicName.h" +#include "lib/UnAckedMessageTrackerDisabled.h" +#include "lib/UnAckedMessageTrackerEnabled.h" +#include "lib/Utils.h" +#include "lib/stats/ProducerStatsImpl.h" DECLARE_LOG_OBJECT() @@ -3512,7 +3509,7 @@ class AckGroupingTrackerMock : public AckGroupingTracker { explicit AckGroupingTrackerMock(bool mockAck) : mockAck_(mockAck) {} bool callDoImmediateAck(ClientConnectionWeakPtr connWeakPtr, uint64_t consumerId, const MessageId &msgId, - proto::CommandAck_AckType ackType) { + CommandAck_AckType ackType) { if (!this->mockAck_) { // Not mocking ACK, expose this method. return this->doImmediateAck(connWeakPtr, consumerId, msgId, ackType); @@ -3586,7 +3583,7 @@ TEST(BasicEndToEndTest, testAckGroupingTrackerSingleAckBehavior) { auto connPtr = connWeakPtr.lock(); ASSERT_NE(connPtr, nullptr); ASSERT_TRUE(tracker.callDoImmediateAck(connWeakPtr, consumerImpl.getConsumerId(), recvMsgId[msgIdx], - proto::CommandAck::Individual)); + CommandAck_AckType_Individual)); } Message msg; ASSERT_EQ(ResultTimeout, consumer.receive(msg, 1000)); diff --git a/tests/BatchMessageTest.cc b/tests/BatchMessageTest.cc index 62fd5fff..273146ca 100644 --- a/tests/BatchMessageTest.cc +++ b/tests/BatchMessageTest.cc @@ -16,28 +16,27 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include +#include + #include #include #include -#include #include #include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include #include "ConsumerTest.h" #include "CustomRoutingPolicy.h" #include "HttpHelper.h" #include "PulsarFriend.h" +#include "lib/Commands.h" +#include "lib/Future.h" +#include "lib/Latch.h" +#include "lib/LogUtils.h" +#include "lib/ProtoApiEnums.h" +#include "lib/Utils.h" +#include "lib/stats/ProducerStatsImpl.h" DECLARE_LOG_OBJECT(); @@ -330,8 +329,8 @@ TEST(BatchMessageTest, testSmallReceiverQueueSize) { } ConsumerStatsImplPtr consumerStatsImplPtr = PulsarFriend::getConsumerStatsPtr(consumer); - unsigned long t = consumerStatsImplPtr->getAckedMsgMap().at( - std::make_pair(ResultOk, proto::CommandAck_AckType_Individual)); + unsigned long t = + consumerStatsImplPtr->getAckedMsgMap().at(std::make_pair(ResultOk, CommandAck_AckType_Individual)); ASSERT_EQ(t, numOfMessages); ASSERT_EQ(PulsarFriend::sum(consumerStatsImplPtr->getAckedMsgMap()), numOfMessages); ASSERT_EQ(PulsarFriend::sum(consumerStatsImplPtr->getTotalAckedMsgMap()), numOfMessages); @@ -581,8 +580,8 @@ TEST(BatchMessageTest, testCumulativeAck) { ASSERT_EQ(consumerStatsImplPtr->getReceivedMsgMap().at(ResultTimeout), 1); ASSERT_EQ(PulsarFriend::sum(consumerStatsImplPtr->getAckedMsgMap()), 1); ASSERT_EQ(producerStatsImplPtr->getNumBytesSent(), consumerStatsImplPtr->getNumBytesRecieved()); - unsigned long t = consumerStatsImplPtr->getAckedMsgMap().at( - std::make_pair(ResultOk, proto::CommandAck_AckType_Cumulative)); + unsigned long t = + consumerStatsImplPtr->getAckedMsgMap().at(std::make_pair(ResultOk, CommandAck_AckType_Cumulative)); ASSERT_EQ(t, 1); // Number of messages produced @@ -612,8 +611,7 @@ TEST(BatchMessageTest, testCumulativeAck) { } ASSERT_EQ(PulsarFriend::sum(consumerStatsImplPtr->getAckedMsgMap()), 1); - t = consumerStatsImplPtr->getAckedMsgMap().at( - std::make_pair(ResultOk, proto::CommandAck_AckType_Cumulative)); + t = consumerStatsImplPtr->getAckedMsgMap().at(std::make_pair(ResultOk, CommandAck_AckType_Cumulative)); ASSERT_EQ(t, 1); // Number of messages consumed diff --git a/tests/BlockingQueueTest.cc b/tests/BlockingQueueTest.cc index 94b0a1bb..0b86b5b4 100644 --- a/tests/BlockingQueueTest.cc +++ b/tests/BlockingQueueTest.cc @@ -17,13 +17,14 @@ * under the License. */ #include -#include -#include #include #include #include +#include "lib/BlockingQueue.h" +#include "lib/Latch.h" + class ProducerWorker { private: std::thread producerThread_; diff --git a/tests/ClientDeduplicationTest.cc b/tests/ClientDeduplicationTest.cc index c3373d5d..90991b99 100644 --- a/tests/ClientDeduplicationTest.cc +++ b/tests/ClientDeduplicationTest.cc @@ -16,15 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -#include - #include - -#include "HttpHelper.h" +#include #include #include +#include "HttpHelper.h" + using namespace pulsar; static std::string serviceUrl = "pulsar://localhost:6650"; diff --git a/tests/ClientTest.cc b/tests/ClientTest.cc index aa48bdc4..bdb05555 100644 --- a/tests/ClientTest.cc +++ b/tests/ClientTest.cc @@ -17,14 +17,16 @@ * under the License. */ #include +#include + +#include #include "HttpHelper.h" #include "PulsarFriend.h" - -#include -#include -#include "../lib/checksum/ChecksumProvider.h" +#include "lib/ClientConnection.h" #include "lib/LogUtils.h" +#include "lib/checksum/ChecksumProvider.h" +#include "lib/stats/ProducerStatsImpl.h" DECLARE_LOG_OBJECT() diff --git a/tests/CompressionCodecSnappyTest.cc b/tests/CompressionCodecSnappyTest.cc index 27d668f3..c05c4026 100644 --- a/tests/CompressionCodecSnappyTest.cc +++ b/tests/CompressionCodecSnappyTest.cc @@ -18,7 +18,7 @@ */ #include -#include "../lib/CompressionCodecSnappy.h" +#include "lib/CompressionCodecSnappy.h" using namespace pulsar; diff --git a/tests/ConsumerConfigurationTest.cc b/tests/ConsumerConfigurationTest.cc index 20cd8f4b..fde87364 100644 --- a/tests/ConsumerConfigurationTest.cc +++ b/tests/ConsumerConfigurationTest.cc @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include #include #include +#include + #include "NoOpsCryptoKeyReader.h" DECLARE_LOG_OBJECT() diff --git a/tests/ConsumerStatsTest.cc b/tests/ConsumerStatsTest.cc index c398a532..3c1959a2 100644 --- a/tests/ConsumerStatsTest.cc +++ b/tests/ConsumerStatsTest.cc @@ -18,17 +18,18 @@ */ #include #include -#include -#include -#include "lib/Future.h" -#include "lib/Utils.h" -#include "PulsarFriend.h" -#include "ConsumerTest.h" -#include "HttpHelper.h" -#include #include #include + +#include "ConsumerTest.h" +#include "HttpHelper.h" +#include "PulsarFriend.h" +#include "lib/Future.h" +#include "lib/Latch.h" +#include "lib/LogUtils.h" +#include "lib/MultiTopicsBrokerConsumerStatsImpl.h" +#include "lib/Utils.h" DECLARE_LOG_OBJECT(); using namespace pulsar; diff --git a/tests/ConsumerTest.cc b/tests/ConsumerTest.cc index c8a07e6c..e35a1f02 100644 --- a/tests/ConsumerTest.cc +++ b/tests/ConsumerTest.cc @@ -16,22 +16,27 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include + #include -#include -#include -#include +#include #include +#include +#include #include -#include "gtest/gtest.h" - -#include "pulsar/Client.h" +#include "HttpHelper.h" #include "PulsarFriend.h" +#include "lib/ClientConnection.h" #include "lib/Future.h" -#include "lib/Utils.h" #include "lib/LogUtils.h" #include "lib/MultiTopicsConsumerImpl.h" -#include "HttpHelper.h" +#include "lib/TimeUtils.h" +#include "lib/UnAckedMessageTrackerDisabled.h" +#include "lib/UnAckedMessageTrackerEnabled.h" +#include "lib/Utils.h" +#include "lib/stats/ProducerStatsImpl.h" static const std::string lookupUrl = "pulsar://localhost:6650"; static const std::string adminUrl = "http://localhost:8080/"; diff --git a/tests/ConsumerTest.h b/tests/ConsumerTest.h index 8c7d3f7d..ca84aa7e 100644 --- a/tests/ConsumerTest.h +++ b/tests/ConsumerTest.h @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include "lib/ConsumerImpl.h" #include +#include "lib/ConsumerImpl.h" + using std::string; namespace pulsar { diff --git a/tests/CustomLoggerTest.cc b/tests/CustomLoggerTest.cc index bd80c312..dde4056d 100644 --- a/tests/CustomLoggerTest.cc +++ b/tests/CustomLoggerTest.cc @@ -16,13 +16,15 @@ * specific language governing permissions and limitations * under the License. */ +#include #include #include -#include -#include + #include #include +#include "lib/LogUtils.h" + using namespace pulsar; static std::vector logLines; diff --git a/tests/CustomRoutingPolicy.h b/tests/CustomRoutingPolicy.h index ed10c5ba..0cbf8b40 100644 --- a/tests/CustomRoutingPolicy.h +++ b/tests/CustomRoutingPolicy.h @@ -19,10 +19,11 @@ #ifndef CUSTOM_ROUTER_POLICY_HEADER_ #define CUSTOM_ROUTER_POLICY_HEADER_ -#include // rand() -#include #include +#include +#include // rand() + namespace pulsar { class CustomRoutingPolicy : public MessageRoutingPolicy { /** @deprecated */ diff --git a/tests/HashTest.cc b/tests/HashTest.cc index bd6de09e..76a49fd0 100644 --- a/tests/HashTest.cc +++ b/tests/HashTest.cc @@ -16,14 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include #include +#include +#include + #include -#include "../lib/BoostHash.h" -#include "../lib/JavaStringHash.h" -#include "../lib/Murmur3_32Hash.h" +#include "lib/BoostHash.h" +#include "lib/JavaStringHash.h" +#include "lib/Murmur3_32Hash.h" using ::testing::AtLeast; using ::testing::Return; diff --git a/tests/KeyBasedBatchingTest.cc b/tests/KeyBasedBatchingTest.cc index fcb558a7..e5962669 100644 --- a/tests/KeyBasedBatchingTest.cc +++ b/tests/KeyBasedBatchingTest.cc @@ -16,14 +16,15 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include #include + #include #include #include -#include -#include -#include +#include "lib/Latch.h" using namespace pulsar; diff --git a/tests/KeySharedConsumerTest.cc b/tests/KeySharedConsumerTest.cc index 46603340..74da5b30 100644 --- a/tests/KeySharedConsumerTest.cc +++ b/tests/KeySharedConsumerTest.cc @@ -16,17 +16,17 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include + #include #include -#include #include - -#include -#include -#include "lib/LogUtils.h" +#include #include "HttpHelper.h" #include "LogHelper.h" +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/tests/KeySharedPolicyTest.cc b/tests/KeySharedPolicyTest.cc index 49fef3fd..ff43102a 100644 --- a/tests/KeySharedPolicyTest.cc +++ b/tests/KeySharedPolicyTest.cc @@ -16,18 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include -#include -#include - #include #include #include -#include "lib/LogUtils.h" + +#include +#include +#include +#include #include "HttpHelper.h" #include "LogHelper.h" +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/tests/LatchTest.cc b/tests/LatchTest.cc index c69141ef..041d2e38 100644 --- a/tests/LatchTest.cc +++ b/tests/LatchTest.cc @@ -17,9 +17,11 @@ * under the License. */ #include -#include + #include -#include "LogUtils.h" + +#include "lib/Latch.h" +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/tests/LoggerTest.cc b/tests/LoggerTest.cc index d26ccc6b..2032744f 100644 --- a/tests/LoggerTest.cc +++ b/tests/LoggerTest.cc @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include "LogUtils.h" #include +#include "lib/LogUtils.h" + DECLARE_LOG_OBJECT() TEST(LoggerTest, testLogger) { diff --git a/tests/LookupServiceTest.cc b/tests/LookupServiceTest.cc index 77c1e1aa..5ac41224 100644 --- a/tests/LookupServiceTest.cc +++ b/tests/LookupServiceTest.cc @@ -17,21 +17,23 @@ * under the License. */ #include -#include -#include - -#include #include +#include #include -#include "ConnectionPool.h" -#include "HttpHelper.h" +#include #include -#include -#include "LogUtils.h" -#include "RetryableLookupService.h" -#include "PulsarFriend.h" +#include #include +#include + +#include "HttpHelper.h" +#include "PulsarFriend.h" +#include "lib/ClientConnection.h" +#include "lib/ConnectionPool.h" +#include "lib/LogUtils.h" +#include "lib/RetryableLookupService.h" +#include "lib/TimeUtils.h" using namespace pulsar; diff --git a/tests/MapCacheTest.cc b/tests/MapCacheTest.cc index 12a89ee1..2140937f 100644 --- a/tests/MapCacheTest.cc +++ b/tests/MapCacheTest.cc @@ -17,7 +17,8 @@ * under the License. */ #include -#include + +#include "lib/MapCache.h" using namespace pulsar; diff --git a/tests/MemoryLimitControllerTest.cc b/tests/MemoryLimitControllerTest.cc index eb63760e..a462aefe 100644 --- a/tests/MemoryLimitControllerTest.cc +++ b/tests/MemoryLimitControllerTest.cc @@ -18,10 +18,11 @@ */ #include + #include -#include "../lib/MemoryLimitController.h" -#include "../lib/Latch.h" +#include "lib/Latch.h" +#include "lib/MemoryLimitController.h" using namespace pulsar; @@ -127,4 +128,4 @@ TEST(MemoryLimitControllerTest, testStepRelease) { t1.join(); t2.join(); t3.join(); -} \ No newline at end of file +} diff --git a/tests/MemoryLimitTest.cc b/tests/MemoryLimitTest.cc index cb0b47ae..39c0f6e2 100644 --- a/tests/MemoryLimitTest.cc +++ b/tests/MemoryLimitTest.cc @@ -18,15 +18,15 @@ */ #include +#include + #include #include -#include "../lib/MemoryLimitController.h" -#include "../lib/Latch.h" -#include "../lib/Future.h" -#include "../lib/Utils.h" - -#include +#include "lib/Future.h" +#include "lib/Latch.h" +#include "lib/MemoryLimitController.h" +#include "lib/Utils.h" using namespace pulsar; @@ -160,4 +160,4 @@ TEST(MemoryLimitTest, testNoProducerQueueSize) { Result res = p.getFuture().get(id); ASSERT_EQ(res, ResultOk); } -} \ No newline at end of file +} diff --git a/tests/MessageChunkingTest.cc b/tests/MessageChunkingTest.cc index ae0114ce..61a97144 100644 --- a/tests/MessageChunkingTest.cc +++ b/tests/MessageChunkingTest.cc @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include + #include #include -#include -#include -#include "lib/LogUtils.h" #include "PulsarFriend.h" +#include "lib/LogUtils.h" DECLARE_LOG_OBJECT() diff --git a/tests/MessageIdTest.cc b/tests/MessageIdTest.cc index 55fa181d..55257d92 100644 --- a/tests/MessageIdTest.cc +++ b/tests/MessageIdTest.cc @@ -16,14 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include "lib/MessageIdUtil.h" -#include "PulsarFriend.h" - #include +#include #include +#include "PulsarFriend.h" +#include "lib/MessageIdUtil.h" + using namespace pulsar; TEST(MessageIdTest, testSerialization) { diff --git a/tests/MessageTest.cc b/tests/MessageTest.cc index fcc22e97..7e26431e 100644 --- a/tests/MessageTest.cc +++ b/tests/MessageTest.cc @@ -16,11 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include #include +#include +#include + #include -#include + +#include "lib/LogUtils.h" using namespace pulsar; TEST(MessageTest, testMessageContents) { diff --git a/tests/MessagesImplTest.cc b/tests/MessagesImplTest.cc index e963501a..a0fdc7a3 100644 --- a/tests/MessagesImplTest.cc +++ b/tests/MessagesImplTest.cc @@ -17,8 +17,9 @@ * under the License. */ #include -#include -#include "pulsar/MessageBuilder.h" +#include + +#include "lib/MessagesImpl.h" using namespace pulsar; diff --git a/tests/NamespaceNameTest.cc b/tests/NamespaceNameTest.cc index 51975c82..132506ea 100644 --- a/tests/NamespaceNameTest.cc +++ b/tests/NamespaceNameTest.cc @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -#include - #include + +#include "lib/NamespaceName.h" using namespace pulsar; TEST(NamespaceNameTest, testNamespaceName) { diff --git a/tests/PartitionsUpdateTest.cc b/tests/PartitionsUpdateTest.cc index 845e4477..010e5cb0 100644 --- a/tests/PartitionsUpdateTest.cc +++ b/tests/PartitionsUpdateTest.cc @@ -19,13 +19,13 @@ #include #include -#include #include -#include #include +#include +#include -#include "HttpHelper.h" #include "CustomRoutingPolicy.h" +#include "HttpHelper.h" using namespace pulsar; diff --git a/tests/PeriodicTaskTest.cc b/tests/PeriodicTaskTest.cc index 2c1da70e..7b048173 100644 --- a/tests/PeriodicTaskTest.cc +++ b/tests/PeriodicTaskTest.cc @@ -17,9 +17,11 @@ * under the License. */ #include + #include #include #include + #include "lib/ExecutorService.h" #include "lib/LogUtils.h" #include "lib/PeriodicTask.h" diff --git a/tests/ProducerConfigurationTest.cc b/tests/ProducerConfigurationTest.cc index 5c541295..df5867c1 100644 --- a/tests/ProducerConfigurationTest.cc +++ b/tests/ProducerConfigurationTest.cc @@ -18,6 +18,7 @@ */ #include #include + #include "NoOpsCryptoKeyReader.h" using namespace pulsar; diff --git a/tests/ProducerTest.cc b/tests/ProducerTest.cc index 36b23eed..ee07cbb9 100644 --- a/tests/ProducerTest.cc +++ b/tests/ProducerTest.cc @@ -16,17 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include + #include #include "HttpHelper.h" - #include "lib/Future.h" -#include "lib/Utils.h" #include "lib/Latch.h" #include "lib/LogUtils.h" #include "lib/ProducerImpl.h" +#include "lib/Utils.h" DECLARE_LOG_OBJECT() using namespace pulsar; diff --git a/tests/PromiseTest.cc b/tests/PromiseTest.cc index 73c6f8c2..25b6b723 100644 --- a/tests/PromiseTest.cc +++ b/tests/PromiseTest.cc @@ -17,12 +17,14 @@ * under the License. */ #include -#include + #include #include #include #include +#include "lib/Future.h" + using namespace pulsar; TEST(PromiseTest, testSetValue) { diff --git a/tests/ProtobufNativeSchemaTest.cc b/tests/ProtobufNativeSchemaTest.cc index f9557bd0..37bd7804 100644 --- a/tests/ProtobufNativeSchemaTest.cc +++ b/tests/ProtobufNativeSchemaTest.cc @@ -19,9 +19,11 @@ #include #include #include + #include + #include "PaddingDemo.pb.h" -#include "Test.pb.h" // generated from "pulsar-client/src/test/proto/Test.proto" +#include "Test.pb.h" using namespace pulsar; diff --git a/tests/PulsarFriend.h b/tests/PulsarFriend.h index df8e3dc0..18f2bb66 100644 --- a/tests/PulsarFriend.h +++ b/tests/PulsarFriend.h @@ -19,13 +19,17 @@ #include +#include "lib/ClientConnection.h" #include "lib/ClientImpl.h" -#include "lib/ProducerImpl.h" -#include "lib/PartitionedProducerImpl.h" #include "lib/ConsumerImpl.h" #include "lib/MultiTopicsConsumerImpl.h" +#include "lib/NamespaceName.h" +#include "lib/PartitionedProducerImpl.h" +#include "lib/ProducerImpl.h" #include "lib/ReaderImpl.h" #include "lib/RetryableLookupService.h" +#include "lib/stats/ConsumerStatsImpl.h" +#include "lib/stats/ProducerStatsImpl.h" using std::string; diff --git a/tests/ReaderConfigurationTest.cc b/tests/ReaderConfigurationTest.cc index 8dc60f44..5783ac98 100644 --- a/tests/ReaderConfigurationTest.cc +++ b/tests/ReaderConfigurationTest.cc @@ -22,8 +22,9 @@ */ #include #include -#include + #include "NoOpsCryptoKeyReader.h" +#include "lib/ReaderImpl.h" using namespace pulsar; diff --git a/tests/ReaderTest.cc b/tests/ReaderTest.cc index 799702f9..b88da998 100644 --- a/tests/ReaderTest.cc +++ b/tests/ReaderTest.cc @@ -16,19 +16,19 @@ * specific language governing permissions and limitations * under the License. */ +#include #include #include -#include "HttpHelper.h" -#include "PulsarFriend.h" - -#include - #include + #include -#include -#include -#include +#include "HttpHelper.h" +#include "PulsarFriend.h" +#include "lib/ClientConnection.h" +#include "lib/Latch.h" +#include "lib/LogUtils.h" +#include "lib/ReaderImpl.h" DECLARE_LOG_OBJECT() using namespace pulsar; diff --git a/tests/RoundRobinMessageRouterTest.cc b/tests/RoundRobinMessageRouterTest.cc index ce5ad170..56a76050 100644 --- a/tests/RoundRobinMessageRouterTest.cc +++ b/tests/RoundRobinMessageRouterTest.cc @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ +#include #include #include -#include + #include -#include "../lib/RoundRobinMessageRouter.h" -#include "../lib/TopicMetadataImpl.h" +#include "lib/RoundRobinMessageRouter.h" +#include "lib/TopicMetadataImpl.h" using namespace pulsar; diff --git a/tests/SemaphoreTest.cc b/tests/SemaphoreTest.cc index 0cdec79f..de3da4f2 100644 --- a/tests/SemaphoreTest.cc +++ b/tests/SemaphoreTest.cc @@ -18,10 +18,11 @@ */ #include + #include -#include "../lib/Semaphore.h" -#include "../lib/Latch.h" +#include "lib/Latch.h" +#include "lib/Semaphore.h" using namespace pulsar; diff --git a/tests/ServiceURITest.cc b/tests/ServiceURITest.cc index 9d4c88fc..4e01ab2d 100644 --- a/tests/ServiceURITest.cc +++ b/tests/ServiceURITest.cc @@ -17,6 +17,7 @@ * under the License. */ #include + #include "lib/ServiceURI.h" using namespace pulsar; diff --git a/tests/ShutdownTest.cc b/tests/ShutdownTest.cc index 39513474..1dca019c 100644 --- a/tests/ShutdownTest.cc +++ b/tests/ShutdownTest.cc @@ -16,14 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include #include #include -#include "lib/ClientImpl.h" + +#include +#include + #include "HttpHelper.h" #include "PulsarFriend.h" #include "WaitUtils.h" +#include "lib/ClientImpl.h" using namespace pulsar; diff --git a/tests/SinglePartitionMessageRouterTest.cc b/tests/SinglePartitionMessageRouterTest.cc index e82d080f..788f19b3 100644 --- a/tests/SinglePartitionMessageRouterTest.cc +++ b/tests/SinglePartitionMessageRouterTest.cc @@ -16,17 +16,17 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include #include #include + #include -#include -#include +#include "lib/SinglePartitionMessageRouter.h" +#include "lib/TopicMetadataImpl.h" #include "tests/mocks/GMockMessage.h" -#include "../lib/SinglePartitionMessageRouter.h" -#include "../lib/TopicMetadataImpl.h" - using ::testing::AtLeast; using ::testing::Return; using ::testing::ReturnRef; @@ -70,4 +70,4 @@ TEST(SinglePartitionMessageRouterTest, DISABLED_getPartitionWithPartitionKey) { ASSERT_EQ(expectedParrtition1, router.getPartition(message1, TopicMetadataImpl(numPartitons))); ASSERT_EQ(expectedParrtition2, router.getPartition(message2, TopicMetadataImpl(numPartitons))); -} \ No newline at end of file +} diff --git a/tests/SynchronizedHashMapTest.cc b/tests/SynchronizedHashMapTest.cc index 8d74a240..87cbe1e3 100644 --- a/tests/SynchronizedHashMapTest.cc +++ b/tests/SynchronizedHashMapTest.cc @@ -17,11 +17,13 @@ * under the License. */ #include + #include #include #include #include #include + #include "lib/Latch.h" #include "lib/SynchronizedHashMap.h" diff --git a/tests/TopicMetadataImplTest.cc b/tests/TopicMetadataImplTest.cc index 091dd265..bdc68789 100644 --- a/tests/TopicMetadataImplTest.cc +++ b/tests/TopicMetadataImplTest.cc @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include -#include "../lib/TopicMetadataImpl.h" +#include "lib/TopicMetadataImpl.h" using namespace pulsar; diff --git a/tests/TopicNameTest.cc b/tests/TopicNameTest.cc index 3db2a6c0..44b3dc2d 100644 --- a/tests/TopicNameTest.cc +++ b/tests/TopicNameTest.cc @@ -17,9 +17,11 @@ * under the License. */ #include -#include + #include +#include "lib/TopicName.h" + using namespace pulsar; TEST(TopicNameTest, testLookup) { diff --git a/tests/UnboundedBlockingQueueTest.cc b/tests/UnboundedBlockingQueueTest.cc index 819c22e5..bccb68d9 100644 --- a/tests/UnboundedBlockingQueueTest.cc +++ b/tests/UnboundedBlockingQueueTest.cc @@ -17,12 +17,13 @@ * under the License. */ #include -#include -#include #include #include +#include "lib/Latch.h" +#include "lib/UnboundedBlockingQueue.h" + class UnboundedProducerWorker { private: std::thread producerThread_; diff --git a/tests/UrlTest.cc b/tests/UrlTest.cc index 6cd2d1c8..c470f5b2 100644 --- a/tests/UrlTest.cc +++ b/tests/UrlTest.cc @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include "Url.h" #include +#include "lib/Url.h" + using namespace pulsar; TEST(UrlTest, testUrl) { diff --git a/tests/VersionTest.cc b/tests/VersionTest.cc index 57e1e783..0fa790dc 100644 --- a/tests/VersionTest.cc +++ b/tests/VersionTest.cc @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include TEST(VersionTest, testMacro) { #ifdef PULSAR_VERSION diff --git a/tests/ZLibCompressionTest.cc b/tests/ZLibCompressionTest.cc index c510db15..249d7ee9 100644 --- a/tests/ZLibCompressionTest.cc +++ b/tests/ZLibCompressionTest.cc @@ -17,7 +17,8 @@ * under the License. */ #include -#include + +#include "lib/CompressionCodecZLib.h" using namespace pulsar; diff --git a/tests/ZTSClientTest.cc b/tests/ZTSClientTest.cc index b338e795..fe6de0bf 100644 --- a/tests/ZTSClientTest.cc +++ b/tests/ZTSClientTest.cc @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include "lib/auth/athenz/ZTSClient.h" #include +#include "lib/auth/athenz/ZTSClient.h" + using namespace pulsar; namespace pulsar { diff --git a/tests/ZeroQueueSizeTest.cc b/tests/ZeroQueueSizeTest.cc index 59780fed..644f42c4 100644 --- a/tests/ZeroQueueSizeTest.cc +++ b/tests/ZeroQueueSizeTest.cc @@ -18,8 +18,7 @@ */ #include #include -#include -#include "ConsumerTest.h" + #include #include #include @@ -27,6 +26,10 @@ #include #include +#include "ConsumerTest.h" +#include "lib/Latch.h" +#include "lib/LogUtils.h" + DECLARE_LOG_OBJECT() using namespace pulsar; diff --git a/tests/c/c_BasicEndToEndTest.cc b/tests/c/c_BasicEndToEndTest.cc index ae01befe..04aa1dcd 100644 --- a/tests/c/c_BasicEndToEndTest.cc +++ b/tests/c/c_BasicEndToEndTest.cc @@ -17,12 +17,12 @@ * under the License. */ -#include +#include +#include #include #include -#include -#include +#include struct send_ctx { pulsar_result result; diff --git a/tests/c/c_ConsumerConfigurationTest.cc b/tests/c/c_ConsumerConfigurationTest.cc index c6afeefb..e877867f 100644 --- a/tests/c/c_ConsumerConfigurationTest.cc +++ b/tests/c/c_ConsumerConfigurationTest.cc @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include TEST(C_ConsumerConfigurationTest, testCApiConfig) { pulsar_consumer_configuration_t *consumer_conf = pulsar_consumer_configuration_create(); diff --git a/tests/c/c_ProducerConfigurationTest.cc b/tests/c/c_ProducerConfigurationTest.cc index 507b3a91..16142272 100644 --- a/tests/c/c_ProducerConfigurationTest.cc +++ b/tests/c/c_ProducerConfigurationTest.cc @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include TEST(C_ProducerConfigurationTest, testCApiConfig) { pulsar_producer_configuration_t *producer_conf = pulsar_producer_configuration_create(); diff --git a/tests/main.cc b/tests/main.cc index 06599258..6104501a 100644 --- a/tests/main.cc +++ b/tests/main.cc @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -#include #include int main(int argc, char **argv) { diff --git a/wireshark/pulsarDissector.cc b/wireshark/pulsarDissector.cc index 9ff311e2..38bfa4e3 100644 --- a/wireshark/pulsarDissector.cc +++ b/wireshark/pulsarDissector.cc @@ -16,15 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include +#include #include #include #include #include -#include -#include #include +#include #include #include "PulsarApi.pb.h" From ecc1995cdc26fa79f5a33f0fe9a17d4ec4657e22 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 25 Oct 2022 03:54:51 +0800 Subject: [PATCH 17/19] Stick clang-format version to 11 and add back docker-format.sh (#45) * Stick clang-format version to 11 and add back docker-format.sh ### Motivation Currently the code is formatted by `clang-format` 11. If it's formatted by other versions of `clang-format`, the style might be a little different. ### Modifications - Add back `docker-format.sh` and the associated `Dockerfile.format`. This script build the image with `python3` and `clang-format-11` installed and the entrypoint is formatting the code. - Explain the `clang-format` version in README. * Move Dockerfile and docker-format.sh to build-support --- README.md | 4 +++- build-support/Dockerfile.format | 35 ++++++++++++++++++++++++++++++ build-support/docker-format.sh | 32 +++++++++++++++++++++++++++ cmake_modules/FindClangTools.cmake | 4 ++-- 4 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 build-support/Dockerfile.format create mode 100755 build-support/docker-format.sh diff --git a/README.md b/README.md index 4cea378a..9701ac33 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,8 @@ cd tests ## Requirements for Contributors -It's required to install [LLVM](https://llvm.org/builds/) for `clang-tidy` and `clang-format`. Pulsar C++ client use `clang-format` 6.0+ to format files. `make format` automatically formats the files. +It's required to install [LLVM](https://llvm.org/builds/) for `clang-tidy` and `clang-format`. Pulsar C++ client use `clang-format` **11** to format files. `make format` automatically formats the files. + +For Ubuntu users, you can install `clang-format-11` via `apt install clang-format-11`. For other users, run `./build-support/docker-format.sh` if you have Docker installed. We welcome contributions from the open source community, kindly make sure your changes are backward compatible with GCC 4.8 and Boost 1.53. diff --git a/build-support/Dockerfile.format b/build-support/Dockerfile.format new file mode 100644 index 00000000..92d53b03 --- /dev/null +++ b/build-support/Dockerfile.format @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +FROM ubuntu:20.04 + +WORKDIR /app + +RUN apt update -y && apt install -y python3 clang-format-11 +ENTRYPOINT ["python3", "./build-support/run_clang_format.py", \ + "clang-format-11", \ + "0", \ + "./build-support/clang_format_exclusions.txt", \ + "./lib", \ + "./perf", \ + "./examples", \ + "./tests", \ + "./include", \ + "./wireshark"] diff --git a/build-support/docker-format.sh b/build-support/docker-format.sh new file mode 100755 index 00000000..622a4f15 --- /dev/null +++ b/build-support/docker-format.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd $ROOT_DIR/build-support + +IMAGE_NAME=apachepulsar/cpp-client-format +docker image inspect apachepulsar/cpp-client-format 1>/dev/null 2>&1 +OK=$? +set -e +if [[ $OK -ne 0 ]]; then + echo "The image $IMAGE_NAME doesn't exist, build it" + docker build -t $IMAGE_NAME -f ./Dockerfile.format . +fi +docker run -v $ROOT_DIR:/app --rm $IMAGE_NAME diff --git a/cmake_modules/FindClangTools.cmake b/cmake_modules/FindClangTools.cmake index 4b8fc18e..67128818 100644 --- a/cmake_modules/FindClangTools.cmake +++ b/cmake_modules/FindClangTools.cmake @@ -82,8 +82,8 @@ if (CLANG_FORMAT_VERSION) endif() else() find_program(CLANG_FORMAT_BIN - NAMES clang-format-5 - clang-format-5.0 + NAMES clang-format-11 + clang-format-11.0 clang-format PATHS ${CLANG_SEARCH_PATHS} NO_DEFAULT_PATH From 712b484bcb52a7a0e8efbfdd70bbc746f2bb996f Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Fri, 7 Oct 2022 19:47:59 +0800 Subject: [PATCH 18/19] [improve] Support KeyValue Schema. --- .gitignore | 4 +- examples/CMakeLists.txt | 51 ++++++---- examples/SampleKeyValueSchemaConsumer.cc | 61 ++++++++++++ examples/SampleKeyValueSchemaProducer.cc | 60 ++++++++++++ include/pulsar/KeyValue.h | 83 ++++++++++++++++ include/pulsar/Message.h | 8 ++ include/pulsar/MessageBuilder.h | 8 ++ include/pulsar/Schema.h | 31 ++++++ lib/Commands.cc | 3 + lib/ConsumerImpl.cc | 4 + lib/KeyValue.cc | 37 ++++++++ lib/KeyValueImpl.cc | 76 +++++++++++++++ lib/KeyValueImpl.h | 49 ++++++++++ lib/Message.cc | 4 +- lib/MessageBuilder.cc | 7 +- lib/MessageImpl.cc | 35 +++++++ lib/MessageImpl.h | 5 + lib/ProducerImpl.cc | 2 + lib/Schema.cc | 80 ++++++++++++++++ tests/KeyValueImplTest.cc | 115 +++++++++++++++++++++++ tests/KeyValueSchemaTest.cc | 87 +++++++++++++++++ tests/MessageTest.cc | 49 +++++++++- tests/SchemaTest.cc | 50 +++++++++- 23 files changed, 882 insertions(+), 27 deletions(-) create mode 100644 examples/SampleKeyValueSchemaConsumer.cc create mode 100644 examples/SampleKeyValueSchemaProducer.cc create mode 100644 include/pulsar/KeyValue.h create mode 100644 lib/KeyValue.cc create mode 100644 lib/KeyValueImpl.cc create mode 100644 lib/KeyValueImpl.h create mode 100644 tests/KeyValueImplTest.cc create mode 100644 tests/KeyValueSchemaTest.cc diff --git a/.gitignore b/.gitignore index 46092dc2..4fd7cf1b 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,8 @@ apache-pulsar-client-cpp-*.tar.gz /examples/SampleConsumerListener /examples/SampleConsumerListenerCApi /examples/SampleReaderCApi +/examples/SampleKeyValueSchemaConsumer +/examples/SampleKeyValueSchemaProducer /examples/SampleFileLogger /tests/main /tests/pulsar-tests @@ -98,4 +100,4 @@ vcpkg_installed/ *.rej .tests-container-id.txt Testing -.test-token.txt \ No newline at end of file +.test-token.txt diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 72422d28..e84fbbbf 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -60,25 +60,36 @@ set(SAMPLE_CONSUMER_LISTENER_C_SOURCES set(SAMPLE_READER_C_SOURCES SampleReaderCApi.c ) +set(SAMPLE_KEY_VALUE_SCHEMA_CONSUMER + SampleKeyValueSchemaConsumer.cc +) + +set(SAMPLE_KEY_VALUE_SCHEMA_PRODUCER + SampleKeyValueSchemaProducer.cc +) -add_executable(SampleAsyncProducer ${SAMPLE_ASYNC_PRODUCER_SOURCES}) -add_executable(SampleConsumer ${SAMPLE_CONSUMER_SOURCES}) -add_executable(SampleConsumerListener ${SAMPLE_CONSUMER_LISTENER_SOURCES}) -add_executable(SampleProducer ${SAMPLE_PRODUCER_SOURCES}) -add_executable(SampleFileLogger ${SAMPLE_FILE_LOGGER_SOURCES}) -add_executable(SampleProducerCApi ${SAMPLE_PRODUCER_C_SOURCES}) -add_executable(SampleConsumerCApi ${SAMPLE_CONSUMER_C_SOURCES}) -add_executable(SampleAsyncConsumerCApi ${SAMPLE_CONSUMER_LISTENER_C_SOURCES}) -add_executable(SampleConsumerListenerCApi ${SAMPLE_CONSUMER_LISTENER_C_SOURCES}) -add_executable(SampleReaderCApi ${SAMPLE_READER_C_SOURCES}) +add_executable(SampleAsyncProducer ${SAMPLE_ASYNC_PRODUCER_SOURCES}) +add_executable(SampleConsumer ${SAMPLE_CONSUMER_SOURCES}) +add_executable(SampleConsumerListener ${SAMPLE_CONSUMER_LISTENER_SOURCES}) +add_executable(SampleProducer ${SAMPLE_PRODUCER_SOURCES}) +add_executable(SampleFileLogger ${SAMPLE_FILE_LOGGER_SOURCES}) +add_executable(SampleProducerCApi ${SAMPLE_PRODUCER_C_SOURCES}) +add_executable(SampleConsumerCApi ${SAMPLE_CONSUMER_C_SOURCES}) +add_executable(SampleAsyncConsumerCApi ${SAMPLE_CONSUMER_LISTENER_C_SOURCES}) +add_executable(SampleConsumerListenerCApi ${SAMPLE_CONSUMER_LISTENER_C_SOURCES}) +add_executable(SampleReaderCApi ${SAMPLE_READER_C_SOURCES}) +add_executable(SampleKeyValueSchemaConsumer ${SAMPLE_KEY_VALUE_SCHEMA_CONSUMER}) +add_executable(SampleKeyValueSchemaProducer ${SAMPLE_KEY_VALUE_SCHEMA_PRODUCER}) -target_link_libraries(SampleAsyncProducer ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleConsumer ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleConsumerListener ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleProducer ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleFileLogger ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleProducerCApi ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleConsumerCApi ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleAsyncConsumerCApi ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleConsumerListenerCApi ${CLIENT_LIBS} pulsarShared) -target_link_libraries(SampleReaderCApi ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleAsyncProducer ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleConsumer ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleConsumerListener ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleProducer ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleFileLogger ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleProducerCApi ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleConsumerCApi ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleAsyncConsumerCApi ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleConsumerListenerCApi ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleReaderCApi ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleKeyValueSchemaConsumer ${CLIENT_LIBS} pulsarShared) +target_link_libraries(SampleKeyValueSchemaProducer ${CLIENT_LIBS} pulsarShared) diff --git a/examples/SampleKeyValueSchemaConsumer.cc b/examples/SampleKeyValueSchemaConsumer.cc new file mode 100644 index 00000000..a61ddc17 --- /dev/null +++ b/examples/SampleKeyValueSchemaConsumer.cc @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include + +DECLARE_LOG_OBJECT() + +using namespace pulsar; + +int main() { + Client client("pulsar://localhost:6650"); + + std::string jsonSchema = + R"({"type":"record","name":"cpx","fields":[{"name":"re","type":"double"},{"name":"im","type":"double"}]})"; + + SchemaInfo keySchema(JSON, "key-json", jsonSchema); + SchemaInfo valueSchema(JSON, "value-json", jsonSchema); + SchemaInfo keyValueSchema(keySchema, valueSchema, KeyValueEncodingType::INLINE); + ConsumerConfiguration consumerConfiguration; + consumerConfiguration.setSchema(keyValueSchema); + + Consumer consumer; + Result result = client.subscribe("persistent://public/default/kv-schema", "consumer-1", + consumerConfiguration, consumer); + if (result != ResultOk) { + LOG_ERROR("Failed to subscribe: " << result); + return -1; + } + + LOG_INFO("Start receive message.") + + Message msg; + while (true) { + consumer.receive(msg); + LOG_INFO("Received: " << msg << " with payload '" << msg.getDataAsString() << "'"); + LOG_INFO("Received: " << msg << " with partitionKey '" << msg.getPartitionKey() << "'"); + KeyValue keyValue = msg.getKeyValueData(); + LOG_INFO("Received: " << msg << " with key '" << keyValue.getKey() << "'"); + LOG_INFO("Received: " << msg << " with value '" << keyValue.getValueAsString() << "'"); + consumer.acknowledge(msg); + } + + client.close(); +} diff --git a/examples/SampleKeyValueSchemaProducer.cc b/examples/SampleKeyValueSchemaProducer.cc new file mode 100644 index 00000000..0973236a --- /dev/null +++ b/examples/SampleKeyValueSchemaProducer.cc @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include +#include + +DECLARE_LOG_OBJECT() + +using namespace pulsar; + +int main() { + Client client("pulsar://localhost:6650"); + + std::string jsonSchema = + R"({"type":"record","name":"cpx","fields":[{"name":"re","type":"double"},{"name":"im","type":"double"}]})"; + + SchemaInfo keySchema(JSON, "key-json", jsonSchema); + SchemaInfo valueSchema(JSON, "value-json", jsonSchema); + SchemaInfo keyValueSchema(keySchema, valueSchema, KeyValueEncodingType::INLINE); + LOG_INFO("KeyValue schema content: " << keyValueSchema.getSchema()); + + ProducerConfiguration producerConfiguration; + producerConfiguration.setSchema(keyValueSchema); + + Producer producer; + Result result = + client.createProducer("persistent://public/default/kv-schema", producerConfiguration, producer); + if (result != ResultOk) { + LOG_ERROR("Error creating producer: " << result); + return -1; + } + + std::string jsonData = "{\"re\":2.1,\"im\":1.23}"; + + KeyValue keyValue(std::move(jsonData), std::move(jsonData)); + + Message msg = MessageBuilder().setContent(keyValue).setProperty("x", "1").build(); + result = producer.send(msg); + if (result == ResultOk) { + LOG_INFO("send message ok"); + } + client.close(); +} diff --git a/include/pulsar/KeyValue.h b/include/pulsar/KeyValue.h new file mode 100644 index 00000000..8189b02e --- /dev/null +++ b/include/pulsar/KeyValue.h @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef KEY_VALUE_HPP_ +#define KEY_VALUE_HPP_ + +#include +#include +#include "defines.h" +#include "Schema.h" + +namespace pulsar { + +class KeyValueImpl; + +/** + * Use to when the user uses key value schema. + */ +class PULSAR_PUBLIC KeyValue { + public: + /** + * Constructor key value, according to keyValueEncodingType, whether key and value be encoded together. + * + * @param key key data. + * @param value value data. + * @param keyValueEncodingType key value encoding type. + */ + KeyValue(std::string &&key, std::string &&value); + + /** + * Get the key of KeyValue. + * + * @return character stream for key + */ + std::string getKey() const; + + /** + * Get the value of the KeyValue. + * + * + * @return the pointer to the KeyValue value + */ + const void *getValue() const; + + /** + * Get the value length of the keyValue. + * + * @return the length of the KeyValue value + */ + size_t getValueLength() const; + + /** + * Get string representation of the KeyValue value. + * + * @return the string representation of the KeyValue value + */ + std::string getValueAsString() const; + + private: + typedef std::shared_ptr KeyValueImplPtr; + KeyValue(KeyValueImplPtr keyValueImplPtr); + KeyValueImplPtr impl_; + friend class Message; + friend class MessageBuilder; +}; +} // namespace pulsar + +#endif /* KEY_VALUE_HPP_ */ diff --git a/include/pulsar/Message.h b/include/pulsar/Message.h index 0c4afc28..69c1f121 100644 --- a/include/pulsar/Message.h +++ b/include/pulsar/Message.h @@ -26,6 +26,7 @@ #include #include "MessageId.h" +#include "KeyValue.h" namespace pulsar { namespace proto { @@ -92,6 +93,13 @@ class PULSAR_PUBLIC Message { */ std::string getDataAsString() const; + /** + * Get key value message. + * + * @return key value message. + */ + KeyValue getKeyValueData() const; + /** * Get the unique message ID associated with this message. * diff --git a/include/pulsar/MessageBuilder.h b/include/pulsar/MessageBuilder.h index 2b84d208..a668dd4f 100644 --- a/include/pulsar/MessageBuilder.h +++ b/include/pulsar/MessageBuilder.h @@ -19,6 +19,7 @@ #ifndef MESSAGE_BUILDER_H #define MESSAGE_BUILDER_H +#include #include #include @@ -60,6 +61,13 @@ class PULSAR_PUBLIC MessageBuilder { */ MessageBuilder& setContent(std::string&& data); + /** + * Set the key value content of the message + * + * @param data the content of the key value. + */ + MessageBuilder& setContent(const KeyValue& data); + /** * Set content of the message to a buffer already allocated by the caller. No copies of * this buffer will be made. The caller is responsible to ensure the memory buffer is diff --git a/include/pulsar/Schema.h b/include/pulsar/Schema.h index ec0802e9..4094e667 100644 --- a/include/pulsar/Schema.h +++ b/include/pulsar/Schema.h @@ -27,6 +27,27 @@ namespace pulsar { +/** + * Encoding types of supported KeyValueSchema for Pulsar messages. + */ +enum class KeyValueEncodingType +{ + /** + * Key is stored as message key, while value is stored as message payload. + */ + SEPARATED, + + /** + * Key and value are stored as message payload. + */ + INLINE +}; + +// Return string representation of result code +PULSAR_PUBLIC const char *strEncodingType(pulsar::KeyValueEncodingType encodingType); + +PULSAR_PUBLIC const KeyValueEncodingType enumEncodingType(std::string encodingTypeStr); + enum SchemaType { /** @@ -143,6 +164,14 @@ class PULSAR_PUBLIC SchemaInfo { SchemaInfo(SchemaType schemaType, const std::string &name, const std::string &schema, const StringMap &properties = StringMap()); + /** + * @param keySchema the key schema. + * @param valueSchema the value schema. + * @param keyValueEncodingType Encoding types of supported KeyValueSchema for Pulsar messages. + */ + SchemaInfo(const SchemaInfo &keySchema, const SchemaInfo &valueSchema, + const KeyValueEncodingType &keyValueEncodingType = KeyValueEncodingType::INLINE); + /** * @return the schema type */ @@ -171,3 +200,5 @@ class PULSAR_PUBLIC SchemaInfo { } // namespace pulsar PULSAR_PUBLIC std::ostream &operator<<(std::ostream &s, pulsar::SchemaType schemaType); + +PULSAR_PUBLIC std::ostream &operator<<(std::ostream &s, pulsar::KeyValueEncodingType encodingType); diff --git a/lib/Commands.cc b/lib/Commands.cc index 13febd09..69492c6f 100644 --- a/lib/Commands.cc +++ b/lib/Commands.cc @@ -69,6 +69,7 @@ static inline bool isBuiltInSchema(SchemaType schemaType) { case AVRO: case PROTOBUF: case PROTOBUF_NATIVE: + case KEY_VALUE: return true; default: @@ -90,6 +91,8 @@ static inline proto::Schema_Type getSchemaType(SchemaType type) { return proto::Schema_Type_Avro; case PROTOBUF_NATIVE: return proto::Schema_Type_ProtobufNative; + case KEY_VALUE: + return proto::Schema_Type_KeyValue; default: return proto::Schema_Type_None; } diff --git a/lib/ConsumerImpl.cc b/lib/ConsumerImpl.cc index 155c5bf5..19d5055c 100644 --- a/lib/ConsumerImpl.cc +++ b/lib/ConsumerImpl.cc @@ -458,6 +458,9 @@ void ConsumerImpl::messageReceived(const ClientConnectionPtr& cnx, const proto:: Lock lock(mutex_); numOfMessageReceived = receiveIndividualMessagesFromBatch(cnx, m, msg.redelivery_count()); } else { + // try convery key value data. + m.impl_->convertPayloadToKeyValue(config_.getSchema()); + const auto startMessageId = startMessageId_.get(); if (isPersistent_ && startMessageId.is_present() && m.getMessageId().ledgerId() == startMessageId.value().ledgerId() && @@ -582,6 +585,7 @@ uint32_t ConsumerImpl::receiveIndividualMessagesFromBatch(const ClientConnection Message msg = Commands::deSerializeSingleMessageInBatch(batchedMessage, i); msg.impl_->setRedeliveryCount(redeliveryCount); msg.impl_->setTopicName(batchedMessage.getTopicName()); + msg.impl_->convertPayloadToKeyValue(config_.getSchema()); if (startMessageId.is_present()) { const MessageId& msgId = msg.getMessageId(); diff --git a/lib/KeyValue.cc b/lib/KeyValue.cc new file mode 100644 index 00000000..f9fd3051 --- /dev/null +++ b/lib/KeyValue.cc @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include "KeyValueImpl.h" + +namespace pulsar { + +KeyValue::KeyValue(KeyValueImplPtr impl) : impl_(impl) {} + +KeyValue::KeyValue(std::string &&key, std::string &&value) + : impl_(std::make_shared(std::move(key), std::move(value))) {} + +std::string KeyValue::getKey() const { return impl_->getKey(); } + +const void *KeyValue::getValue() const { return impl_->getValue(); } + +size_t KeyValue::getValueLength() const { return impl_->getValueLength(); } + +std::string KeyValue::getValueAsString() const { return impl_->getValueAsString(); } + +} // namespace pulsar diff --git a/lib/KeyValueImpl.cc b/lib/KeyValueImpl.cc new file mode 100644 index 00000000..621fd12b --- /dev/null +++ b/lib/KeyValueImpl.cc @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include "SharedBuffer.h" +#include "KeyValueImpl.h" + +using namespace pulsar; + +namespace pulsar { + +KeyValueImpl::KeyValueImpl(const char *data, int length, KeyValueEncodingType keyValueEncodingType) { + if (keyValueEncodingType == KeyValueEncodingType::INLINE) { + SharedBuffer buffer = SharedBuffer::wrap(const_cast(data), length); + auto keySize = buffer.readUnsignedInt(); + if (keySize != INVALID_SIZE) { + SharedBuffer keyContent = buffer.slice(0, keySize); + key_ = std::string(keyContent.data(), keySize); + buffer.consume(keySize); + } + auto valueSize = buffer.readUnsignedInt(); + if (valueSize != INVALID_SIZE) { + valueBuffer_ = buffer.slice(0, valueSize); + } + } else { + valueBuffer_ = SharedBuffer::wrap(const_cast(data), length); + } +} + +KeyValueImpl::KeyValueImpl(std::string &&key, std::string &&value) + : key_(std::move(key)), valueBuffer_(SharedBuffer::take(std::move(value))) {} + +SharedBuffer KeyValueImpl::getContent(KeyValueEncodingType keyValueEncodingType) { + if (keyValueEncodingType == KeyValueEncodingType::INLINE) { + auto keySize = key_.length(); + auto valueSize = valueBuffer_.readableBytes(); + auto buffSize = sizeof(keySize) + keySize + sizeof(valueSize) + valueSize; + SharedBuffer buffer = SharedBuffer::allocate(buffSize); + buffer.writeUnsignedInt(keySize == 0 ? INVALID_SIZE : keySize); + buffer.write(key_.c_str(), keySize); + + buffer.writeUnsignedInt(valueSize == 0 ? INVALID_SIZE : valueSize); + buffer.write(valueBuffer_.data(), valueSize); + + return buffer; + } else { + return SharedBuffer::copyFrom(valueBuffer_, valueBuffer_.readableBytes()); + } +} + +std::string KeyValueImpl::getKey() const { return key_; } + +const void *KeyValueImpl::getValue() const { return valueBuffer_.data(); } + +size_t KeyValueImpl::getValueLength() const { return valueBuffer_.readableBytes(); } + +std::string KeyValueImpl::getValueAsString() const { + return std::string(valueBuffer_.data(), valueBuffer_.readableBytes()); +} + +} // namespace pulsar diff --git a/lib/KeyValueImpl.h b/lib/KeyValueImpl.h new file mode 100644 index 00000000..6740e1d2 --- /dev/null +++ b/lib/KeyValueImpl.h @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef LIB_KEY_VALUEIMPL_H_ +#define LIB_KEY_VALUEIMPL_H_ + +#include +#include "SharedBuffer.h" +#include "Utils.h" + +using namespace pulsar; + +namespace pulsar { + +class PULSAR_PUBLIC KeyValueImpl { + public: + KeyValueImpl(); + KeyValueImpl(const char* data, int length, KeyValueEncodingType keyValueEncodingType); + KeyValueImpl(std::string&& key, std::string&& value); + std::string getKey() const; + const void* getValue() const; + size_t getValueLength() const; + std::string getValueAsString() const; + SharedBuffer getContent(KeyValueEncodingType keyValueEncodingType); + + private: + std::string key_; + SharedBuffer valueBuffer_; + static constexpr uint32_t INVALID_SIZE = 0xFFFFFFFF; +}; + +} /* namespace pulsar */ + +#endif /* LIB_COMMANDS_H_ */ diff --git a/lib/Message.cc b/lib/Message.cc index cb7a75e3..119ef3cf 100644 --- a/lib/Message.cc +++ b/lib/Message.cc @@ -21,7 +21,7 @@ #include #include - +#include "KeyValueImpl.h" #include "MessageImpl.h" #include "PulsarApi.pb.h" #include "SharedBuffer.h" @@ -190,6 +190,8 @@ uint64_t Message::getEventTimestamp() const { return impl_ ? impl_->getEventTime bool Message::operator==(const Message& msg) const { return getMessageId() == msg.getMessageId(); } +KeyValue Message::getKeyValueData() const { return KeyValue(impl_->keyValuePtr); } + PULSAR_PUBLIC std::ostream& operator<<(std::ostream& s, const Message::StringMap& map) { // Output at most 10 elements -- appropriate if used for logging. s << '{'; diff --git a/lib/MessageBuilder.cc b/lib/MessageBuilder.cc index 7d8d8cb7..b33394ed 100644 --- a/lib/MessageBuilder.cc +++ b/lib/MessageBuilder.cc @@ -20,9 +20,9 @@ #include #include -#include #include +#include "KeyValueImpl.h" #include "LogUtils.h" #include "MessageImpl.h" #include "ObjectPool.h" @@ -80,6 +80,11 @@ MessageBuilder& MessageBuilder::setContent(std::string&& data) { return *this; } +MessageBuilder& MessageBuilder::setContent(const KeyValue& data) { + impl_->keyValuePtr = data.impl_; + return *this; +} + MessageBuilder& MessageBuilder::setProperty(const std::string& name, const std::string& value) { checkMetadata(); proto::KeyValue* keyValue = proto::KeyValue().New(); diff --git a/lib/MessageImpl.cc b/lib/MessageImpl.cc index 5d1edbfe..63232e70 100644 --- a/lib/MessageImpl.cc +++ b/lib/MessageImpl.cc @@ -102,4 +102,39 @@ void MessageImpl::setSchemaVersion(const std::string& schemaVersion) { schemaVer const std::string& MessageImpl::getSchemaVersion() const { return metadata.schema_version(); } +void MessageImpl::convertKeyValueToPayload(const pulsar::SchemaInfo& schemaInfo) { + if (schemaInfo.getSchemaType() != KEY_VALUE) { + // ignore not key_value schema. + return; + } + KeyValueEncodingType keyValueEncodingType = getKeyValueEncodingType(schemaInfo); + payload = keyValuePtr->getContent(keyValueEncodingType); + if (keyValueEncodingType == KeyValueEncodingType::SEPARATED) { + setPartitionKey(keyValuePtr->getKey()); + } +} + +void MessageImpl::convertPayloadToKeyValue(const pulsar::SchemaInfo& schemaInfo) { + if (schemaInfo.getSchemaType() != KEY_VALUE) { + // ignore not key_value schema. + return; + } + keyValuePtr = + std::make_shared(static_cast(payload.data()), payload.readableBytes(), + getKeyValueEncodingType(schemaInfo)); +} + +KeyValueEncodingType MessageImpl::getKeyValueEncodingType(SchemaInfo schemaInfo) { + if (schemaInfo.getSchemaType() != KEY_VALUE) { + throw std::invalid_argument("Schema not key value type."); + } + const StringMap& properties = schemaInfo.getProperties(); + auto data = properties.find("kv.encoding.type"); + if (data == properties.end()) { + throw std::invalid_argument("Not found kv.encoding.type by properties"); + } else { + return enumEncodingType(data->second); + } +} + } // namespace pulsar diff --git a/lib/MessageImpl.h b/lib/MessageImpl.h index 587b6638..790a0211 100644 --- a/lib/MessageImpl.h +++ b/lib/MessageImpl.h @@ -22,6 +22,7 @@ #include #include +#include "KeyValueImpl.h" #include "PulsarApi.pb.h" #include "SharedBuffer.h" @@ -40,6 +41,7 @@ class MessageImpl { proto::MessageMetadata metadata; SharedBuffer payload; + std::shared_ptr keyValuePtr; MessageId messageId; ClientConnection* cnx_; const std::string* topicName_; @@ -72,6 +74,9 @@ class MessageImpl { bool hasSchemaVersion() const; const std::string& getSchemaVersion() const; void setSchemaVersion(const std::string& value); + void convertKeyValueToPayload(const SchemaInfo& schemaInfo); + void convertPayloadToKeyValue(const SchemaInfo& schemaInfo); + KeyValueEncodingType getKeyValueEncodingType(SchemaInfo schemaInfo); friend class PulsarWrapper; friend class MessageBuilder; diff --git a/lib/ProducerImpl.cc b/lib/ProducerImpl.cc index 1213fcef..05ab13d3 100644 --- a/lib/ProducerImpl.cc +++ b/lib/ProducerImpl.cc @@ -406,6 +406,8 @@ void ProducerImpl::sendAsyncWithStatsUpdate(const Message& msg, const SendCallba return; } + // Convert the payload before sending the message. + msg.impl_->convertKeyValueToPayload(conf_.getSchema()); const auto& uncompressedPayload = msg.impl_->payload; const uint32_t uncompressedSize = uncompressedPayload.readableBytes(); const auto result = canEnqueueRequest(uncompressedSize); diff --git a/lib/Schema.cc b/lib/Schema.cc index 17a301e6..2fb3dbf0 100644 --- a/lib/Schema.cc +++ b/lib/Schema.cc @@ -22,13 +22,54 @@ #include #include #include +#include +#include +#include "SharedBuffer.h" +using boost::property_tree::ptree; +using boost::property_tree::read_json; +using boost::property_tree::write_json; PULSAR_PUBLIC std::ostream &operator<<(std::ostream &s, pulsar::SchemaType schemaType) { return s << strSchemaType(schemaType); } +PULSAR_PUBLIC std::ostream &operator<<(std::ostream &s, pulsar::KeyValueEncodingType encodingType) { + return s << strEncodingType(encodingType); +} + namespace pulsar { +static const std::string KEY_SCHEMA_NAME = "key.schema.name"; +static const std::string KEY_SCHEMA_TYPE = "key.schema.type"; +static const std::string KEY_SCHEMA_PROPS = "key.schema.properties"; +static const std::string VALUE_SCHEMA_NAME = "value.schema.name"; +static const std::string VALUE_SCHEMA_TYPE = "value.schema.type"; +static const std::string VALUE_SCHEMA_PROPS = "value.schema.properties"; +static const std::string KV_ENCODING_TYPE = "kv.encoding.type"; + +PULSAR_PUBLIC const char *strEncodingType(KeyValueEncodingType encodingType) { + switch (encodingType) { + case KeyValueEncodingType::INLINE: + return "INLINE"; + case KeyValueEncodingType::SEPARATED: + return "SEPARATED"; + }; + // NOTE : Do not add default case in the switch above. In future if we get new cases for + // Schema and miss them in the switch above we would like to get notified. Adding + // return here to make the compiler happy. + return "UnknownSchemaType"; +} + +PULSAR_PUBLIC const KeyValueEncodingType enumEncodingType(std::string encodingTypeStr) { + if (encodingTypeStr == "INLINE") { + return KeyValueEncodingType::INLINE; + } else if (encodingTypeStr == "SEPARATED") { + return KeyValueEncodingType::SEPARATED; + } else { + throw std::invalid_argument("No match encoding type: " + encodingTypeStr); + } +} + PULSAR_PUBLIC const char *strSchemaType(SchemaType schemaType) { switch (schemaType) { case NONE: @@ -90,6 +131,45 @@ SchemaInfo::SchemaInfo(SchemaType schemaType, const std::string &name, const std const StringMap &properties) : impl_(std::make_shared(schemaType, name, schema, properties)) {} +SchemaInfo::SchemaInfo(const SchemaInfo &keySchema, const SchemaInfo &valueSchema, + const KeyValueEncodingType &keyValueEncodingType) { + std::string keySchemaStr = keySchema.getSchema(); + std::string valueSchemaStr = valueSchema.getSchema(); + uint32_t keySize = keySchemaStr.size(); + uint32_t valueSize = valueSchemaStr.size(); + + auto buffSize = sizeof keySize + keySize + sizeof valueSize + valueSize; + SharedBuffer buffer = SharedBuffer::allocate(buffSize); + buffer.writeUnsignedInt(keySize == 0 ? -1 : static_cast(keySize)); + buffer.write(keySchemaStr.c_str(), static_cast(keySize)); + buffer.writeUnsignedInt(valueSize == 0 ? -1 : static_cast(valueSize)); + buffer.write(valueSchemaStr.c_str(), static_cast(valueSize)); + + auto writeJson = [](const StringMap &properties) { + ptree pt; + for (auto &entry : properties) { + pt.put(entry.first, entry.second); + } + std::ostringstream buf; + write_json(buf, pt, false); + auto s = buf.str(); + s.pop_back(); + return s; + }; + + StringMap properties; + properties.emplace(KEY_SCHEMA_NAME, keySchema.getName()); + properties.emplace(KEY_SCHEMA_TYPE, strSchemaType(keySchema.getSchemaType())); + properties.emplace(KEY_SCHEMA_PROPS, writeJson(keySchema.getProperties())); + properties.emplace(VALUE_SCHEMA_NAME, valueSchema.getName()); + properties.emplace(VALUE_SCHEMA_TYPE, strSchemaType(valueSchema.getSchemaType())); + properties.emplace(VALUE_SCHEMA_PROPS, writeJson(valueSchema.getProperties())); + properties.emplace(KV_ENCODING_TYPE, strEncodingType(keyValueEncodingType)); + + impl_ = std::make_shared(KEY_VALUE, "KeyValue", std::string(buffer.data(), buffSize), + properties); +} + SchemaType SchemaInfo::getSchemaType() const { return impl_->type_; } const std::string &SchemaInfo::getName() const { return impl_->name_; } diff --git a/tests/KeyValueImplTest.cc b/tests/KeyValueImplTest.cc new file mode 100644 index 00000000..01f246ce --- /dev/null +++ b/tests/KeyValueImplTest.cc @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include + +using namespace pulsar; + +TEST(KeyValueTest, testEncodeAndDeCode) { + const char* keyContent = "keyContent"; + const char* valueContent = "valueContent"; + + { + // test inline encode + KeyValueImpl keyValue(keyContent, valueContent); + const SharedBuffer content = keyValue.getContent(KeyValueEncodingType::INLINE); + ASSERT_EQ(content.readableBytes(), 8 + strlen(keyContent) + strlen(valueContent)); + + // test inline decode + KeyValueImpl deCodeKeyValue(content.data(), content.readableBytes(), KeyValueEncodingType::INLINE); + const SharedBuffer deCodeContent = deCodeKeyValue.getContent(KeyValueEncodingType::INLINE); + ASSERT_EQ(deCodeKeyValue.getKey(), keyContent); + ASSERT_EQ(deCodeKeyValue.getValueAsString(), valueContent); + ASSERT_TRUE(std::string(deCodeContent.data(), deCodeContent.readableBytes()).compare(valueContent) != + 0); + } + + { + // test separated encode + KeyValueImpl sepKeyValue(keyContent, valueContent); + const SharedBuffer content = sepKeyValue.getContent(KeyValueEncodingType::SEPARATED); + ASSERT_EQ(sepKeyValue.getKey(), keyContent); + ASSERT_EQ(sepKeyValue.getValueAsString(), valueContent); + ASSERT_EQ(std::string(content.data(), content.readableBytes()), valueContent); + + // test separated decode + KeyValueImpl sepDeKeyValue(content.data(), content.readableBytes(), KeyValueEncodingType::SEPARATED); + const SharedBuffer deCodeContent = sepKeyValue.getContent(KeyValueEncodingType::SEPARATED); + ASSERT_EQ(sepDeKeyValue.getKey(), ""); + ASSERT_EQ(sepDeKeyValue.getValueAsString(), valueContent); + ASSERT_EQ(std::string(deCodeContent.data(), deCodeContent.readableBytes()), valueContent); + } +} + +TEST(KeyValueTest, testKeyIsEmpty) { + const char* keyContent = ""; + const char* valueContent = "valueContent"; + + { + // test inline encode + KeyValueImpl keyValue(keyContent, valueContent); + const SharedBuffer content = keyValue.getContent(KeyValueEncodingType::INLINE); + ASSERT_EQ(content.readableBytes(), 8 + strlen(keyContent) + strlen(valueContent)); + + // test inline decode + KeyValueImpl deCodeKeyValue(content.data(), content.readableBytes(), KeyValueEncodingType::INLINE); + const SharedBuffer deCodeContent = deCodeKeyValue.getContent(KeyValueEncodingType::INLINE); + ASSERT_EQ(deCodeKeyValue.getKey(), keyContent); + ASSERT_EQ(deCodeKeyValue.getValueAsString(), valueContent); + ASSERT_TRUE(std::string(deCodeContent.data(), deCodeContent.readableBytes()).compare(valueContent) != + 0); + } + + { + // test separated type + KeyValueImpl sepKeyValue(keyContent, valueContent); + const SharedBuffer content = sepKeyValue.getContent(KeyValueEncodingType::SEPARATED); + ASSERT_EQ(sepKeyValue.getKey(), keyContent); + ASSERT_EQ(sepKeyValue.getValueAsString(), valueContent); + ASSERT_EQ(std::string(content.data(), content.readableBytes()), valueContent); + } +} + +TEST(KeyValueTest, testValueIsEmpty) { + const char* keyContent = "keyContent"; + const char* valueContent = ""; + + { + // test inline encode + KeyValueImpl keyValue(keyContent, valueContent); + const SharedBuffer content = keyValue.getContent(KeyValueEncodingType::INLINE); + ASSERT_EQ(content.readableBytes(), 8 + strlen(keyContent) + strlen(valueContent)); + + // test inline decode + KeyValueImpl deCodeKeyValue(content.data(), content.readableBytes(), KeyValueEncodingType::INLINE); + const SharedBuffer deCodeContent = keyValue.getContent(KeyValueEncodingType::INLINE); + ASSERT_EQ(deCodeKeyValue.getKey(), keyContent); + ASSERT_EQ(deCodeKeyValue.getValueAsString(), valueContent); + ASSERT_NE(std::string(deCodeContent.data(), deCodeContent.readableBytes()), valueContent); + } + + { + // test separated type + KeyValueImpl sepKeyValue(keyContent, valueContent); + const SharedBuffer content = sepKeyValue.getContent(KeyValueEncodingType::SEPARATED); + ASSERT_EQ(sepKeyValue.getKey(), keyContent); + ASSERT_EQ(sepKeyValue.getValueAsString(), valueContent); + ASSERT_EQ(std::string(content.data(), content.readableBytes()), valueContent); + } +} diff --git a/tests/KeyValueSchemaTest.cc b/tests/KeyValueSchemaTest.cc new file mode 100644 index 00000000..26ec324e --- /dev/null +++ b/tests/KeyValueSchemaTest.cc @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include "lib/LogUtils.h" + +using namespace pulsar; + +static const std::string lookupUrl = "pulsar://localhost:6650"; + +class KeyValueSchemaTest : public ::testing::TestWithParam { + public: + void TearDown() override { client.close(); } + + void createProducer(const std::string& topic, Producer& producer) { + ProducerConfiguration configProducer; + configProducer.setSchema(getKeyValueSchema()); + configProducer.setBatchingEnabled(false); + ASSERT_EQ(ResultOk, client.createProducer(topic, configProducer, producer)); + } + + void createConsumer(const std::string& topic, Consumer& consumer) { + ConsumerConfiguration configConsumer; + configConsumer.setSchema(getKeyValueSchema()); + ASSERT_EQ(ResultOk, client.subscribe(topic, "sub-kv", configConsumer, consumer)); + } + + SchemaInfo getKeyValueSchema() { + SchemaInfo keySchema(JSON, "key-json", jsonSchema); + SchemaInfo valueSchema(JSON, "value-json", jsonSchema); + return SchemaInfo(keySchema, valueSchema, GetParam()); + } + + private: + Client client{lookupUrl}; + std::string jsonSchema = + R"({"type":"record","name":"cpx","fields":[{"name":"re","type":"double"},{"name":"im","type":"double"}]})"; +}; + +TEST_P(KeyValueSchemaTest, testKeyValueSchema) { + const std::string topicName = "testKeyValueSchema" + std::to_string(time(nullptr)); + + Producer producer; + createProducer(topicName, producer); + Consumer consumer; + createConsumer(topicName, consumer); + + // Sending and receiving messages. + std::string keyData = "{\"re\":2.1,\"im\":1.23}"; + std::string valueData = "{\"re\":2.1,\"im\":1.23}"; + KeyValue keyValue((std::string(keyData)), std::string(valueData)); + Message msg = MessageBuilder().setContent(keyValue).setProperty("x", "1").build(); + ASSERT_EQ(ResultOk, producer.send(msg)); + + Message receiveMsg; + consumer.receive(receiveMsg); + KeyValue keyValueData = receiveMsg.getKeyValueData(); + + auto encodingType = GetParam(); + if (encodingType == pulsar::KeyValueEncodingType::INLINE) { + ASSERT_EQ(receiveMsg.getPartitionKey(), ""); + ASSERT_EQ(keyValueData.getKey(), keyData); + } else { + ASSERT_EQ(receiveMsg.getPartitionKey(), keyData); + ASSERT_EQ(keyValueData.getKey(), ""); + } + ASSERT_EQ(keyValueData.getValueAsString(), valueData); +} + +INSTANTIATE_TEST_CASE_P(Pulsar, KeyValueSchemaTest, + ::testing::Values(KeyValueEncodingType::INLINE, KeyValueEncodingType::SEPARATED)); diff --git a/tests/MessageTest.cc b/tests/MessageTest.cc index 7e26431e..c11428b1 100644 --- a/tests/MessageTest.cc +++ b/tests/MessageTest.cc @@ -21,8 +21,7 @@ #include #include - -#include "lib/LogUtils.h" +#include "MessageImpl.h" using namespace pulsar; TEST(MessageTest, testMessageContents) { @@ -101,3 +100,49 @@ TEST(MessageTest, testMessageBuilder) { ASSERT_EQ(msg.getData(), originalAddress); } } + +TEST(MessageTest, testMessageImplKeyValuePayloadCovert) { + const char* keyContent = "keyContent"; + const char* valueContent = "valueContent"; + + std::string jsonSchema = + R"({"type":"record","name":"cpx","fields":[{"name":"re","type":"double"},{"name":"im","type":"double"}]})"; + SchemaInfo keySchema(JSON, "key-json", jsonSchema); + SchemaInfo valueSchema(JSON, "value-json", jsonSchema); + + // test inline encoding type. + { + SchemaInfo keyValueSchema(keySchema, valueSchema, KeyValueEncodingType::INLINE); + MessageImpl msgImpl; + std::shared_ptr keyValuePtr = std::make_shared(keyContent, valueContent); + msgImpl.keyValuePtr = keyValuePtr; + msgImpl.convertKeyValueToPayload(keyValueSchema); + ASSERT_EQ(msgImpl.payload.readableBytes(), 30); + ASSERT_EQ(msgImpl.getPartitionKey(), ""); + + MessageImpl deMsgImpl; + deMsgImpl.payload = msgImpl.payload; + deMsgImpl.convertPayloadToKeyValue(keyValueSchema); + + ASSERT_EQ(deMsgImpl.keyValuePtr->getKey(), keyContent); + ASSERT_EQ(deMsgImpl.keyValuePtr->getValueAsString(), valueContent); + } + + // test separated encoding type. + { + SchemaInfo keyValueSchema(keySchema, valueSchema, KeyValueEncodingType::SEPARATED); + MessageImpl msgImpl; + std::shared_ptr keyValuePtr = std::make_shared(keyContent, valueContent); + msgImpl.keyValuePtr = keyValuePtr; + msgImpl.convertKeyValueToPayload(keyValueSchema); + ASSERT_EQ(msgImpl.payload.readableBytes(), 12); + ASSERT_EQ(msgImpl.getPartitionKey(), keyContent); + + MessageImpl deMsgImpl; + deMsgImpl.payload = msgImpl.payload; + deMsgImpl.convertPayloadToKeyValue(keyValueSchema); + + ASSERT_EQ(deMsgImpl.keyValuePtr->getKey(), ""); + ASSERT_EQ(deMsgImpl.keyValuePtr->getValueAsString(), valueContent); + } +} diff --git a/tests/SchemaTest.cc b/tests/SchemaTest.cc index f1536521..265304a3 100644 --- a/tests/SchemaTest.cc +++ b/tests/SchemaTest.cc @@ -18,14 +18,14 @@ */ #include #include +#include "SharedBuffer.h" using namespace pulsar; static std::string lookupUrl = "pulsar://localhost:6650"; static const std::string exampleSchema = - "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\"," - "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}"; + R"({"type":"record","name":"Example","namespace":"test","fields":[{"name":"a","type":"int"},{"name":"b","type":"int"}]})"; TEST(SchemaTest, testSchema) { ClientConfiguration config; @@ -107,3 +107,49 @@ TEST(SchemaTest, testHasSchemaVersion) { client.close(); } + +TEST(SchemaTest, testKeyValueSchema) { + SchemaInfo keySchema(SchemaType::AVRO, "String", exampleSchema); + SchemaInfo valueSchema(SchemaType::AVRO, "String", exampleSchema); + SchemaInfo keyValueSchema(keySchema, valueSchema, KeyValueEncodingType::INLINE); + ASSERT_EQ(keyValueSchema.getSchemaType(), KEY_VALUE); + ASSERT_EQ(keyValueSchema.getSchema().size(), + 8 + keySchema.getSchema().size() + valueSchema.getSchema().size()); +} + +TEST(SchemaTest, testKeySchemaIsEmpty) { + SchemaInfo keySchema(SchemaType::AVRO, "String", ""); + SchemaInfo valueSchema(SchemaType::AVRO, "String", exampleSchema); + SchemaInfo keyValueSchema(keySchema, valueSchema, KeyValueEncodingType::INLINE); + ASSERT_EQ(keyValueSchema.getSchemaType(), KEY_VALUE); + ASSERT_EQ(keyValueSchema.getSchema().size(), + 8 + keySchema.getSchema().size() + valueSchema.getSchema().size()); + + SharedBuffer buffer = SharedBuffer::wrap(const_cast(keyValueSchema.getSchema().c_str()), + keyValueSchema.getSchema().size()); + int keySchemaSize = buffer.readUnsignedInt(); + ASSERT_EQ(keySchemaSize, -1); + int valueSchemaSize = buffer.readUnsignedInt(); + ASSERT_EQ(valueSchemaSize, valueSchema.getSchema().size()); + std::string valueSchemaStr(buffer.slice(0, valueSchemaSize).data(), valueSchemaSize); + ASSERT_EQ(valueSchema.getSchema(), valueSchemaStr); +} + +TEST(SchemaTest, testValueSchemaIsEmpty) { + SchemaInfo keySchema(SchemaType::AVRO, "String", exampleSchema); + SchemaInfo valueSchema(SchemaType::AVRO, "String", ""); + SchemaInfo keyValueSchema(keySchema, valueSchema, KeyValueEncodingType::INLINE); + ASSERT_EQ(keyValueSchema.getSchemaType(), KEY_VALUE); + ASSERT_EQ(keyValueSchema.getSchema().size(), + 8 + keySchema.getSchema().size() + valueSchema.getSchema().size()); + + SharedBuffer buffer = SharedBuffer::wrap(const_cast(keyValueSchema.getSchema().c_str()), + keyValueSchema.getSchema().size()); + int keySchemaSize = buffer.readUnsignedInt(); + ASSERT_EQ(keySchemaSize, keySchema.getSchema().size()); + std::string keySchemaStr(buffer.slice(0, keySchemaSize).data(), keySchemaSize); + ASSERT_EQ(keySchemaStr, keySchema.getSchema()); + buffer.consume(keySchemaSize); + int valueSchemaSize = buffer.readUnsignedInt(); + ASSERT_EQ(valueSchemaSize, -1); +} From f32d4f02905c415d4b881375d5724f7b54643401 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Tue, 25 Oct 2022 09:59:07 +0800 Subject: [PATCH 19/19] Code format --- examples/SampleKeyValueSchemaConsumer.cc | 5 +++-- examples/SampleKeyValueSchemaProducer.cc | 5 +++-- include/pulsar/KeyValue.h | 5 +++-- include/pulsar/Message.h | 2 +- lib/KeyValue.cc | 1 + lib/KeyValueImpl.cc | 4 +++- lib/KeyValueImpl.h | 1 + lib/Message.cc | 1 + lib/Schema.cc | 5 +++-- tests/KeyValueImplTest.cc | 2 +- tests/KeyValueSchemaTest.cc | 1 + tests/MessageTest.cc | 1 + tests/SchemaTest.cc | 1 + 13 files changed, 23 insertions(+), 11 deletions(-) diff --git a/examples/SampleKeyValueSchemaConsumer.cc b/examples/SampleKeyValueSchemaConsumer.cc index a61ddc17..b5ff1b51 100644 --- a/examples/SampleKeyValueSchemaConsumer.cc +++ b/examples/SampleKeyValueSchemaConsumer.cc @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -#include -#include #include +#include + +#include DECLARE_LOG_OBJECT() diff --git a/examples/SampleKeyValueSchemaProducer.cc b/examples/SampleKeyValueSchemaProducer.cc index 0973236a..e52cb357 100644 --- a/examples/SampleKeyValueSchemaProducer.cc +++ b/examples/SampleKeyValueSchemaProducer.cc @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include + #include #include -#include -#include DECLARE_LOG_OBJECT() diff --git a/include/pulsar/KeyValue.h b/include/pulsar/KeyValue.h index 8189b02e..2ccb26c4 100644 --- a/include/pulsar/KeyValue.h +++ b/include/pulsar/KeyValue.h @@ -19,10 +19,11 @@ #ifndef KEY_VALUE_HPP_ #define KEY_VALUE_HPP_ -#include #include -#include "defines.h" +#include + #include "Schema.h" +#include "defines.h" namespace pulsar { diff --git a/include/pulsar/Message.h b/include/pulsar/Message.h index 69c1f121..74427a26 100644 --- a/include/pulsar/Message.h +++ b/include/pulsar/Message.h @@ -25,8 +25,8 @@ #include #include -#include "MessageId.h" #include "KeyValue.h" +#include "MessageId.h" namespace pulsar { namespace proto { diff --git a/lib/KeyValue.cc b/lib/KeyValue.cc index f9fd3051..e031f527 100644 --- a/lib/KeyValue.cc +++ b/lib/KeyValue.cc @@ -17,6 +17,7 @@ * under the License. */ #include + #include "KeyValueImpl.h" namespace pulsar { diff --git a/lib/KeyValueImpl.cc b/lib/KeyValueImpl.cc index 621fd12b..79018d00 100644 --- a/lib/KeyValueImpl.cc +++ b/lib/KeyValueImpl.cc @@ -16,9 +16,11 @@ * specific language governing permissions and limitations * under the License. */ +#include "KeyValueImpl.h" + #include + #include "SharedBuffer.h" -#include "KeyValueImpl.h" using namespace pulsar; diff --git a/lib/KeyValueImpl.h b/lib/KeyValueImpl.h index 6740e1d2..00ed33dc 100644 --- a/lib/KeyValueImpl.h +++ b/lib/KeyValueImpl.h @@ -20,6 +20,7 @@ #define LIB_KEY_VALUEIMPL_H_ #include + #include "SharedBuffer.h" #include "Utils.h" diff --git a/lib/Message.cc b/lib/Message.cc index 119ef3cf..84f203f7 100644 --- a/lib/Message.cc +++ b/lib/Message.cc @@ -21,6 +21,7 @@ #include #include + #include "KeyValueImpl.h" #include "MessageImpl.h" #include "PulsarApi.pb.h" diff --git a/lib/Schema.cc b/lib/Schema.cc index 2fb3dbf0..961a13f9 100644 --- a/lib/Schema.cc +++ b/lib/Schema.cc @@ -19,11 +19,12 @@ #include #include +#include +#include #include #include #include -#include -#include + #include "SharedBuffer.h" using boost::property_tree::ptree; using boost::property_tree::read_json; diff --git a/tests/KeyValueImplTest.cc b/tests/KeyValueImplTest.cc index 01f246ce..89770c41 100644 --- a/tests/KeyValueImplTest.cc +++ b/tests/KeyValueImplTest.cc @@ -16,8 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -#include #include +#include using namespace pulsar; diff --git a/tests/KeyValueSchemaTest.cc b/tests/KeyValueSchemaTest.cc index 26ec324e..28313c78 100644 --- a/tests/KeyValueSchemaTest.cc +++ b/tests/KeyValueSchemaTest.cc @@ -18,6 +18,7 @@ */ #include #include + #include "lib/LogUtils.h" using namespace pulsar; diff --git a/tests/MessageTest.cc b/tests/MessageTest.cc index c11428b1..3dfe2217 100644 --- a/tests/MessageTest.cc +++ b/tests/MessageTest.cc @@ -21,6 +21,7 @@ #include #include + #include "MessageImpl.h" using namespace pulsar; diff --git a/tests/SchemaTest.cc b/tests/SchemaTest.cc index 265304a3..cf0f8c7c 100644 --- a/tests/SchemaTest.cc +++ b/tests/SchemaTest.cc @@ -18,6 +18,7 @@ */ #include #include + #include "SharedBuffer.h" using namespace pulsar;