From 0c758fadb61645467b9a293179e667dd36fa5b5f Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 30 Jul 2025 07:02:16 +0530 Subject: [PATCH 01/49] Implemented Couchbase binary protocol support --- example/couchbase_c++/CMakeLists.txt | 130 +++ example/couchbase_c++/couchbase_client.cpp | 97 ++ src/brpc/couchbase.cpp | 1064 ++++++++++++++++++++ src/brpc/couchbase.h | 178 ++++ src/brpc/global.cpp | 11 + src/brpc/options.proto | 1 + src/brpc/policy/couchbase_protocol.cpp | 248 +++++ src/brpc/policy/couchbase_protocol.h | 166 +++ src/brpc/proto_base.proto | 3 + 9 files changed, 1898 insertions(+) create mode 100644 example/couchbase_c++/CMakeLists.txt create mode 100644 example/couchbase_c++/couchbase_client.cpp create mode 100644 src/brpc/couchbase.cpp create mode 100644 src/brpc/couchbase.h create mode 100644 src/brpc/policy/couchbase_protocol.cpp create mode 100644 src/brpc/policy/couchbase_protocol.h diff --git a/example/couchbase_c++/CMakeLists.txt b/example/couchbase_c++/CMakeLists.txt new file mode 100644 index 0000000000..50a67397ce --- /dev/null +++ b/example/couchbase_c++/CMakeLists.txt @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 2.8.10) +project(couchbase_c++ C CXX) + +option(LINK_SO "Whether examples are linked dynamically" OFF) + +execute_process( + COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'" + OUTPUT_VARIABLE OUTPUT_PATH +) + +set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) + +include(FindThreads) +include(FindProtobuf) +protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto) +# include PROTO_HEADER +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +# Search for libthrift* by best effort. If it is not found and brpc is +# compiled with thrift protocol enabled, a link error would be reported. +find_library(THRIFT_LIB NAMES thrift) +if (NOT THRIFT_LIB) + set(THRIFT_LIB "") +endif() + +find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) +if(LINK_SO) + find_library(BRPC_LIB NAMES brpc) +else() + find_library(BRPC_LIB NAMES libbrpc.a brpc) +endif() +if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) + message(FATAL_ERROR "Fail to find brpc") +endif() +include_directories(${BRPC_INCLUDE_PATH}) + +find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) +find_library(GFLAGS_LIBRARY NAMES gflags libgflags) +if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) + message(FATAL_ERROR "Fail to find gflags") +endif() +include_directories(${GFLAGS_INCLUDE_PATH}) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + include(CheckFunctionExists) + CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME) + if(NOT HAVE_CLOCK_GETTIME) + set(DEFINE_CLOCK_GETTIME "-DNO_CLOCK_GETTIME_IN_MAC") + endif() +endif() + +set(CMAKE_CXX_FLAGS "${DEFINE_CLOCK_GETTIME} -DNDEBUG -O2 -D__const__=__unused__ -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") + +if(CMAKE_VERSION VERSION_LESS "3.1.3") + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() +else() + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD_REQUIRED ON) +endif() + +find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) +find_library(LEVELDB_LIB NAMES leveldb) +if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) + message(FATAL_ERROR "Fail to find leveldb") +endif() +include_directories(${LEVELDB_INCLUDE_PATH}) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + set(OPENSSL_ROOT_DIR + "/usr/local/opt/openssl" # Homebrew installed OpenSSL + ) +endif() + +find_package(OpenSSL) +include_directories(${OPENSSL_INCLUDE_DIR}) + +set(DYNAMIC_LIB + ${CMAKE_THREAD_LIBS_INIT} + ${GFLAGS_LIBRARY} + ${PROTOBUF_LIBRARIES} + ${LEVELDB_LIB} + ${OPENSSL_CRYPTO_LIBRARY} + ${OPENSSL_SSL_LIBRARY} + ${THRIFT_LIB} + dl + ) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + set(DYNAMIC_LIB ${DYNAMIC_LIB} + pthread + "-framework CoreFoundation" + "-framework CoreGraphics" + "-framework CoreData" + "-framework CoreText" + "-framework Security" + "-framework Foundation" + "-Wl,-U,_MallocExtension_ReleaseFreeMemory" + "-Wl,-U,_ProfilerStart" + "-Wl,-U,_ProfilerStop" + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) +endif() + +add_executable(couchbase_client couchbase_client.cpp) + +target_link_libraries(couchbase_client ${BRPC_LIB} ${DYNAMIC_LIB}) diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp new file mode 100644 index 0000000000..a44ef4e53c --- /dev/null +++ b/example/couchbase_c++/couchbase_client.cpp @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include +#include +#include +#include +DEFINE_string(server, "127.0.0.1:11210", "IP Address of server"); +DEFINE_string(connection_type, "pooled", "Connection type. Available values: single, pooled, short"); +DEFINE_string(username, "Administrator", "Couchbase username"); +DEFINE_string(password, "password", "Couchbase password"); +DEFINE_string(bucket_name, "testing", "Couchbase bucket name"); +DEFINE_string(load_balancer, "", "The algorithm for load balancing"); +DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); +DEFINE_bool(dont_fail, false, "Print fatal when some call failed"); +DEFINE_int32(exptime, 0, "The to-be-got data will be expired after so many seconds"); +bvar::LatencyRecorder g_latency_recorder("client"); +bvar::Adder g_error_count("client_error_count"); +butil::static_atomic g_sender_count = BUTIL_STATIC_ATOMIC_INIT(0); +int main(){ + brpc::Channel channel; + + // Initialize the channel, NULL means using default options. + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_COUCHBASE; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = 2000 /*milliseconds*/; + options.max_retry = 2; + + if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; + } + std::cout<<"Channel initialized successfully"<() { + SharedCtor(); +} + +CouchbaseRequest::CouchbaseRequest(const CouchbaseRequest& from) + : NonreflectableMessage(from) { + SharedCtor(); + MergeFrom(from); +} + +void CouchbaseRequest::SharedCtor() { + _pipelined_count = 0; + _cached_size_ = 0; +} + +CouchbaseRequest::~CouchbaseRequest() { + SharedDtor(); +} + +void CouchbaseRequest::SharedDtor() { +} + +void CouchbaseRequest::SetCachedSize(int size) const { + _cached_size_ = size; +} + +void CouchbaseRequest::Clear() { + _buf.clear(); + _pipelined_count = 0; +} + +// Get the Scope ID for a given scope name +bool CouchbaseRequest::GetScopeId(const butil::StringPiece& scope_name) { + if (scope_name.empty()) { + LOG(ERROR) << "Empty scope name"; + return false; + } + // Opcode 0xBC for Get Scope ID (see Collections.md) + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_GET_SCOPE_ID, + butil::HostToNet16(scope_name.size()), + 0, // no extras + policy::CB_BINARY_RAW_BYTES, + 0, // no vbucket + butil::HostToNet32(scope_name.size()), + 0, // opaque + 0 // no CAS + }; + if (_buf.append(&header, sizeof(header))) { + return false; + } + if (_buf.append(scope_name.data(), scope_name.size())) { + return false; + } + ++_pipelined_count; + return true; +} + +bool CouchbaseRequest::SelectBucket(const butil::StringPiece &bucket_name) { + if (bucket_name.empty()) { + LOG(ERROR) << "Empty bucket name"; + return false; + } + //construct the request header + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_SELECT_BUCKET, + butil::HostToNet16(bucket_name.size()), + 0, + policy::CB_BINARY_RAW_BYTES, + 0, + butil::HostToNet32(bucket_name.size()), + 0, + 0 + }; + if (_buf.append(&header, sizeof(header))) { + std::cout<<"Failed to append header to buffer"<clear(); + auth_str->append(reinterpret_cast(&header), sizeof(header)); + auth_str->append(kPlainAuthCommand, sizeof(kPlainAuthCommand) - 1); + auth_str->append(username.data()); + auth_str->append(kPadding, sizeof(kPadding)); + auth_str->append(username.data()); + auth_str->append(kPadding, sizeof(kPadding)); + auth_str->append(password.data()); + if(_buf.append(auth_str->data(), auth_str->size())) { + std::cout<<"Failed to append auth string to buffer"<GetDirectBufferPointer(&data, &size)) { + tmp.append(data, size); + input->Skip(size); + } + const butil::IOBuf saved = tmp; + int count = 0; + for (; !tmp.empty(); ++count) { + char aux_buf[sizeof(policy::CouchbaseRequestHeader)]; + const policy::CouchbaseRequestHeader* header = + (const policy::CouchbaseRequestHeader*)tmp.fetch(aux_buf, sizeof(aux_buf)); + if (header == NULL) { + return false; + } + if (header->magic != (uint8_t)policy::CB_MAGIC_REQUEST) { + return false; + } + uint32_t total_body_length = butil::NetToHost32(header->total_body_length); + if (tmp.size() < sizeof(*header) + total_body_length) { + return false; + } + tmp.pop_front(sizeof(*header) + total_body_length); + } + _buf.append(saved); + _pipelined_count += count; + return true; +} + +void CouchbaseRequest::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + LOG(WARNING) << "You're not supposed to serialize a CouchbaseRequest"; + + // simple approach just making it work. + butil::IOBufAsZeroCopyInputStream wrapper(_buf); + const void* data = NULL; + int size = 0; + while (wrapper.Next(&data, &size)) { + output->WriteRaw(data, size); + } +} + +size_t CouchbaseRequest::ByteSizeLong() const { + int total_size = static_cast(_buf.size()); + _cached_size_ = total_size; + return total_size; +} + +void CouchbaseRequest::MergeFrom(const CouchbaseRequest& from) { + CHECK_NE(&from, this); + _buf.append(from._buf); + _pipelined_count += from._pipelined_count; +} + +bool CouchbaseRequest::IsInitialized() const { + return _pipelined_count != 0; +} + +void CouchbaseRequest::Swap(CouchbaseRequest* other) { + if (other != this) { + _buf.swap(other->_buf); + std::swap(_pipelined_count, other->_pipelined_count); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::google::protobuf::Metadata CouchbaseRequest::GetMetadata() const { + ::google::protobuf::Metadata metadata{}; + metadata.descriptor = CouchbaseRequestBase::descriptor(); + metadata.reflection = nullptr; + return metadata; +} +//redifinition +CouchbaseResponse::CouchbaseResponse() + : NonreflectableMessage() { + SharedCtor(); +} + +//redifinition +CouchbaseResponse::CouchbaseResponse(const CouchbaseResponse& from) + : NonreflectableMessage(from) { + SharedCtor(); + MergeFrom(from); +} + +void CouchbaseResponse::SharedCtor() { + _cached_size_ = 0; +} + +//redifinition +CouchbaseResponse::~CouchbaseResponse() { + SharedDtor(); +} + +void CouchbaseResponse::SharedDtor() { +} + +void CouchbaseResponse::SetCachedSize(int size) const { + _cached_size_ = size; +} + +void CouchbaseResponse::Clear() { +} + +bool CouchbaseResponse::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { + LOG(WARNING) << "You're not supposed to parse a CouchbaseResponse"; + + // simple approach just making it work. + const void* data = NULL; + int size = 0; + while (input->GetDirectBufferPointer(&data, &size)) { + _buf.append(data, size); + input->Skip(size); + } + return true; +} + +void CouchbaseResponse::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + LOG(WARNING) << "You're not supposed to serialize a CouchbaseResponse"; + + // simple approach just making it work. + butil::IOBufAsZeroCopyInputStream wrapper(_buf); + const void* data = NULL; + int size = 0; + while (wrapper.Next(&data, &size)) { + output->WriteRaw(data, size); + } +} + +size_t CouchbaseResponse::ByteSizeLong() const { + int total_size = static_cast(_buf.size()); + _cached_size_ = total_size; + return total_size; +} + +void CouchbaseResponse::MergeFrom(const CouchbaseResponse& from) { + CHECK_NE(&from, this); + _err = from._err; + // responses of memcached according to their binary layout, should be + // directly concatenatible. + _buf.append(from._buf); +} + +bool CouchbaseResponse::IsInitialized() const { + return !_buf.empty(); +} + +void CouchbaseResponse::Swap(CouchbaseResponse* other) { + if (other != this) { + _buf.swap(other->_buf); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::google::protobuf::Metadata CouchbaseResponse::GetMetadata() const { + ::google::protobuf::Metadata metadata{}; + metadata.descriptor = CouchbaseResponseBase::descriptor(); + metadata.reflection = nullptr; + return metadata; +} + +// =================================================================== + +const char* CouchbaseResponse::status_str(Status st) { + switch (st) { + case STATUS_SUCCESS: + return "SUCCESS"; + case STATUS_KEY_ENOENT: + return "The key does not exist"; + case STATUS_KEY_EEXISTS: + return "The key exists"; + case STATUS_E2BIG: + return "Arg list is too long"; + case STATUS_EINVAL: + return "Invalid argument"; + case STATUS_NOT_STORED: + return "Not stored"; + case STATUS_DELTA_BADVAL: + return "Bad delta"; + case STATUS_AUTH_ERROR: + return "authentication error"; + case STATUS_AUTH_CONTINUE: + return "authentication continue"; + case STATUS_UNKNOWN_COMMAND: + return "Unknown command"; + case STATUS_ENOMEM: + return "Out of memory"; + } + return "Unknown status"; +} + +// MUST NOT have extras. +// MUST have key. +// MUST NOT have value. +bool CouchbaseRequest::GetOrDelete(uint8_t command, const butil::StringPiece& key) { + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + command, + butil::HostToNet16(key.size()), + 0, + policy::CB_BINARY_RAW_BYTES, + 0, + butil::HostToNet32(key.size()), + 0, + 0 + }; + if (_buf.append(&header, sizeof(header))) { + return false; + } + if (_buf.append(key.data(), key.size())) { + return false; + } + ++_pipelined_count; + return true; +} + +bool CouchbaseRequest::Get(const butil::StringPiece& key) { + return GetOrDelete(policy::CB_BINARY_GET, key); +} + +bool CouchbaseRequest::Delete(const butil::StringPiece& key) { + return GetOrDelete(policy::CB_BINARY_DELETE, key); +} + +struct FlushHeaderWithExtras { + policy::CouchbaseRequestHeader header; + uint32_t exptime; +} __attribute__((packed)); +BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); + +// MAY have extras. +// MUST NOT have key. +// MUST NOT have value. +// Extra data for flush: +// Byte/ 0 | 1 | 2 | 3 | +// / | | | | +// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +// +---------------+---------------+---------------+---------------+ +// 0| Expiration | +// +---------------+---------------+---------------+---------------+ +// Total 4 bytes +bool CouchbaseRequest::Flush(uint32_t timeout) { + const uint8_t FLUSH_EXTRAS = (timeout == 0 ? 0 : 4); + FlushHeaderWithExtras header_with_extras = {{ + policy::CB_MAGIC_REQUEST, + policy::CB_BINARY_FLUSH, + 0, + FLUSH_EXTRAS, + policy::CB_BINARY_RAW_BYTES, + 0, + butil::HostToNet32(FLUSH_EXTRAS), + 0, + 0 }, butil::HostToNet32(timeout) }; + if (FLUSH_EXTRAS == 0) { + if (_buf.append(&header_with_extras.header, + sizeof(policy::CouchbaseRequestHeader))) { + return false; + } + } else { + if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { + return false; + } + } + ++_pipelined_count; + return true; +} + +// (if found): +// MUST have extras. +// MAY have key. +// MAY have value. +// Extra data for the get commands: +// Byte/ 0 | 1 | 2 | 3 | +// / | | | | +// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +// +---------------+---------------+---------------+---------------+ +// 0| Flags | +// +---------------+---------------+---------------+---------------+ +// Total 4 bytes +bool CouchbaseResponse::PopGet( + butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != (uint8_t)policy::CB_BINARY_GET) { + butil::string_printf(&_err, "not a GET response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "response=%u < header=%u + body=%u", + (unsigned)n, (unsigned)sizeof(header), header.total_body_length); + return false; + } + if (header.status != (uint16_t)STATUS_SUCCESS) { + LOG_IF(ERROR, header.extras_length != 0) << "GET response must not have flags"; + LOG_IF(ERROR, header.key_length != 0) << "GET response must not have key"; + const int value_size = (int)header.total_body_length - (int)header.extras_length + - (int)header.key_length; + if (value_size < 0) { + butil::string_printf(&_err, "value_size=%d is non-negative", value_size); + return false; + } + _buf.pop_front(sizeof(header) + header.extras_length + + header.key_length); + _err.clear(); + _buf.cutn(&_err, value_size); + return false; + } + if (header.extras_length != 4u) { + butil::string_printf(&_err, "GET response must have flags as extras, actual length=%u", + header.extras_length); + return false; + } + if (header.key_length != 0) { + butil::string_printf(&_err, "GET response must not have key"); + return false; + } + const int value_size = (int)header.total_body_length - (int)header.extras_length + - (int)header.key_length; + if (value_size < 0) { + butil::string_printf(&_err, "value_size=%d is non-negative", value_size); + return false; + } + _buf.pop_front(sizeof(header)); + uint32_t raw_flags = 0; + _buf.cutn(&raw_flags, sizeof(raw_flags)); + if (flags) { + *flags = butil::NetToHost32(raw_flags); + } + if (value) { + value->clear(); + _buf.cutn(value, value_size); + } + if (cas_value) { + *cas_value = header.cas_value; + } + _err.clear(); + return true; +} + +bool CouchbaseResponse::PopGet( + std::string* value, uint32_t* flags, uint64_t* cas_value) { + butil::IOBuf tmp; + if (PopGet(&tmp, flags, cas_value)) { + tmp.copy_to(value); + return true; + } + return false; +} + +// MUST NOT have extras +// MUST NOT have key +// MUST NOT have value +bool CouchbaseResponse::PopDelete() { + return PopStore(policy::CB_BINARY_DELETE, NULL); +} +bool CouchbaseResponse::PopFlush() { + return PopStore(policy::CB_BINARY_FLUSH, NULL); +} + +struct StoreHeaderWithExtras { + policy::CouchbaseRequestHeader header; + uint32_t flags; + uint32_t exptime; +} __attribute__((packed)); +BAIDU_CASSERT(sizeof(StoreHeaderWithExtras) == 32, must_match); +const size_t STORE_EXTRAS = sizeof(StoreHeaderWithExtras) - + sizeof(policy::CouchbaseRequestHeader); +// MUST have extras. +// MUST have key. +// MAY have value. +// Extra data for set/add/replace: +// Byte/ 0 | 1 | 2 | 3 | +// / | | | | +// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +// +---------------+---------------+---------------+---------------+ +// 0| Flags | +// +---------------+---------------+---------------+---------------+ +// 4| Expiration | +// +---------------+---------------+---------------+---------------+ +// Total 8 bytes +bool CouchbaseRequest::Store( + uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value) { + StoreHeaderWithExtras header_with_extras = {{ + policy::CB_MAGIC_REQUEST, + command, + butil::HostToNet16(key.size()), + STORE_EXTRAS, + policy::CB_JSON, + 0, + butil::HostToNet32(STORE_EXTRAS + key.size() + value.size()), + 0, + butil::HostToNet64(cas_value) + }, butil::HostToNet32(flags), butil::HostToNet32(exptime)}; + if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { + return false; + } + if (_buf.append(key.data(), key.size())) { + return false; + } + if (_buf.append(value.data(), value.size())) { + return false; + } + ++_pipelined_count; + return true; +} + +// MUST have CAS +// MUST NOT have extras +// MUST NOT have key +// MUST NOT have value +bool CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != command) { + butil::string_printf(&_err, "Not a STORE response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "Not enough data"); + return false; + } + LOG_IF(ERROR, header.extras_length != 0) << "STORE response must not have flags"; + LOG_IF(ERROR, header.key_length != 0) << "STORE response must not have key"; + int value_size = (int)header.total_body_length - (int)header.extras_length + - (int)header.key_length; + if (header.status != (uint16_t)STATUS_SUCCESS) { + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + _err.clear(); + _buf.cutn(&_err, value_size); + return false; + } + LOG_IF(ERROR, value_size != 0) << "STORE response must not have value, actually=" + << value_size; + _buf.pop_front(sizeof(header) + header.total_body_length); + if (cas_value) { + CHECK(header.cas_value); + *cas_value = header.cas_value; + } + _err.clear(); + return true; +} + +bool CouchbaseRequest::Set( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value) { + return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value); +} + +// Collection Management Methods +bool CouchbaseRequest::GetCollectionsManifest() { + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_GET_COLLECTIONS_MANIFEST, + 0, // no key + 0, // no extras + policy::CB_BINARY_RAW_BYTES, + 0, // no vbucket + 0, // no body + 0, // opaque + 0 // no CAS + }; + if (_buf.append(&header, sizeof(header))) { + return false; + } + ++_pipelined_count; + return true; +} + +bool CouchbaseRequest::GetCollectionId(const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name) { + // Format the collection path as "scope.collection" + std::string collection_path = scope_name.as_string() + "." + collection_name.as_string(); + + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_COLLECTIONS_GET_CID, + butil::HostToNet16(collection_path.size()), + 0, // no extras + policy::CB_BINARY_RAW_BYTES, + 0, // no vbucket + butil::HostToNet32(collection_path.size()), + 0, // opaque + 0 // no CAS + }; + if (_buf.append(&header, sizeof(header))) { + return false; + } + if (_buf.append(collection_path.data(), collection_path.size())) { + return false; + } + ++_pipelined_count; + return true; +} + +// Collection-aware document operations +bool CouchbaseRequest::GetFromCollection(const butil::StringPiece& key, + const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name) { + // First, we need to construct a collection-aware key + // For simplicity, we'll use the format: scope.collection::key + std::string scoped_key = scope_name.as_string() + "." + + collection_name.as_string() + "::" + + key.as_string(); + + return GetOrDelete(policy::CB_BINARY_GET, butil::StringPiece(scoped_key)); +} + +bool CouchbaseRequest::SetToCollection(const butil::StringPiece& key, + const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name) { + // Construct a collection-aware key + std::string scoped_key = scope_name.as_string() + "." + + collection_name.as_string() + "::" + + key.as_string(); + + return Store(policy::CB_BINARY_SET, butil::StringPiece(scoped_key), value, flags, exptime, cas_value); +} + +bool CouchbaseRequest::Add( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value) { + return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value); +} + +bool CouchbaseRequest::Replace( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value) { + return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, cas_value); +} + +bool CouchbaseRequest::Append( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value) { + if (value.empty()) { + LOG(ERROR) << "value to append must be non-empty"; + return false; + } + return Store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value); +} + +bool CouchbaseRequest::Prepend( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value) { + if (value.empty()) { + LOG(ERROR) << "value to prepend must be non-empty"; + return false; + } + return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, cas_value); +} + +bool CouchbaseResponse::PopSet(uint64_t* cas_value) { + return PopStore(policy::CB_BINARY_SET, cas_value); +} +bool CouchbaseResponse::PopAdd(uint64_t* cas_value) { + return PopStore(policy::CB_BINARY_ADD, cas_value); +} +bool CouchbaseResponse::PopReplace(uint64_t* cas_value) { + return PopStore(policy::CB_BINARY_REPLACE, cas_value); +} +bool CouchbaseResponse::PopAppend(uint64_t* cas_value) { + return PopStore(policy::CB_BINARY_APPEND, cas_value); +} +bool CouchbaseResponse::PopPrepend(uint64_t* cas_value) { + return PopStore(policy::CB_BINARY_PREPEND, cas_value); +} + +// Collection-related response methods +bool CouchbaseResponse::PopCollectionsManifest(std::string* manifest_json) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != policy::CB_GET_COLLECTIONS_MANIFEST) { + butil::string_printf(&_err, "Not a collections manifest response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "Not enough data"); + return false; + } + if (header.status != 0) { + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + _err.clear(); + int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; + _buf.cutn(&_err, value_size); + return false; + } + + // Skip header and extras/key if any + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + + // Get the manifest JSON + int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; + if (manifest_json && value_size > 0) { + _buf.cutn(manifest_json, value_size); + } else { + _buf.pop_front(value_size); + } + + _err.clear(); + return true; +} + +bool CouchbaseResponse::PopCollectionId(uint32_t* collection_id) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != policy::CB_COLLECTIONS_GET_CID) { + butil::string_printf(&_err, "Not a collection ID response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "Not enough data"); + return false; + } + if (header.status != 0) { + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + _err.clear(); + int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; + _buf.cutn(&_err, value_size); + return false; + } + + // Skip header and extras/key if any + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + + // Get the collection ID (typically 4 bytes) + int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; + if (collection_id && value_size >= 4) { + uint32_t cid; + _buf.copy_to(&cid, 4); + *collection_id = butil::NetToHost32(cid); + _buf.pop_front(value_size); + } else { + _buf.pop_front(value_size); + } + + _err.clear(); + return true; +} + +// Collection-aware response methods +bool CouchbaseResponse::PopGetFromCollection(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value) { + // Same implementation as PopGet, just aliased for clarity + return PopGet(value, flags, cas_value); +} + +bool CouchbaseResponse::PopSetToCollection(uint64_t* cas_value) { + // Same implementation as PopSet, just aliased for clarity + return PopSet(cas_value); +} + +struct IncrHeaderWithExtras { + policy::CouchbaseRequestHeader header; + uint64_t delta; + uint64_t initial_value; + uint32_t exptime; +} __attribute__((packed)); +BAIDU_CASSERT(sizeof(IncrHeaderWithExtras) == 44, must_match); + +const size_t INCR_EXTRAS = sizeof(IncrHeaderWithExtras) - + sizeof(policy::CouchbaseRequestHeader); + +// MUST have extras. +// MUST have key. +// MUST NOT have value. +// Extra data for incr/decr: +// Byte/ 0 | 1 | 2 | 3 | +// / | | | | +// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +// +---------------+---------------+---------------+---------------+ +// 0| Delta to add / subtract | +// | | +// +---------------+---------------+---------------+---------------+ +// 8| Initial value | +// | | +// +---------------+---------------+---------------+---------------+ +// 16| Expiration | +// +---------------+---------------+---------------+---------------+ +// Total 20 bytes +bool CouchbaseRequest::Counter( + uint8_t command, const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime) { + IncrHeaderWithExtras header_with_extras = {{ + policy::CB_MAGIC_REQUEST, + command, + butil::HostToNet16(key.size()), + INCR_EXTRAS, + policy::CB_BINARY_RAW_BYTES, + 0, + butil::HostToNet32(INCR_EXTRAS + key.size()), + 0, + 0 }, butil::HostToNet64(delta), butil::HostToNet64(initial_value), butil::HostToNet32(exptime) }; + if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { + return false; + } + if (_buf.append(key.data(), key.size())) { + return false; + } + ++_pipelined_count; + return true; +} + +bool CouchbaseRequest::Increment(const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime) { + return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, exptime); +} + +bool CouchbaseRequest::Decrement(const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime) { + return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, exptime); +} + +// MUST NOT have extras. +// MUST NOT have key. +// MUST have value. +// Byte/ 0 | 1 | 2 | 3 | +// / | | | | +// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +// +---------------+---------------+---------------+---------------+ +// 0| 64-bit unsigned response. | +// | | +// +---------------+---------------+---------------+---------------+ +// Total 8 bytes +bool CouchbaseResponse::PopCounter( + uint8_t command, uint64_t* new_value, uint64_t* cas_value) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != command) { + butil::string_printf(&_err, "not a INCR/DECR response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "response=%u < header=%u + body=%u", + (unsigned)n, (unsigned)sizeof(header), header.total_body_length); + return false; + } + LOG_IF(ERROR, header.extras_length != 0) << "INCR/DECR response must not have flags"; + LOG_IF(ERROR, header.key_length != 0) << "INCR/DECR response must not have key"; + const int value_size = (int)header.total_body_length - (int)header.extras_length + - (int)header.key_length; + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + + if (header.status != (uint16_t)STATUS_SUCCESS) { + if (value_size < 0) { + butil::string_printf(&_err, "value_size=%d is negative", value_size); + } else { + _err.clear(); + _buf.cutn(&_err, value_size); + } + return false; + } + if (value_size != 8) { + butil::string_printf(&_err, "value_size=%d is not 8", value_size); + return false; + } + uint64_t raw_value = 0; + _buf.cutn(&raw_value, sizeof(raw_value)); + *new_value = butil::NetToHost64(raw_value); + if (cas_value) { + *cas_value = header.cas_value; + } + _err.clear(); + return true; +} + +bool CouchbaseResponse::PopIncrement(uint64_t* new_value, uint64_t* cas_value) { + return PopCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); +} +bool CouchbaseResponse::PopDecrement(uint64_t* new_value, uint64_t* cas_value) { + return PopCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); +} + +// MUST have extras. +// MUST have key. +// MUST NOT have value. +struct TouchHeaderWithExtras { + policy::CouchbaseRequestHeader header; + uint32_t exptime; +} __attribute__((packed)); +BAIDU_CASSERT(sizeof(TouchHeaderWithExtras) == 28, must_match); +const size_t TOUCH_EXTRAS = sizeof(TouchHeaderWithExtras) - sizeof(policy::CouchbaseRequestHeader); + +// MAY have extras. +// MUST NOT have key. +// MUST NOT have value. +// Extra data for touch: +// Byte/ 0 | 1 | 2 | 3 | +// / | | | | +// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| +// +---------------+---------------+---------------+---------------+ +// 0| Expiration | +// +---------------+---------------+---------------+---------------+ +// Total 4 bytes +bool CouchbaseRequest::Touch(const butil::StringPiece& key, uint32_t exptime) { + TouchHeaderWithExtras header_with_extras = {{ + policy::CB_MAGIC_REQUEST, + policy::CB_BINARY_TOUCH, + butil::HostToNet16(key.size()), + TOUCH_EXTRAS, + policy::CB_BINARY_RAW_BYTES, + 0, + butil::HostToNet32(TOUCH_EXTRAS + key.size()), + 0, + 0 }, butil::HostToNet32(exptime) }; + if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { + return false; + } + if (_buf.append(key.data(), key.size())) { + return false; + } + ++_pipelined_count; + return true; +} + +// MUST NOT have extras. +// MUST NOT have key. +// MUST NOT have value. +bool CouchbaseRequest::Version() { + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_BINARY_VERSION, + 0, + 0, + policy::CB_BINARY_RAW_BYTES, + 0, + 0, + 0, + 0 + }; + if (_buf.append(&header, sizeof(header))) { + return false; + } + ++_pipelined_count; + return true; +} + +// MUST NOT have extras. +// MUST NOT have key. +// MUST have value. +bool CouchbaseResponse::PopVersion(std::string* version) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != policy::CB_BINARY_VERSION) { + butil::string_printf(&_err, "not a VERSION response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "response=%u < header=%u + body=%u", + (unsigned)n, (unsigned)sizeof(header), header.total_body_length); + return false; + } + LOG_IF(ERROR, header.extras_length != 0) << "VERSION response must not have flags"; + LOG_IF(ERROR, header.key_length != 0) << "VERSION response must not have key"; + const int value_size = (int)header.total_body_length - (int)header.extras_length + - (int)header.key_length; + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + if (value_size < 0) { + butil::string_printf(&_err, "value_size=%d is negative", value_size); + return false; + } + if (header.status != (uint16_t)STATUS_SUCCESS) { + _err.clear(); + _buf.cutn(&_err, value_size); + return false; + } + if (version) { + version->clear(); + _buf.cutn(version, value_size); + } + _err.clear(); + return true; +} + +} // namespace brpc diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h new file mode 100644 index 0000000000..aecd6b7bfd --- /dev/null +++ b/src/brpc/couchbase.h @@ -0,0 +1,178 @@ +#ifndef BRPC_COUCHBASE_H +#define BRPC_COUCHBASE_H + +#endif // COUCHBASE_MEMCACHE_H + +#include +#include "butil/iobuf.h" +#include "butil/strings/string_piece.h" +#include "brpc/nonreflectable_message.h" +#include "brpc/pb_compat.h" + +namespace brpc { + class CouchbaseRequest : public NonreflectableMessage { + private: + int _pipelined_count; + butil::IOBuf _buf; + mutable int _cached_size_; + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + bool GetOrDelete(uint8_t command, const butil::StringPiece& key); + bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + + bool Store(uint8_t command, const butil::StringPiece& key, + const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + public: + CouchbaseRequest(); + ~CouchbaseRequest() override; + CouchbaseRequest(const CouchbaseRequest& from); + inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { + CopyFrom(from); + return *this; + } + + bool SelectBucket(const butil::StringPiece &bucket_name); + bool Authenticate(const butil::StringPiece &username, + const butil::StringPiece &password); + + // Collection Management Methods + bool GetCollectionsManifest(); + bool GetCollectionId(const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name); + + bool GetScopeId(const butil::StringPiece& scope_name); + + // Collection-aware document operations + bool Get(const butil::StringPiece& key); + bool GetFromCollection(const butil::StringPiece& key, + const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name); + + bool Set(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + bool SetToCollection(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + const butil::StringPiece& scope_name, const butil::StringPiece& collection_name); + + bool Add(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + + bool Replace(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + + bool Append(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + + bool Prepend(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + + bool Delete(const butil::StringPiece& key); + bool Flush(uint32_t timeout); + + bool Increment(const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + bool Decrement(const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + + bool Touch(const butil::StringPiece& key, uint32_t exptime); + + bool Version(); + + int pipelined_count() const { return _pipelined_count; } + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + void Swap(CouchbaseRequest* other); + void MergeFrom(const CouchbaseRequest& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; + + }; + + class CouchbaseResponse : public NonreflectableMessage { + private: + std::string _err; + butil::IOBuf _buf; + mutable int _cached_size_; + bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); + bool PopStore(uint8_t command, uint64_t* cas_value); + + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + public: + CouchbaseResponse(); + ~CouchbaseResponse() override; + CouchbaseResponse(const CouchbaseResponse& from); + inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { + CopyFrom(from); + return *this; + } + enum Status { + STATUS_SUCCESS = 0x00, + STATUS_KEY_ENOENT = 0x01, + STATUS_KEY_EEXISTS = 0x02, + STATUS_E2BIG = 0x03, + STATUS_EINVAL = 0x04, + STATUS_NOT_STORED = 0x05, + STATUS_DELTA_BADVAL = 0x06, + STATUS_AUTH_ERROR = 0x20, + STATUS_AUTH_CONTINUE = 0x21, + STATUS_UNKNOWN_COMMAND = 0x81, + STATUS_ENOMEM = 0x82 + }; + + void MergeFrom(const CouchbaseResponse& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + static const char* status_str(Status); + // Add methods to handle response parsing + void Swap(CouchbaseResponse* other); + bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); + bool PopGet(std::string* value, uint32_t* flags, uint64_t* cas_value); + const std::string& LastError() const { return _err; } + bool PopSet(uint64_t* cas_value); + bool PopAdd(uint64_t* cas_value); + bool PopReplace(uint64_t* cas_value); + bool PopAppend(uint64_t* cas_value); + bool PopPrepend(uint64_t* cas_value); + + // Collection-related response methods + bool PopCollectionsManifest(std::string* manifest_json); + bool PopCollectionId(uint32_t* collection_id); + + // Collection-aware response methods (same as regular but for documentation) + bool PopGetFromCollection(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); + bool PopSetToCollection(uint64_t* cas_value); + + bool PopDelete(); + bool PopFlush(); + bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); + bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); + bool PopTouch(); + bool PopVersion(std::string* version); + }; +} \ No newline at end of file diff --git a/src/brpc/global.cpp b/src/brpc/global.cpp index 0196b6d008..1c55a842c1 100644 --- a/src/brpc/global.cpp +++ b/src/brpc/global.cpp @@ -75,6 +75,7 @@ #include "brpc/policy/ubrpc2pb_protocol.h" #include "brpc/policy/sofa_pbrpc_protocol.h" #include "brpc/policy/memcache_binary_protocol.h" +#include "brpc/policy/couchbase_protocol.h" #include "brpc/policy/streaming_rpc_protocol.h" #include "brpc/policy/mongo_protocol.h" #include "brpc/policy/redis_protocol.h" @@ -518,6 +519,16 @@ static void GlobalInitializeOrDieImpl() { exit(1); } + Protocol couchbase_protocol = { ParseCouchbaseMessage, + SerializeCouchbaseRequest, + PackCouchbaseRequest, + NULL, ProcessCouchbaseResponse, + NULL, NULL, GetCouchbaseMethodName, + CONNECTION_TYPE_POOLED_AND_SHORT, "couchbase" }; + if (RegisterProtocol(PROTOCOL_COUCHBASE, couchbase_protocol) != 0) { + exit(1); + } + Protocol redis_protocol = { ParseRedisMessage, SerializeRedisRequest, PackRedisRequest, diff --git a/src/brpc/options.proto b/src/brpc/options.proto index 34001d7bb8..4ad97aa828 100644 --- a/src/brpc/options.proto +++ b/src/brpc/options.proto @@ -64,6 +64,7 @@ enum ProtocolType { PROTOCOL_CDS_AGENT = 24; // Client side only PROTOCOL_ESP = 25; // Client side only PROTOCOL_H2 = 26; + PROTOCOL_COUCHBASE = 27; } enum CompressType { diff --git a/src/brpc/policy/couchbase_protocol.cpp b/src/brpc/policy/couchbase_protocol.cpp new file mode 100644 index 0000000000..ec47a9a1ea --- /dev/null +++ b/src/brpc/policy/couchbase_protocol.cpp @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include // MethodDescriptor +#include // Message +#include +#include "butil/logging.h" // LOG() +#include "butil/time.h" +#include "butil/iobuf.h" // butil::IOBuf +#include "butil/sys_byteorder.h" +#include "brpc/controller.h" // Controller +#include "brpc/details/controller_private_accessor.h" +#include "brpc/socket.h" // Socket +#include "brpc/server.h" // Server +#include "brpc/details/server_private_accessor.h" +#include "brpc/span.h" +#include "brpc/compress.h" // ParseFromCompressedData +#include "brpc/policy/couchbase_protocol.h" +#include "brpc/couchbase.h" // CouchbaseRequest, CouchbaseResponse +#include "brpc/policy/most_common_message.h" +#include "butil/containers/flat_map.h" + + +namespace brpc { + +DECLARE_bool(enable_rpcz); + +namespace policy { + +BAIDU_CASSERT(sizeof(CouchbaseRequestHeader) == 24, must_match); +BAIDU_CASSERT(sizeof(CouchbaseResponseHeader) == 24, must_match); + +static uint64_t supported_cmd_map[8]; +static pthread_once_t supported_cmd_map_once = PTHREAD_ONCE_INIT; + +static void InitSupportedCommandMap() { + butil::bit_array_clear(supported_cmd_map, 256); + butil::bit_array_set(supported_cmd_map, CB_BINARY_GET); + butil::bit_array_set(supported_cmd_map, CB_SELECT_BUCKET); + butil::bit_array_set(supported_cmd_map, CB_GET_SCOPE_ID); + butil::bit_array_set(supported_cmd_map, CB_BINARY_SET); + butil::bit_array_set(supported_cmd_map, CB_BINARY_ADD); + butil::bit_array_set(supported_cmd_map, CB_BINARY_REPLACE); + butil::bit_array_set(supported_cmd_map, CB_BINARY_DELETE); + butil::bit_array_set(supported_cmd_map, CB_BINARY_INCREMENT); + butil::bit_array_set(supported_cmd_map, CB_BINARY_DECREMENT); + butil::bit_array_set(supported_cmd_map, CB_BINARY_FLUSH); + butil::bit_array_set(supported_cmd_map, CB_BINARY_VERSION); + butil::bit_array_set(supported_cmd_map, CB_BINARY_NOOP); + butil::bit_array_set(supported_cmd_map, CB_BINARY_APPEND); + butil::bit_array_set(supported_cmd_map, CB_BINARY_PREPEND); + butil::bit_array_set(supported_cmd_map, CB_BINARY_STAT); + butil::bit_array_set(supported_cmd_map, CB_BINARY_TOUCH); + butil::bit_array_set(supported_cmd_map, CB_BINARY_SASL_AUTH); + // Collection management commands + butil::bit_array_set(supported_cmd_map, CB_GET_COLLECTIONS_MANIFEST); + butil::bit_array_set(supported_cmd_map, CB_COLLECTIONS_GET_CID); + butil::bit_array_set(supported_cmd_map, CB_COLLECTIONS_GET_SCOPE_ID); +} + +inline bool IsSupportedCommand(uint8_t command) { + pthread_once(&supported_cmd_map_once, InitSupportedCommandMap); + return butil::bit_array_get(supported_cmd_map, command); +} + +ParseResult ParseCouchbaseMessage(butil::IOBuf* source, + Socket* socket, bool /*read_eof*/, const void */*arg*/) { + while (1) { + const uint8_t* p_mcmagic = (const uint8_t*)source->fetch1(); + if (NULL == p_mcmagic) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + if (*p_mcmagic != (uint8_t)CB_MAGIC_RESPONSE) { + return MakeParseError(PARSE_ERROR_TRY_OTHERS); + } + char buf[24]; + const uint8_t* p = (const uint8_t*)source->fetch(buf, sizeof(buf)); + if (NULL == p) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + const CouchbaseResponseHeader* header = (const CouchbaseResponseHeader*)p; + uint32_t total_body_length = butil::NetToHost32(header->total_body_length); + if (source->size() < sizeof(*header) + total_body_length) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + + if (!IsSupportedCommand(header->command)) { + LOG(WARNING) << "Not support command=" << header->command; + source->pop_front(sizeof(*header) + total_body_length); + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + + PipelinedInfo pi; + if (!socket->PopPipelinedInfo(&pi)) { + LOG(WARNING) << "No corresponding PipelinedInfo in socket, drop"; + source->pop_front(sizeof(*header) + total_body_length); + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + MostCommonMessage* msg = + static_cast(socket->parsing_context()); + if (msg == NULL) { + msg = MostCommonMessage::Get(); + socket->reset_parsing_context(msg); + } + + // endianness conversions. + const CouchbaseResponseHeader local_header = { + header->magic, + header->command, + butil::NetToHost16(header->key_length), + header->extras_length, + header->data_type, + butil::NetToHost16(header->status), + total_body_length, + butil::NetToHost32(header->opaque), + butil::NetToHost64(header->cas_value), + }; + msg->meta.append(&local_header, sizeof(local_header)); + source->pop_front(sizeof(*header)); + source->cutn(&msg->meta, total_body_length); + if (header->command == CB_BINARY_SASL_AUTH) { + if (header->status != 0) { + LOG(ERROR) << "Failed to authenticate the couchbase Server."; + return MakeParseError(PARSE_ERROR_NO_RESOURCE, + "Fail to authenticate with the couchbase Server"); + } + msg = static_cast(socket->release_parsing_context()); + msg->pi = pi; + return MakeMessage(msg); + } else { + std::cout<<"Received Couchbase Response with command: " << std::hex << header->command << std::endl; + std::cout<<"Status: " << std::hex << header->status << std::hex; + if (++msg->pi.count >= pi.count) { + CHECK_EQ(msg->pi.count, pi.count); + msg = static_cast(socket->release_parsing_context()); + msg->pi = pi; + return MakeMessage(msg); + } else { + socket->GivebackPipelinedInfo(pi); + } + } + } +} + +void ProcessCouchbaseResponse(InputMessageBase* msg_base) { + const int64_t start_parse_us = butil::cpuwide_time_us(); + DestroyingPtr msg(static_cast(msg_base)); + + const bthread_id_t cid = msg->pi.id_wait; + Controller* cntl = NULL; + const int rc = bthread_id_lock(cid, (void**)&cntl); + if (rc != 0) { + LOG_IF(ERROR, rc != EINVAL && rc != EPERM) + << "Fail to lock correlation_id=" << cid << ": " << berror(rc); + return; + } + + ControllerPrivateAccessor accessor(cntl); + Span* span = accessor.span(); + if (span) { + span->set_base_real_us(msg->base_real_us()); + span->set_received_us(msg->received_us()); + span->set_response_size(msg->meta.length()); + span->set_start_parse_us(start_parse_us); + } + const int saved_error = cntl->ErrorCode(); + if (cntl->response() == NULL) { + cntl->SetFailed(ERESPONSE, "response is NULL!"); + } else if (cntl->response()->GetDescriptor() != CouchbaseResponse::descriptor()) { + cntl->SetFailed(ERESPONSE, "Must be CouchbaseResponse"); + } else { + // We work around ParseFrom of pb which is just a placeholder. + ((CouchbaseResponse*)cntl->response())->raw_buffer() = msg->meta.movable(); + if (msg->pi.count != accessor.pipelined_count()) { + cntl->SetFailed(ERESPONSE, "pipelined_count=%d of response does " + "not equal request's=%d", + msg->pi.count, accessor.pipelined_count()); + } + } + // Unlocks correlation_id inside. Revert controller's + // error code if it version check of `cid' fails + msg.reset(); // optional, just release resource ASAP + accessor.OnResponse(cid, saved_error); +} + +void SerializeCouchbaseRequest(butil::IOBuf* buf, + Controller* cntl, + const google::protobuf::Message* request) { + if (request == NULL) { + return cntl->SetFailed(EREQUEST, "request is NULL"); + } + if (request->GetDescriptor() != CouchbaseRequest::descriptor()) { + return cntl->SetFailed(EREQUEST, "Must be CouchbaseRequest"); + } + const CouchbaseRequest* mr = (const CouchbaseRequest*)request; + // We work around SerializeTo of pb which is just a placeholder. + *buf = mr->raw_buffer(); + ControllerPrivateAccessor(cntl).set_pipelined_count(mr->pipelined_count()); +} + +void PackCouchbaseRequest(butil::IOBuf* buf, + SocketMessage**, + uint64_t /*correlation_id*/, + const google::protobuf::MethodDescriptor*, + Controller* cntl, + const butil::IOBuf& request, + const Authenticator* auth) { + if (auth) { + std::cout<<"Appending authentication data to request"<GenerateCredential(&auth_str) != 0) { + return cntl->SetFailed(EREQUEST, "Fail to generate credential"); + } + if (auth_str.empty()) { + return cntl->SetFailed(EREQUEST, "Empty auth_str"); + } + buf->append(auth_str); + // pipelined_count(); + } + else{ + buf->append(request); + } +} + +const std::string& GetCouchbaseMethodName( + const google::protobuf::MethodDescriptor*, + const Controller*) { + const static std::string CouchbaseD_STR = "Couchbase"; + return CouchbaseD_STR; +} + +} // namespace policy +} // namespace brpc diff --git a/src/brpc/policy/couchbase_protocol.h b/src/brpc/policy/couchbase_protocol.h new file mode 100644 index 0000000000..0944a4283b --- /dev/null +++ b/src/brpc/policy/couchbase_protocol.h @@ -0,0 +1,166 @@ + +#ifndef BRPC_POLICY_COUCHBASE_BINARY_PROTOCOL_H +#define BRPC_POLICY_COUCHBASE_BINARY_PROTOCOL_H + +#include "brpc/protocol.h" + +namespace brpc { +namespace policy { + + enum CouchbaseMagic { + CB_MAGIC_REQUEST = 0x80, + CB_MAGIC_RESPONSE = 0x81 + }; + + // Definition of the data types in the packet + // https://github.com/couchbase/kv_engine/blob/master/docs/BinaryProtocol.md + enum CouchbaseBinaryDataType { + CB_BINARY_RAW_BYTES = 0x00 + }; + + enum CouchbaseJsonDataType { + CB_JSON = 0x01 + }; + + // Definition of the different command opcodes. + // https://github.com/couchbase/kv_engine/blob/master/docs/BinaryProtocol.md + enum CouchbaseBinaryCommand { + CB_SELECT_BUCKET = 0x89, + CB_GET_SCOPE_ID = 0xBC, + CB_BINARY_GET = 0x00, + CB_BINARY_SET = 0x01, + CB_BINARY_ADD = 0x02, + CB_BINARY_REPLACE = 0x03, + CB_BINARY_DELETE = 0x04, + CB_BINARY_INCREMENT = 0x05, + CB_BINARY_DECREMENT = 0x06, + CB_BINARY_QUIT = 0x07, + CB_BINARY_FLUSH = 0x08, + CB_BINARY_GETQ = 0x09, + CB_BINARY_NOOP = 0x0a, + CB_BINARY_VERSION = 0x0b, + CB_BINARY_GETK = 0x0c, + CB_BINARY_GETKQ = 0x0d, + CB_BINARY_APPEND = 0x0e, + CB_BINARY_PREPEND = 0x0f, + CB_BINARY_STAT = 0x10, + CB_BINARY_SETQ = 0x11, + CB_BINARY_ADDQ = 0x12, + CB_BINARY_REPLACEQ = 0x13, + CB_BINARY_DELETEQ = 0x14, + CB_BINARY_INCREMENTQ = 0x15, + CB_BINARY_DECREMENTQ = 0x16, + CB_BINARY_QUITQ = 0x17, + CB_BINARY_FLUSHQ = 0x18, + CB_BINARY_APPENDQ = 0x19, + CB_BINARY_PREPENDQ = 0x1a, + CB_BINARY_TOUCH = 0x1c, + CB_BINARY_GAT = 0x1d, + CB_BINARY_GATQ = 0x1e, + CB_BINARY_GATK = 0x23, + CB_BINARY_GATKQ = 0x24, + + CB_BINARY_SASL_LIST_MECHS = 0x20, + CB_BINARY_SASL_AUTH = 0x21, + CB_BINARY_SASL_STEP = 0x22, + + // Collection Management Commands (Couchbase 7.0+) + CB_GET_CLUSTER_CONFIG = 0xb5, + CB_GET_COLLECTIONS_MANIFEST = 0xba, + CB_COLLECTIONS_GET_CID = 0xbb, + CB_COLLECTIONS_GET_SCOPE_ID = 0xbc, + + }; + + struct CouchbaseRequestHeader { + // Magic number identifying the package (See Couchbase Binary Protocol#Magic_Byte) + uint8_t magic; + + // Command code (See Couchbase Binary Protocol#Command_opcodes) + uint8_t command; + + // Length in bytes of the text key that follows the command extras + uint16_t key_length; + + // Length in bytes of the command extras + uint8_t extras_length; + + // Reserved for future use (See Couchbase Binary Protocol#Data_Type) + uint8_t data_type; + + // The virtual bucket for this command + uint16_t vbucket_id; + + // Length in bytes of extra + key + value + uint32_t total_body_length; + + // Will be copied back to you in the response + uint32_t opaque; + + // Data version check + uint64_t cas_value; + }; + + struct CouchbaseResponseHeader { + // Magic number identifying the package (See Couchbase Binary Protocol#Magic_Byte) + uint8_t magic; + + // Command code (See Couchbase Binary Protocol#Command_opcodes) + uint8_t command; + + // Length in bytes of the text key that follows the command extras + uint16_t key_length; + + // Length in bytes of the command extras + uint8_t extras_length; + + // Reserved for future use (See Couchbase Binary Protocol#Data_Type) + uint8_t data_type; + + // Status of the response (non-zero on error) + uint16_t status; + + // Length in bytes of extra + key + value + uint32_t total_body_length; + + // Will be copied back to you in the response + uint32_t opaque; + + // Data version check + uint64_t cas_value; + }; + + // Parse couchbase messages. + ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket *socket, bool read_eof, + const void *arg); + + // Actions to a couchbase response. + void ProcessCouchbaseResponse(InputMessageBase* msg); + + // Serialize a couchbase request. + void SerializeCouchbaseRequest(butil::IOBuf* buf, + Controller* cntl, + const google::protobuf::Message* request); + + // Pack `request' to `method' into `buf'. + void PackCouchbaseRequest(butil::IOBuf* buf, + SocketMessage**, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* controller, + const butil::IOBuf& request, + const Authenticator* auth); + + //process couchbase request. + //since, there is no server side instance running, this function is not implemented. + //void ProcessCouchbaseRequest(InputMessageBase* msg); + + const std::string& GetCouchbaseMethodName( + const google::protobuf::MethodDescriptor*, + const Controller*); + + } // namespace policy +} // namespace brpc + + +#endif // BRPC_POLICY_COUCHBASE_BINARY_PROTOCOL_H diff --git a/src/brpc/proto_base.proto b/src/brpc/proto_base.proto index b278ddb6bf..3fcdda09fa 100644 --- a/src/brpc/proto_base.proto +++ b/src/brpc/proto_base.proto @@ -25,6 +25,9 @@ message RedisResponseBase {} message EspMessageBase {} +message CouchbaseRequestBase {} +message CouchbaseResponseBase {} + message MemcacheRequestBase {} message MemcacheResponseBase {} From aa034c72494b59ce38f457c76074b48c7089508e Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 31 Jul 2025 10:38:19 +0530 Subject: [PATCH 02/49] added support for single connection type for couchbase --- src/brpc/global.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/brpc/global.cpp b/src/brpc/global.cpp index 1c55a842c1..82ec32e9e5 100644 --- a/src/brpc/global.cpp +++ b/src/brpc/global.cpp @@ -524,7 +524,7 @@ static void GlobalInitializeOrDieImpl() { PackCouchbaseRequest, NULL, ProcessCouchbaseResponse, NULL, NULL, GetCouchbaseMethodName, - CONNECTION_TYPE_POOLED_AND_SHORT, "couchbase" }; + CONNECTION_TYPE_ALL, "couchbase" }; if (RegisterProtocol(PROTOCOL_COUCHBASE, couchbase_protocol) != 0) { exit(1); } From 9d816696c99302d4641bd779646fcaf9dc916102 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 31 Jul 2025 10:39:26 +0530 Subject: [PATCH 03/49] removed unnecessary cout statements --- src/brpc/policy/couchbase_protocol.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/brpc/policy/couchbase_protocol.cpp b/src/brpc/policy/couchbase_protocol.cpp index ec47a9a1ea..695f7927c1 100644 --- a/src/brpc/policy/couchbase_protocol.cpp +++ b/src/brpc/policy/couchbase_protocol.cpp @@ -51,6 +51,7 @@ static pthread_once_t supported_cmd_map_once = PTHREAD_ONCE_INIT; static void InitSupportedCommandMap() { butil::bit_array_clear(supported_cmd_map, 256); butil::bit_array_set(supported_cmd_map, CB_BINARY_GET); + butil::bit_array_set(supported_cmd_map, CB_HELLO_SELECT_FEATURES); butil::bit_array_set(supported_cmd_map, CB_SELECT_BUCKET); butil::bit_array_set(supported_cmd_map, CB_GET_SCOPE_ID); butil::bit_array_set(supported_cmd_map, CB_BINARY_SET); @@ -143,8 +144,6 @@ ParseResult ParseCouchbaseMessage(butil::IOBuf* source, msg->pi = pi; return MakeMessage(msg); } else { - std::cout<<"Received Couchbase Response with command: " << std::hex << header->command << std::endl; - std::cout<<"Status: " << std::hex << header->status << std::hex; if (++msg->pi.count >= pi.count) { CHECK_EQ(msg->pi.count, pi.count); msg = static_cast(socket->release_parsing_context()); From 7946cb41647934472ad1352bdb558e9a37430f38 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 31 Jul 2025 10:40:27 +0530 Subject: [PATCH 04/49] added protocol code for helo packet --- src/brpc/policy/couchbase_protocol.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/brpc/policy/couchbase_protocol.h b/src/brpc/policy/couchbase_protocol.h index 0944a4283b..3d15fb6d70 100644 --- a/src/brpc/policy/couchbase_protocol.h +++ b/src/brpc/policy/couchbase_protocol.h @@ -25,6 +25,7 @@ namespace policy { // Definition of the different command opcodes. // https://github.com/couchbase/kv_engine/blob/master/docs/BinaryProtocol.md enum CouchbaseBinaryCommand { + CB_HELLO_SELECT_FEATURES = 0x1f, CB_SELECT_BUCKET = 0x89, CB_GET_SCOPE_ID = 0xBC, CB_BINARY_GET = 0x00, From d2ca0d4de170f141375c8200f08fb1eb86657caa Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 31 Jul 2025 10:41:42 +0530 Subject: [PATCH 05/49] fixed vbucketID code for identification, fixed add and get functions --- src/brpc/couchbase.cpp | 179 ++++++++++++++++++++++++++++++++++++----- src/brpc/couchbase.h | 2 + 2 files changed, 162 insertions(+), 19 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index c65215c9c1..5483970e30 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -18,6 +18,7 @@ #include "brpc/couchbase.h" #include "brpc/policy/couchbase_protocol.h" +#include //for crc32 Vbucket_id #include "brpc/proto_base.pb.h" #include "butil/logging.h" #include "butil/macros.h" @@ -26,6 +27,56 @@ namespace brpc { +static uint32_t hash_crc32(const char *key, size_t key_length) +{ + + static const uint32_t crc32tab[256] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, + 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 0x646ba8c0, 0xfd62f97a, + 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, + 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, 0x6b6b51f4, + 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, + 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, + 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 0xfed41b76, + 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, + 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, + 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, + 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 0xbdbdf21c, 0xcabac28a, 0x53b39330, + 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, + }; + + + uint64_t x; + uint32_t crc = UINT32_MAX; + + for (x = 0; x < key_length; x++) + crc = (crc >> 8) ^ crc32tab[(crc ^ (uint64_t)key[x]) & 0xff]; + + + #ifdef __APPLE__ + return ((~crc) >> 16) % 64; + #else + return ((~crc) >> 16) % 1024; + #endif +} + //redefinition CouchbaseRequest::CouchbaseRequest() : NonreflectableMessage() { @@ -116,12 +167,96 @@ bool CouchbaseRequest::SelectBucket(const butil::StringPiece &bucket_name) { return true; } +//HelloRequest sends a Hello request to the Couchbase server, which specifies the client features and capabilities. +// This is typically the first request sent after connecting to the server. +// It includes the agent name and a randomly generated connection ID in JSON format. +bool CouchbaseRequest::HelloRequest(){ + + std::string agent = "brpc/1.0.0 ("; + #ifdef __APPLE__ + agent += "Darwin/"; + #elif defined(__linux__) + agent += "Linux/"; + #else + agent += "UnknownOS/"; + #endif + #if defined(__x86_64__) + agent += "x86_64"; + #elif defined(__aarch64__) + agent += "arm64"; + #else + agent += "unknown"; + #endif + agent += ";bssl/0x1010107f)"; + + // Generate a random 33-byte connection id as hex string (66 hex chars) + unsigned char raw_id[33]; + FILE* urandom = fopen("/dev/urandom", "rb"); + if (!urandom || fread(raw_id, 1, 33, urandom) != 33) { + if (urandom) fclose(urandom); + std::cout << "Failed to generate random connection id" << std::endl; + return false; + } + fclose(urandom); + char hex_id[67] = {0}; + for (int i = 0; i < 33; ++i) { + sprintf(hex_id + i * 2, "%02x", raw_id[i]); + } + + // Format key as JSON: {"a":"agent","i":"hex_id"} + std::string key = std::string("{\"a\":\"") + agent + "\",\"i\":\"" + hex_id + "\"}"; + + const uint16_t key_len = key.size(); + uint16_t features[] = { + butil::HostToNet16(0x0001), // Datatype + butil::HostToNet16(0x0006), // XError + butil::HostToNet16(0x0007), // SelectBucket + butil::HostToNet16(0x000b), // Snappy + butil::HostToNet16(0x0012) // Collections + }; + + const uint32_t value_len = sizeof(features); + const uint32_t total_body_len = key_len + value_len; + + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_HELLO_SELECT_FEATURES, + butil::HostToNet16(key_len), // key length + 0, // extras length + policy::CB_BINARY_RAW_BYTES, // data type + 0, // vbucket id + butil::HostToNet32(total_body_len), // total body length + 0, // opaque + 0 // cas value + }; + + if (_buf.append(&header, sizeof(header))) { + std::cout << "Failed to append Hello header to buffer" << std::endl; + return false; + } + if (_buf.append(key.data(), key_len)) { + std::cout << "Failed to append Hello JSON key to buffer" << std::endl; + return false; + } + if (_buf.append(reinterpret_cast(features), value_len)) { + std::cout << "Failed to append Hello features to buffer" << std::endl; + return false; + } + ++_pipelined_count; + return true; +} + bool CouchbaseRequest::Authenticate(const butil::StringPiece &username, const butil::StringPiece &password) { if (username.empty() || password.empty()) { LOG(ERROR) << "Empty username or password"; return false; } + //insert the features to get enabled, calling function HelloRequest() will do this. + if (!HelloRequest()) { + LOG(ERROR) << "Failed to send HelloRequest for authentication"; + return false; + } // Construct the request header constexpr char kPlainAuthCommand[] = "PLAIN"; constexpr char kPadding[1] = {'\0'}; @@ -352,20 +487,26 @@ const char* CouchbaseResponse::status_str(Status st) { // MUST have key. // MUST NOT have value. bool CouchbaseRequest::GetOrDelete(uint8_t command, const butil::StringPiece& key) { + //Collection ID + uint8_t collection_id = 0; + uint16_t VBucket_id = hash_crc32(key.data(), key.size()); const policy::CouchbaseRequestHeader header = { policy::CB_MAGIC_REQUEST, command, - butil::HostToNet16(key.size()), - 0, - policy::CB_BINARY_RAW_BYTES, - 0, - butil::HostToNet32(key.size()), + butil::HostToNet16(key.size() + 1), //collection id is part of key, so adding it in the key length + 0, // extras length + policy::CB_BINARY_RAW_BYTES, // data type + butil::HostToNet16(VBucket_id), + butil::HostToNet32(key.size()+sizeof(collection_id)), // total body length includes key and collection id 0, 0 }; if (_buf.append(&header, sizeof(header))) { return false; } + if( _buf.append(&collection_id, sizeof(collection_id))) { + return false; + } if (_buf.append(key.data(), key.size())) { return false; } @@ -545,20 +686,27 @@ const size_t STORE_EXTRAS = sizeof(StoreHeaderWithExtras) - bool CouchbaseRequest::Store( uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value) { + //add collection id + // uint16_t collection_id = 0x00; + uint8_t collection_id = 0x00; + uint16_t vBucket_id = hash_crc32(key.data(),key.size()); StoreHeaderWithExtras header_with_extras = {{ policy::CB_MAGIC_REQUEST, command, - butil::HostToNet16(key.size()), + butil::HostToNet16(key.size()+1), //collection id is part of key, so including it in key length. STORE_EXTRAS, policy::CB_JSON, - 0, - butil::HostToNet32(STORE_EXTRAS + key.size() + value.size()), + butil::HostToNet16(vBucket_id), + butil::HostToNet32(STORE_EXTRAS +sizeof(uint8_t) + key.size() + value.size()), // total body length 0, butil::HostToNet64(cas_value) }, butil::HostToNet32(flags), butil::HostToNet32(exptime)}; if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { return false; } + if( _buf.append(&collection_id, sizeof(uint8_t))) { + return false; + } if (_buf.append(key.data(), key.size())) { return false; } @@ -666,12 +814,8 @@ bool CouchbaseRequest::GetCollectionId(const butil::StringPiece& scope_name, bool CouchbaseRequest::GetFromCollection(const butil::StringPiece& key, const butil::StringPiece& scope_name, const butil::StringPiece& collection_name) { - // First, we need to construct a collection-aware key - // For simplicity, we'll use the format: scope.collection::key - std::string scoped_key = scope_name.as_string() + "." + - collection_name.as_string() + "::" + - key.as_string(); - + // Construct a collection-aware key: collection::key + std::string scoped_key = collection_name.as_string() + "::" + key.as_string(); return GetOrDelete(policy::CB_BINARY_GET, butil::StringPiece(scoped_key)); } @@ -680,11 +824,8 @@ bool CouchbaseRequest::SetToCollection(const butil::StringPiece& key, uint32_t flags, uint32_t exptime, uint64_t cas_value, const butil::StringPiece& scope_name, const butil::StringPiece& collection_name) { - // Construct a collection-aware key - std::string scoped_key = scope_name.as_string() + "." + - collection_name.as_string() + "::" + - key.as_string(); - + // Construct a collection-aware key: collection::key + std::string scoped_key = collection_name.as_string() + "::" + key.as_string(); return Store(policy::CB_BINARY_SET, butil::StringPiece(scoped_key), value, flags, exptime, cas_value); } diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index aecd6b7bfd..3ff89ab772 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -10,6 +10,7 @@ #include "brpc/pb_compat.h" namespace brpc { + static uint32_t hash_crc32(const char *key, size_t key_length); class CouchbaseRequest : public NonreflectableMessage { private: int _pipelined_count; @@ -37,6 +38,7 @@ namespace brpc { bool SelectBucket(const butil::StringPiece &bucket_name); bool Authenticate(const butil::StringPiece &username, const butil::StringPiece &password); + bool HelloRequest(); // Collection Management Methods bool GetCollectionsManifest(); From d1497c155c1631107201c821fc626bbf028f8684 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 31 Jul 2025 10:42:25 +0530 Subject: [PATCH 06/49] Added test cases for threaded get and add functions --- example/couchbase_c++/couchbase_client.cpp | 200 ++++++++++++++++++--- 1 file changed, 175 insertions(+), 25 deletions(-) diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index a44ef4e53c..b87c782f97 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -6,18 +6,71 @@ #include #include #include + DEFINE_string(server, "127.0.0.1:11210", "IP Address of server"); -DEFINE_string(connection_type, "pooled", "Connection type. Available values: single, pooled, short"); -DEFINE_string(username, "Administrator", "Couchbase username"); -DEFINE_string(password, "password", "Couchbase password"); -DEFINE_string(bucket_name, "testing", "Couchbase bucket name"); +DEFINE_string(connection_type, "single", "Connection type. Available values: single, pooled, short"); +// DEFINE_string(username, "Administrator", "Couchbase username"); +// DEFINE_string(password, "password", "Couchbase password"); +// DEFINE_string(bucket_name, "testing", "Couchbase bucket name"); +DEFINE_bool(use_bthread, true, "Use bthread to send requests"); DEFINE_string(load_balancer, "", "The algorithm for load balancing"); DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); DEFINE_bool(dont_fail, false, "Print fatal when some call failed"); DEFINE_int32(exptime, 0, "The to-be-got data will be expired after so many seconds"); + +uint32_t batch_size = 10; + bvar::LatencyRecorder g_latency_recorder("client"); bvar::Adder g_error_count("client_error_count"); butil::static_atomic g_sender_count = BUTIL_STATIC_ATOMIC_INIT(0); + +static void* sender(void* arg) { + google::protobuf::RpcChannel* channel = + static_cast(arg); + const int base_index = g_sender_count.fetch_add(1, butil::memory_order_relaxed); + + std::string value; + std::string key = "test_brpc_"; + // Example batch size, can be parameterized + + brpc::CouchbaseRequest request; + for (int i = 0; i < batch_size; ++i) { + CHECK(request.Get(butil::string_printf("%s%d", key.c_str(), i))); + } + while (!brpc::IsAskedToQuit()) { + // We will receive response synchronously, safe to put variables + // on stack. + brpc::CouchbaseResponse response; + brpc::Controller cntl; + + // Because `done'(last parameter) is NULL, this function waits until + // the response comes back or error occurs(including timedout). + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + const int64_t elp = cntl.latency_us(); + if (!cntl.Failed()) { + int i = 0; + g_latency_recorder << cntl.latency_us(); + for (i = 0; i < batch_size; ++i) { + uint32_t flags; + if (!response.PopGet(&value, &flags, NULL)) { + LOG(INFO) << "Fail to GET the key, " << response.LastError(); + std::cout<<"thread id "<< bthread_self() << " failed to get key: " << butil::string_printf("%s%d", key.c_str(), i) << std::endl; + break; + } + std::cout<<"thread id "<< bthread_self() <<"Key: " << butil::string_printf("%s%d", key.c_str(), i) << ", Value: " << value << std::endl; + } + bthread_usleep(5000000); // Sleep for 50ms before exiting + return NULL; + + } else { + g_error_count << 1; + bthread_usleep(5000000); + return NULL; + } + } + return NULL; +} + int main(){ brpc::Channel channel; @@ -41,7 +94,26 @@ int main(){ brpc::Controller cntl; brpc::CouchbaseRequest auth_request; brpc::CouchbaseResponse auth_response; - if(!auth_request.Authenticate(FLAGS_username.c_str(), FLAGS_password.c_str())) { + + // Ask username and password for authentication + std::string username; + std::string password; + while(username.empty() || password.empty()) { + std::cout << "Enter Couchbase username: "; + std::cin >> username; + if(username.empty()) { + std::cout << "Username cannot be empty. Please enter again." << std::endl; + continue; + } + std::cout << "Enter Couchbase password: "; + std::cin >> password; + if(password.empty()) { + std::cout << "Password cannot be empty. Please enter again." << std::endl; + continue; + } + } + + if(!auth_request.Authenticate(username.c_str(), password.c_str())) { LOG(ERROR) << "Fail to create authentication request"; return -1; } @@ -51,10 +123,22 @@ int main(){ return -1; } cntl.Reset(); + + std::cout << "Authentication successful, proceeding with Couchbase operations..." << std::endl; + + // Select bucket + std::string bucket_name; + while(bucket_name.empty()) { + std::cout << "Enter Couchbase bucket name: "; + std::cin >> bucket_name; + if(bucket_name.empty()) { + std::cout << "Bucket name cannot be empty. Please enter again." << std::endl; + continue; + } + } brpc::CouchbaseRequest select_bucket_request; brpc::CouchbaseResponse select_bucket_response; - - if (!select_bucket_request.SelectBucket(FLAGS_bucket_name.c_str())) { + if (!select_bucket_request.SelectBucket(bucket_name.c_str())) { LOG(ERROR) << "Fail to SELECT bucket request"; return -1; } @@ -64,31 +148,97 @@ int main(){ return -1; } cntl.Reset(); - //print the content of the select_bucket_response variable - std::cout << "Select Bucket Response: " << select_bucket_response.raw_buffer().to_string() << std::endl; - cntl.Reset(); - if (!select_bucket_request.Add("document-key3", - "{\"hello\":\"world\"}", // <-- JSON string - 0, FLAGS_exptime, 0)) { // 0x01 = JSON datatype - LOG(ERROR) << "Fail to ADD request"; - return -1; -} - // request.Set("dummy", "value", 0, 1000, 0); // Will fail, but triggers SASL authentication - channel.CallMethod(NULL, &cntl, &select_bucket_request, &select_bucket_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + + if (select_bucket_response.LastError().empty()) { + std::cout << "Bucket selected successfully: " << bucket_name << std::endl; + } else { + std::cout << "Failed to select bucket: " << select_bucket_response.LastError() << std::endl; return -1; } - cntl.Reset(); - if (!select_bucket_request.Get(butil::string_printf("%s", "document-key3"))) { - LOG(ERROR) << "Fail to GET request"; - return -1; + + //enter batch size + std::cout << "Enter batch size for operations (e.g., 10): "; + std::cin >> batch_size; + + // Add operation + brpc::CouchbaseRequest add_request; + brpc::CouchbaseResponse add_response; + + for(int i = 0;i bids; + std::vector pids; + if (!FLAGS_use_bthread) { + pids.resize( batch_size); + for (int i = 0; i < batch_size; ++i) { + if (pthread_create(&pids[i], NULL, sender, &channel) != 0) { + LOG(ERROR) << "Fail to create pthread"; + return -1; + } + } + } else { + bids.resize( batch_size); + for (int i = 0; i < batch_size; ++i) { + if (bthread_start_background( + &bids[i], NULL, sender, &channel) != 0) { + LOG(ERROR) << "Fail to create bthread"; + return -1; + } + } + } + + LOG(INFO) << "couchbase_client is going to quit"; + for (int i = 0; i < batch_size; ++i) { + if (!FLAGS_use_bthread) { + pthread_join(pids[i], NULL); + } else { + bthread_join(bids[i], NULL); + } + } + // // Get operation + // brpc::CouchbaseRequest get_request; + // brpc::CouchbaseResponse get_response; + // if (!get_request.Get(butil::string_printf("%s", "test_brpc"))) { + // LOG(ERROR) << "Fail to GET request"; + // return -1; + // } + // channel.CallMethod(NULL, &cntl, &get_request, &get_response, NULL); + // if (cntl.Failed()) { + // LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + // return -1; + // } + // std::string value; + // uint32_t flags = 0; + // uint64_t cas = 0; + // if (get_response.PopGet(&value, &flags, &cas)) { + // std::cout << "GET value: " << value << std::endl; + // std::cout << "Flags: " << flags << std::endl; + // std::cout << "CAS: " << cas << std::endl; + // } else { + // std::cout << "Raw response (hex): "; + // for (char c : get_response.raw_buffer().to_string()) { + // printf("%02x ", static_cast(c)); + // } + // std::cout << std::endl; + // } return 0; } From aae6a3d121b715d783f0ff47e3178fb1e0bcd015 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 6 Aug 2025 15:36:54 +0530 Subject: [PATCH 07/49] Added Error Handling code and made upsert and delete examples --- example/couchbase_c++/couchbase_client.cpp | 683 +++++++++++++++++---- src/brpc/couchbase.cpp | 185 +++++- src/brpc/couchbase.h | 59 +- 3 files changed, 791 insertions(+), 136 deletions(-) diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index b87c782f97..6768cbb803 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -6,6 +6,14 @@ #include #include #include +#include +#include +#include + +// ANSI color codes for console output +#define GREEN "\033[32m" +#define RED "\033[31m" +#define RESET "\033[0m" DEFINE_string(server, "127.0.0.1:11210", "IP Address of server"); DEFINE_string(connection_type, "single", "Connection type. Available values: single, pooled, short"); @@ -24,54 +32,87 @@ bvar::LatencyRecorder g_latency_recorder("client"); bvar::Adder g_error_count("client_error_count"); butil::static_atomic g_sender_count = BUTIL_STATIC_ATOMIC_INIT(0); -static void* sender(void* arg) { - google::protobuf::RpcChannel* channel = - static_cast(arg); - const int base_index = g_sender_count.fetch_add(1, butil::memory_order_relaxed); +// Global variables for timing statistics +std::vector thread_response_times; +std::mutex timing_mutex; - std::string value; - std::string key = "test_brpc_"; - // Example batch size, can be parameterized - - brpc::CouchbaseRequest request; - for (int i = 0; i < batch_size; ++i) { - CHECK(request.Get(butil::string_printf("%s%d", key.c_str(), i))); - } - while (!brpc::IsAskedToQuit()) { - // We will receive response synchronously, safe to put variables - // on stack. - brpc::CouchbaseResponse response; - brpc::Controller cntl; - - // Because `done'(last parameter) is NULL, this function waits until - // the response comes back or error occurs(including timedout). - channel->CallMethod(NULL, &cntl, &request, &response, NULL); - const int64_t elp = cntl.latency_us(); - if (!cntl.Failed()) { - int i = 0; - g_latency_recorder << cntl.latency_us(); - for (i = 0; i < batch_size; ++i) { - uint32_t flags; - if (!response.PopGet(&value, &flags, NULL)) { - LOG(INFO) << "Fail to GET the key, " << response.LastError(); - std::cout<<"thread id "<< bthread_self() << " failed to get key: " << butil::string_printf("%s%d", key.c_str(), i) << std::endl; - break; - } - std::cout<<"thread id "<< bthread_self() <<"Key: " << butil::string_printf("%s%d", key.c_str(), i) << ", Value: " << value << std::endl; - } - bthread_usleep(5000000); // Sleep for 50ms before exiting - return NULL; +// static void* sender(void* arg) { +// google::protobuf::RpcChannel* channel = +// static_cast(arg); +// const int base_index = g_sender_count.fetch_add(1, butil::memory_order_relaxed); + +// std::string value; +// std::string key = "test_brpc_"; +// // Example batch size, can be parameterized + +// brpc::CouchbaseRequest request; +// for (int i = 0; i < batch_size; ++i) { +// CHECK(request.Get(butil::string_printf("%s%d", key.c_str(), i))); +// } + +// // Start timing for this thread +// auto start_time = std::chrono::high_resolution_clock::now(); + +// while (!brpc::IsAskedToQuit()) { +// // We will receive response synchronously, safe to put variables +// // on stack. +// brpc::CouchbaseResponse response; +// brpc::Controller cntl; + +// // Because `done'(last parameter) is NULL, this function waits until +// // the response comes back or error occurs(including timedout). +// channel->CallMethod(NULL, &cntl, &request, &response, NULL); +// const int64_t elp = cntl.latency_us(); +// if (!cntl.Failed()) { +// int i = 0; +// g_latency_recorder << cntl.latency_us(); +// for (i = 0; i < batch_size; ++i) { +// uint32_t flags; +// if (!response.PopGet(&value, &flags, NULL)) { +// LOG(INFO) << "Fail to GET the key, " << response.LastError(); +// std::cout<<"thread id "<< bthread_self() << " failed to get key: " << butil::string_printf("%s%d", key.c_str(), i) << std::endl; +// break; +// } +// std::cout<<"thread id "<< bthread_self() <<"Key: " << butil::string_printf("%s%d", key.c_str(), i) << ", Value: " << value << std::endl; +// } - } else { - g_error_count << 1; - bthread_usleep(5000000); - return NULL; - } - } - return NULL; -} +// // End timing and calculate response time for this thread +// auto end_time = std::chrono::high_resolution_clock::now(); +// auto duration = std::chrono::duration_cast(end_time - start_time); +// double response_time_ms = duration.count() / 1000.0; // Convert to milliseconds + +// std::cout << "Thread " << bthread_self() << " GET request took: " << response_time_ms << " ms" << std::endl; + +// // Store the response time in thread-safe manner +// { +// std::lock_guard lock(timing_mutex); +// thread_response_times.push_back(response_time_ms); +// } + +// bthread_usleep(5000000); // Sleep for 50ms before exiting +// return NULL; + +// } else { +// g_error_count << 1; + +// // End timing even for failed requests +// auto end_time = std::chrono::high_resolution_clock::now(); +// auto duration = std::chrono::duration_cast(end_time - start_time); +// double response_time_ms = duration.count() / 1000.0; + +// std::cout << "Thread " << bthread_self() << " GET request failed after: " << response_time_ms << " ms" << std::endl; + +// bthread_usleep(5000000); +// return NULL; +// } +// } +// return NULL; +// } int main(){ + + std::vector> operation_times; + brpc::Channel channel; // Initialize the channel, NULL means using default options. @@ -85,7 +126,7 @@ int main(){ LOG(ERROR) << "Fail to initialize channel"; return -1; } - std::cout<<"Channel initialized successfully"<(auth_end_time - auth_start_time); + double auth_time_ms = auth_duration.count() / 1000.0; + cntl.Reset(); - std::cout << "Authentication successful, proceeding with Couchbase operations..." << std::endl; + std::cout << GREEN << "Authentication successful (took " << auth_time_ms << " ms), proceeding with Couchbase operations..." << RESET << std::endl; // Select bucket std::string bucket_name; @@ -142,103 +199,517 @@ int main(){ LOG(ERROR) << "Fail to SELECT bucket request"; return -1; } + + // Start timing for bucket selection + auto bucket_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &select_bucket_request, &select_bucket_response, NULL); if (cntl.Failed()) { LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); return -1; } + + // Check bucket selection response status + if (!select_bucket_response.LastError().empty()) { + LOG(ERROR) << "Bucket selection failed: " << select_bucket_response.LastError(); + return -1; + } + + // End timing for bucket selection + auto bucket_end_time = std::chrono::high_resolution_clock::now(); + auto bucket_duration = std::chrono::duration_cast(bucket_end_time - bucket_start_time); + double bucket_time_ms = bucket_duration.count() / 1000.0; + cntl.Reset(); if (select_bucket_response.LastError().empty()) { - std::cout << "Bucket selected successfully: " << bucket_name << std::endl; + std::cout << GREEN << "Bucket selected successfully: " << bucket_name << " (took " << bucket_time_ms << " ms)" << RESET << std::endl; } else { - std::cout << "Failed to select bucket: " << select_bucket_response.LastError() << std::endl; + std::cout << RED << "Failed to select bucket: " << select_bucket_response.LastError() << RESET << std::endl; return -1; } //enter batch size - std::cout << "Enter batch size for operations (e.g., 10): "; - std::cin >> batch_size; + // std::cout << "Enter batch size for operations (e.g., 10): "; + // std::cin >> batch_size; + + // // Add operation + // brpc::CouchbaseRequest add_request; + // brpc::CouchbaseResponse add_response; + + // for(int i = 0;i(add_end_time - add_start_time); + // double add_time_ms = add_duration.count() / 1000.0; + + // std::cout << "Added " << batch_size << " documents (took " << add_time_ms << " ms)" << std::endl; + + // cntl.Reset(); + + // std::vector bids; + // std::vector pids; + // if (!FLAGS_use_bthread) { + // pids.resize( batch_size); + // for (int i = 0; i < batch_size; ++i) { + // if (pthread_create(&pids[i], NULL, sender, &channel) != 0) { + // LOG(ERROR) << "Fail to create pthread"; + // return -1; + // } + // } + // } else { + // bids.resize( batch_size); + // for (int i = 0; i < batch_size; ++i) { + // if (bthread_start_background( + // &bids[i], NULL, sender, &channel) != 0) { + // LOG(ERROR) << "Fail to create bthread"; + // return -1; + // } + // } + // } + + // LOG(INFO) << "couchbase_client is going to quit"; + // for (int i = 0; i < batch_size; ++i) { + // if (!FLAGS_use_bthread) { + // pthread_join(pids[i], NULL); + // } else { + // bthread_join(bids[i], NULL); + // } + // } + + // // Calculate and print average response time + // if (!thread_response_times.empty()) { + // double total_time = 0.0; + // for (double time : thread_response_times) { + // total_time += time; + // } + // double average_response_time = total_time / thread_response_times.size(); + + // std::cout << "\n=== Performance Summary ===" << std::endl; + // std::cout << "Authentication time: " << auth_time_ms << " ms" << std::endl; + // std::cout << "Bucket selection time: " << bucket_time_ms << " ms" << std::endl; + // std::cout << "ADD operations time: " << add_time_ms << " ms" << std::endl; + // std::cout << "Total threads: " << thread_response_times.size() << std::endl; + // std::cout << "Average GET response time: " << average_response_time << " ms" << std::endl; + // std::cout << "Total time for all GET requests: " << total_time << " ms" << std::endl; + // std::cout << "Total authorization time: " << (auth_time_ms + bucket_time_ms) << " ms" << std::endl; + // std::cout << "=========================" << std::endl; + // } else { + // std::cout << "\n=== Performance Summary ===" << std::endl; + // std::cout << "Authentication time: " << auth_time_ms << " ms" << std::endl; + // std::cout << "Bucket selection time: " << bucket_time_ms << " ms" << std::endl; + // std::cout << "ADD operations time: " << add_time_ms << " ms" << std::endl; + // std::cout << "Total authorization time: " << (auth_time_ms + bucket_time_ms) << " ms" << std::endl; + // std::cout << "No successful GET requests completed." << std::endl; + // std::cout << "=========================" << std::endl; + // } // Add operation brpc::CouchbaseRequest add_request; brpc::CouchbaseResponse add_response; + if (!add_request.Add(butil::string_printf("user::test_brpc_binprot"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + auto add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); + auto add_end_time = std::chrono::high_resolution_clock::now(); + auto add_duration = std::chrono::duration_cast(add_end_time - add_start_time); + operation_times.push_back({"Add user data (first attempt) in microsecond", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + else{ + // Check ADD operation response status + uint64_t cas_value; + if (add_response.PopAdd(&cas_value)) { + std::cout << GREEN << "ADD operation successful, CAS: " << cas_value << RESET << std::endl; + } else { + std::cout << RED << add_response.LastError() << RESET << std::endl; + } + } + + cntl.Reset(); + + if (!add_request.Add(butil::string_printf("user::test_brpc_binprot"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast(add_end_time - add_start_time); + operation_times.push_back({"Add user data (Second attempt - expected failure) in microsecond ", add_duration.count()}); + + // Check second ADD operation response status (should fail with key exists) + if (cntl.Failed()) { + LOG(ERROR) << "RPC call failed: " << cntl.ErrorText(); + } else { + uint64_t cas_value; + if (add_response.PopAdd(&cas_value)) { + std::cout << GREEN << "Second ADD operation unexpectedly successful, CAS: " << cas_value << RESET << std::endl; + } else { + std::cout << RED << "Second ADD operation failed as expected: " << add_response.LastError() << RESET << std::endl; + } + } + + cntl.Reset(); + // Get operation + brpc::CouchbaseRequest get_request; + brpc::CouchbaseResponse get_response; + if (!get_request.Get(butil::string_printf("%s", "user::test_brpc_binprot"))) { + LOG(ERROR) << "Fail to GET request"; + return -1; + } + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &get_request, &get_response, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast(add_end_time - add_start_time); + operation_times.push_back({"Get user data in microsecond ", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check GET operation response status + std::string value; + uint32_t flags = 0; + uint64_t cas = 0; + if (get_response.PopGet(&value, &flags, &cas)) { + std::cout << GREEN << "GET operation successful" << RESET << std::endl; + std::cout << "GET value: " << value << std::endl; + std::cout << "Flags: " << flags << std::endl; + std::cout << "CAS: " << cas << std::endl; + } else { + std::cout << RED << "GET operation failed: " << get_response.LastError() << RESET << std::endl; + std::cout << "Raw response (hex): "; + for (char c : get_response.raw_buffer().to_string()) { + printf("%02x ", static_cast(c)); + } + std::cout << std::endl; + } + cntl.Reset(); + + // Create a new request for binprot item1 + brpc::CouchbaseRequest add_request1; + brpc::CouchbaseResponse add_response1; + if (!add_request1.Add(butil::string_printf("binprot_item1"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + std::cout << "Sending ADD request for binprot_item1, pipelined count: " << add_request1.pipelined_count() << std::endl; + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request1, &add_response1, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast(add_end_time - add_start_time); + operation_times.push_back({"Add binprot item1 in microsecond", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check ADD binprot item1 response status + uint64_t cas_value; + if (add_response1.PopAdd(&cas_value)) { + std::cout << GREEN << "ADD binprot item1 successful, CAS: " << cas_value << RESET << std::endl; + } else { + std::cout << RED << "ADD binprot item1 failed: " << add_response1.LastError() << RESET << std::endl; + } + cntl.Reset(); + + // Create a new request for binprot item2 + brpc::CouchbaseRequest add_request2; + brpc::CouchbaseResponse add_response2; + if (!add_request2.Add(butil::string_printf("binprot_item2"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + std::cout << "Sending ADD request for binprot_item2, pipelined count: " << add_request2.pipelined_count() << std::endl; + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request2, &add_response2, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast(add_end_time - add_start_time); + operation_times.push_back({"Add binprot item2 in microsecond", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check ADD binprot item2 response status + if (add_response2.PopAdd(&cas_value)) { + std::cout << GREEN << "ADD binprot item2 successful, CAS: " << cas_value << RESET << std::endl; + } else { + std::cout << RED << "ADD binprot item2 failed: " << add_response2.LastError() << RESET << std::endl; + } + cntl.Reset(); + + // Create a new request for binprot item3 + brpc::CouchbaseRequest add_request3; + brpc::CouchbaseResponse add_response3; + if (!add_request3.Add(butil::string_printf("binprot_item3"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + std::cout << "Sending ADD request for binprot_item3, pipelined count: " << add_request3.pipelined_count() << std::endl; + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request3, &add_response3, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast(add_end_time - add_start_time); + operation_times.push_back({"Add binprot item3 in microsecond", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check ADD binprot item3 response status + if (add_response3.PopAdd(&cas_value)) { + std::cout << GREEN << "ADD binprot item3 successful, CAS: " << cas_value << RESET << std::endl; + } else { + std::cout << RED << "ADD binprot item3 failed: " << add_response3.LastError() << RESET << std::endl; + } + cntl.Reset(); - for(int i = 0;i bids; - std::vector pids; - if (!FLAGS_use_bthread) { - pids.resize( batch_size); - for (int i = 0; i < batch_size; ++i) { - if (pthread_create(&pids[i], NULL, sender, &channel) != 0) { - LOG(ERROR) << "Fail to create pthread"; - return -1; - } + // Example of mixed pipeline operations (GET operations on existing and non-existing keys) + std::cout << "\n=== Testing Mixed Pipeline Operations ===" << std::endl; + brpc::CouchbaseRequest mixed_pipeline_request; + brpc::CouchbaseResponse mixed_pipeline_response; + + // Add some GET operations to the pipeline - some will succeed, some will fail + std::vector keys_to_get = { + "user::test_brpc_binprot", // Should exist + "binprot_item1", // Should exist + "nonexistent_key1", // Should fail + "binprot_item2", // Should exist + "nonexistent_key2", // Should fail + "binprot_item3" // Should exist + }; + + for (const auto& key : keys_to_get) { + if (!mixed_pipeline_request.Get(key)) { + LOG(ERROR) << "Fail to add GET request for key: " << key; + return -1; } - } else { - bids.resize( batch_size); - for (int i = 0; i < batch_size; ++i) { - if (bthread_start_background( - &bids[i], NULL, sender, &channel) != 0) { - LOG(ERROR) << "Fail to create bthread"; - return -1; - } + } + + std::cout << "Sending mixed pipeline GET request, pipelined count: " << mixed_pipeline_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &mixed_pipeline_request, &mixed_pipeline_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Process each GET response + int successful_gets = 0; + int failed_gets = 0; + + for (size_t i = 0; i < keys_to_get.size(); ++i) { + std::string value; + uint32_t flags; + uint64_t cas; + + if (mixed_pipeline_response.PopGet(&value, &flags, &cas)) { + std::cout << GREEN << "GET '" << keys_to_get[i] << "' successful - Value length: " << value.length() + << ", Flags: " << flags << ", CAS: " << cas << RESET << std::endl; + successful_gets++; + } else { + std::cout << RED << "GET '" << keys_to_get[i] << "' failed: " << mixed_pipeline_response.LastError() << RESET << std::endl; + failed_gets++; } } + + std::cout << "Mixed pipeline summary: " << successful_gets << " successful, " + << failed_gets << " failed GET operations" << std::endl; + + + cntl.Reset(); + + //perform an upsert on the existing key + brpc::CouchbaseRequest upsert_request; + brpc::CouchbaseResponse upsert_response; + if (!upsert_request.Upsert(butil::string_printf("user::test_brpc_binprot"), R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})", 0, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to add UPSERT request for key: user::test_brpc_binprot"; + return -1; + } + + channel.CallMethod(NULL, &cntl, &upsert_request, &upsert_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + // Check UPSERT operation response status + if (upsert_response.PopUpsert(&cas_value)) { + std::cout << GREEN << "UPSERT operation successful when the document exists in the server, CAS: " << cas_value << RESET << std::endl; + } else { + std::cout << RED << "UPSERT operation failed, when the document does exists in the server: " << upsert_response.LastError() << RESET << std::endl; + } - LOG(INFO) << "couchbase_client is going to quit"; - for (int i = 0; i < batch_size; ++i) { - if (!FLAGS_use_bthread) { - pthread_join(pids[i], NULL); + cntl.Reset(); + //do the upsert operation + brpc::CouchbaseRequest upsert_request_new_doc; + brpc::CouchbaseResponse upsert_response_new_doc; + if (!upsert_request_new_doc.Upsert(butil::string_printf("user::test_brpc_new_upsert"), R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})", 0, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to add UPSERT request for key: user::test_brpc_new_upsert"; + return -1; + } + + channel.CallMethod(NULL, &cntl, &upsert_request_new_doc, &upsert_response_new_doc, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + // Check UPSERT operation response status + if (upsert_response_new_doc.PopUpsert(&cas_value)) { + std::cout << GREEN << "UPSERT operation successful when the document doesn't exists in the server, CAS: " << cas_value << RESET << std::endl; + } else { + std::cout << RED << "UPSERT operation failed when document does not exists in the server: " << upsert_response_new_doc.LastError() << RESET << std::endl; + } + + cntl.Reset(); + + //check the upserted data + brpc::CouchbaseRequest check_upsert_request; + brpc::CouchbaseResponse check_upsert_response; + if (!check_upsert_request.Get(butil::string_printf("user::test_brpc_new_upsert"))) { + LOG(ERROR) << "Fail to GET request for key: user::test_brpc_new_upsert"; + return -1; + } + + std::cout << "Sending GET request for user::test_brpc_new_upsert, pipelined count: " << check_upsert_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &check_upsert_request, &check_upsert_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check GET operation response status + if (check_upsert_response.PopGet(&value, &flags, &cas)) { + std::cout << GREEN << "GET after UPSERT operation successful - Value: " << value + << ", Flags: " << flags << ", CAS: " << cas << RESET << std::endl; + } else { + std::cout << RED << "GET after UPSERT operation failed: " << check_upsert_response.LastError() << RESET << std::endl; + } + + cntl.Reset(); + + //delete the Non-existent Key + brpc::CouchbaseRequest delete_request; + brpc::CouchbaseResponse delete_response; + if (!delete_request.Delete(butil::string_printf("Nonexistent_key"))) { + LOG(ERROR) << "Fail to DELETE request for key: Nonexistent_key"; + return -1; + } + + std::cout << "Sending DELETE request for Nonexistent_key, pipelined count: " << delete_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &delete_request, &delete_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check DELETE operation response status + if (delete_response.PopDelete()) { + std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; + } else { + std::cout << RED << "DELETE operation failed: as expected" << delete_response.LastError() << RESET << std::endl; + } + + cntl.Reset(); + + //delete the existing key + brpc::CouchbaseRequest delete_existing_request; + brpc::CouchbaseResponse delete_existing_response; + if (!delete_existing_request.Delete(butil::string_printf("user::test_brpc_binprot"))) { + LOG(ERROR) << "Fail to DELETE request for key: user::test_brpc_binprot"; + return -1; + } + + std::cout << "Sending DELETE request for user::test_brpc_binprot, pipelined count: " << delete_existing_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &delete_existing_request, &delete_existing_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check DELETE operation response status + if (delete_existing_response.PopDelete()) { + std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; + } else { + std::cout << RED << "DELETE operation failed: " << delete_existing_response.LastError() << RESET << std::endl; + } + + cntl.Reset(); + + // Print operation times + long long total_time = 0; + for (const auto& op : operation_times) { + std::cout << std::left << std::setw(40) << op.first << ": "; + if (op.second >= 1000) { + std::cout << std::right << std::setw(8) << (op.second / 1000.0) << " ms" << std::endl; } else { - bthread_join(bids[i], NULL); + std::cout << std::right << std::setw(8) << op.second << " μs" << std::endl; } + total_time += op.second; } - // // Get operation - // brpc::CouchbaseRequest get_request; - // brpc::CouchbaseResponse get_response; - // if (!get_request.Get(butil::string_printf("%s", "test_brpc"))) { - // LOG(ERROR) << "Fail to GET request"; - // return -1; - // } - // channel.CallMethod(NULL, &cntl, &get_request, &get_response, NULL); - // if (cntl.Failed()) { - // LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - // return -1; - // } - // std::string value; - // uint32_t flags = 0; - // uint64_t cas = 0; - // if (get_response.PopGet(&value, &flags, &cas)) { - // std::cout << "GET value: " << value << std::endl; - // std::cout << "Flags: " << flags << std::endl; - // std::cout << "CAS: " << cas << std::endl; - // } else { - // std::cout << "Raw response (hex): "; - // for (char c : get_response.raw_buffer().to_string()) { - // printf("%02x ", static_cast(c)); - // } - // std::cout << std::endl; - // } return 0; } diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 5483970e30..3483bd27ef 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -460,29 +460,130 @@ const char* CouchbaseResponse::status_str(Status st) { case STATUS_SUCCESS: return "SUCCESS"; case STATUS_KEY_ENOENT: - return "The key does not exist"; + return "Key not found"; case STATUS_KEY_EEXISTS: - return "The key exists"; + return "Key already exists"; case STATUS_E2BIG: - return "Arg list is too long"; + return "Value too large"; case STATUS_EINVAL: - return "Invalid argument"; + return "Invalid arguments"; case STATUS_NOT_STORED: - return "Not stored"; + return "Item not stored"; case STATUS_DELTA_BADVAL: - return "Bad delta"; + return "Invalid delta value for increment/decrement"; + case STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER: + return "VBucket belongs to another server"; case STATUS_AUTH_ERROR: - return "authentication error"; + return "Authentication failed"; case STATUS_AUTH_CONTINUE: - return "authentication continue"; + return "Authentication continue"; + case STATUS_ERANGE: + return "Range error"; + case STATUS_ROLLBACK: + return "Rollback required"; + case STATUS_EACCESS: + return "Access denied"; + case STATUS_NOT_INITIALIZED: + return "Not initialized"; case STATUS_UNKNOWN_COMMAND: return "Unknown command"; case STATUS_ENOMEM: return "Out of memory"; + case STATUS_NOT_SUPPORTED: + return "Operation not supported"; + case STATUS_EINTERNAL: + return "Internal server error"; + case STATUS_EBUSY: + return "Server busy"; + case STATUS_ETMPFAIL: + return "Temporary failure"; + case STATUS_UNKNOWN_COLLECTION: + return "Unknown collection"; + case STATUS_NO_COLLECTIONS_MANIFEST: + return "No collections manifest"; + case STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST: + return "Cannot apply collections manifest"; + case STATUS_COLLECTIONS_MANIFEST_IS_AHEAD: + return "Collections manifest is ahead"; + case STATUS_UNKNOWN_SCOPE: + return "Unknown scope"; + case STATUS_DCP_STREAM_ID_INVALID: + return "Invalid DCP stream ID"; + case STATUS_DURABILITY_INVALID_LEVEL: + return "Invalid durability level"; + case STATUS_DURABILITY_IMPOSSIBLE: + return "Durability requirements impossible"; + case STATUS_SYNC_WRITE_IN_PROGRESS: + return "Synchronous write in progress"; + case STATUS_SYNC_WRITE_AMBIGUOUS: + return "Synchronous write result ambiguous"; + case STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS: + return "Synchronous write re-commit in progress"; + case STATUS_SUBDOC_PATH_NOT_FOUND: + return "Sub-document path not found"; + case STATUS_SUBDOC_PATH_MISMATCH: + return "Sub-document path mismatch"; + case STATUS_SUBDOC_PATH_EINVAL: + return "Invalid sub-document path"; + case STATUS_SUBDOC_PATH_E2BIG: + return "Sub-document path too deep"; + case STATUS_SUBDOC_DOC_E2DEEP: + return "Sub-document too deep"; + case STATUS_SUBDOC_VALUE_CANTINSERT: + return "Cannot insert sub-document value"; + case STATUS_SUBDOC_DOC_NOT_JSON: + return "Document is not JSON"; + case STATUS_SUBDOC_NUM_E2BIG: + return "Sub-document number too large"; + case STATUS_SUBDOC_DELTA_E2BIG: + return "Sub-document delta too large"; + case STATUS_SUBDOC_PATH_EEXISTS: + return "Sub-document path already exists"; + case STATUS_SUBDOC_VALUE_E2DEEP: + return "Sub-document value too deep"; + case STATUS_SUBDOC_INVALID_COMBO: + return "Invalid sub-document operation combination"; + case STATUS_SUBDOC_MULTI_PATH_FAILURE: + return "Sub-document multi-path operation failed"; + case STATUS_SUBDOC_SUCCESS_DELETED: + return "Sub-document operation succeeded on deleted document"; + case STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO: + return "Invalid extended attribute flag combination"; + case STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO: + return "Invalid extended attribute key combination"; + case STATUS_SUBDOC_XATTR_UNKNOWN_MACRO: + return "Unknown extended attribute macro"; + case STATUS_SUBDOC_XATTR_UNKNOWN_VATTR: + return "Unknown virtual extended attribute"; + case STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR: + return "Cannot modify virtual extended attribute"; + case STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED: + return "Sub-document multi-path operation failed on deleted document"; + case STATUS_SUBDOC_INVALID_XATTR_ORDER: + return "Invalid extended attribute order"; + case STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO: + return "Unknown virtual extended attribute macro"; + case STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS: + return "Can only revive deleted documents"; + case STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE: + return "Deleted document cannot have a value"; + case STATUS_XATTR_EINVAL: + return "Invalid extended attributes"; } return "Unknown status"; } +// Helper method to format error messages with status codes +std::string CouchbaseResponse::format_error_message(uint16_t status_code, const std::string& operation, const std::string& error_msg) { + if (error_msg.empty()) { + return butil::string_printf("%s failed with status 0x%02x (%s)", + operation.c_str(), status_code, status_str((Status)status_code)); + } else { + return butil::string_printf("%s failed with status 0x%02x (%s): %s", + operation.c_str(), status_code, status_str((Status)status_code), error_msg.c_str()); + } +} + // MUST NOT have extras. // MUST have key. // MUST NOT have value. @@ -606,8 +707,13 @@ bool CouchbaseResponse::PopGet( } _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - _err.clear(); - _buf.cutn(&_err, value_size); + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "GET operation", error_msg); + } else { + _err = format_error_message(header.status, "GET operation"); + } return false; } if (header.extras_length != 4u) { @@ -743,8 +849,13 @@ bool CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { - (int)header.key_length; if (header.status != (uint16_t)STATUS_SUCCESS) { _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - _err.clear(); - _buf.cutn(&_err, value_size); + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "STORE operation", error_msg); + } else { + _err = format_error_message(header.status, "STORE operation"); + } return false; } LOG_IF(ERROR, value_size != 0) << "STORE response must not have value, actually=" @@ -758,7 +869,7 @@ bool CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { return true; } -bool CouchbaseRequest::Set( +bool CouchbaseRequest::Upsert( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value) { return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value); @@ -819,7 +930,7 @@ bool CouchbaseRequest::GetFromCollection(const butil::StringPiece& key, return GetOrDelete(policy::CB_BINARY_GET, butil::StringPiece(scoped_key)); } -bool CouchbaseRequest::SetToCollection(const butil::StringPiece& key, +bool CouchbaseRequest::UpsertToCollection(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, const butil::StringPiece& scope_name, @@ -861,7 +972,7 @@ bool CouchbaseRequest::Prepend( return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, cas_value); } -bool CouchbaseResponse::PopSet(uint64_t* cas_value) { +bool CouchbaseResponse::PopUpsert(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_SET, cas_value); } bool CouchbaseResponse::PopAdd(uint64_t* cas_value) { @@ -896,9 +1007,14 @@ bool CouchbaseResponse::PopCollectionsManifest(std::string* manifest_json) { } if (header.status != 0) { _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - _err.clear(); int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; - _buf.cutn(&_err, value_size); + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "Collections manifest request", error_msg); + } else { + _err = format_error_message(header.status, "Collections manifest request"); + } return false; } @@ -935,9 +1051,14 @@ bool CouchbaseResponse::PopCollectionId(uint32_t* collection_id) { } if (header.status != 0) { _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - _err.clear(); int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; - _buf.cutn(&_err, value_size); + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "Collection ID request", error_msg); + } else { + _err = format_error_message(header.status, "Collection ID request"); + } return false; } @@ -965,9 +1086,9 @@ bool CouchbaseResponse::PopGetFromCollection(butil::IOBuf* value, uint32_t* flag return PopGet(value, flags, cas_value); } -bool CouchbaseResponse::PopSetToCollection(uint64_t* cas_value) { +bool CouchbaseResponse::PopUpsertToCollection(uint64_t* cas_value) { // Same implementation as PopSet, just aliased for clarity - return PopSet(cas_value); + return PopUpsert(cas_value); } struct IncrHeaderWithExtras { @@ -1070,8 +1191,13 @@ bool CouchbaseResponse::PopCounter( if (value_size < 0) { butil::string_printf(&_err, "value_size=%d is negative", value_size); } else { - _err.clear(); - _buf.cutn(&_err, value_size); + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "Counter operation", error_msg); + } else { + _err = format_error_message(header.status, "Counter operation"); + } } return false; } @@ -1163,6 +1289,10 @@ bool CouchbaseRequest::Version() { // MUST NOT have extras. // MUST NOT have key. // MUST have value. +bool CouchbaseResponse::PopTouch() { + return PopStore(policy::CB_BINARY_TOUCH, NULL); +} + bool CouchbaseResponse::PopVersion(std::string* version) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; @@ -1190,8 +1320,13 @@ bool CouchbaseResponse::PopVersion(std::string* version) { return false; } if (header.status != (uint16_t)STATUS_SUCCESS) { - _err.clear(); - _buf.cutn(&_err, value_size); + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "Version request", error_msg); + } else { + _err = format_error_message(header.status, "Version request"); + } return false; } if (version) { diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 3ff89ab772..d287a2aca9 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -53,9 +53,9 @@ namespace brpc { const butil::StringPiece& scope_name, const butil::StringPiece& collection_name); - bool Set(const butil::StringPiece& key, const butil::StringPiece& value, + bool Upsert(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value); - bool SetToCollection(const butil::StringPiece& key, const butil::StringPiece& value, + bool UpsertToCollection(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, const butil::StringPiece& scope_name, const butil::StringPiece& collection_name); @@ -129,10 +129,55 @@ namespace brpc { STATUS_EINVAL = 0x04, STATUS_NOT_STORED = 0x05, STATUS_DELTA_BADVAL = 0x06, + STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, STATUS_AUTH_ERROR = 0x20, STATUS_AUTH_CONTINUE = 0x21, + STATUS_ERANGE = 0x22, + STATUS_ROLLBACK = 0x23, + STATUS_EACCESS = 0x24, + STATUS_NOT_INITIALIZED = 0x25, STATUS_UNKNOWN_COMMAND = 0x81, - STATUS_ENOMEM = 0x82 + STATUS_ENOMEM = 0x82, + STATUS_NOT_SUPPORTED = 0x83, + STATUS_EINTERNAL = 0x84, + STATUS_EBUSY = 0x85, + STATUS_ETMPFAIL = 0x86, + STATUS_UNKNOWN_COLLECTION = 0x88, + STATUS_NO_COLLECTIONS_MANIFEST = 0x89, + STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, + STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, + STATUS_UNKNOWN_SCOPE = 0x8c, + STATUS_DCP_STREAM_ID_INVALID = 0x8d, + STATUS_DURABILITY_INVALID_LEVEL = 0xa0, + STATUS_DURABILITY_IMPOSSIBLE = 0xa1, + STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, + STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, + STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, + STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, + STATUS_SUBDOC_PATH_MISMATCH = 0xc1, + STATUS_SUBDOC_PATH_EINVAL = 0xc2, + STATUS_SUBDOC_PATH_E2BIG = 0xc3, + STATUS_SUBDOC_DOC_E2DEEP = 0xc4, + STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, + STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, + STATUS_SUBDOC_NUM_E2BIG = 0xc7, + STATUS_SUBDOC_DELTA_E2BIG = 0xc8, + STATUS_SUBDOC_PATH_EEXISTS = 0xc9, + STATUS_SUBDOC_VALUE_E2DEEP = 0xca, + STATUS_SUBDOC_INVALID_COMBO = 0xcb, + STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, + STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, + STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, + STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, + STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, + STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, + STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, + STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, + STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, + STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, + STATUS_XATTR_EINVAL = 0xe0 }; void MergeFrom(const CouchbaseResponse& from) override; @@ -151,12 +196,16 @@ namespace brpc { butil::IOBuf& raw_buffer() { return _buf; } const butil::IOBuf& raw_buffer() const { return _buf; } static const char* status_str(Status); + + // Helper method to format error messages with status codes + static std::string format_error_message(uint16_t status_code, const std::string& operation, const std::string& error_msg = ""); + // Add methods to handle response parsing void Swap(CouchbaseResponse* other); bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); bool PopGet(std::string* value, uint32_t* flags, uint64_t* cas_value); const std::string& LastError() const { return _err; } - bool PopSet(uint64_t* cas_value); + bool PopUpsert(uint64_t* cas_value); bool PopAdd(uint64_t* cas_value); bool PopReplace(uint64_t* cas_value); bool PopAppend(uint64_t* cas_value); @@ -168,7 +217,7 @@ namespace brpc { // Collection-aware response methods (same as regular but for documentation) bool PopGetFromCollection(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool PopSetToCollection(uint64_t* cas_value); + bool PopUpsertToCollection(uint64_t* cas_value); bool PopDelete(); bool PopFlush(); From 2ff17ff30b2d6f879a7bd9009db843e6abddd5f3 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 10 Sep 2025 18:14:11 +0530 Subject: [PATCH 08/49] added makefile for example/couchbase_c++ --- example/couchbase_c++/Makefile | 68 ++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 example/couchbase_c++/Makefile diff --git a/example/couchbase_c++/Makefile b/example/couchbase_c++/Makefile new file mode 100644 index 0000000000..84b31031be --- /dev/null +++ b/example/couchbase_c++/Makefile @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +BRPC_PATH = ../../ +include $(BRPC_PATH)/config.mk +CXXFLAGS+=$(CPPFLAGS) -std=c++17 -DNDEBUG -O2 -pipe -W -Wall -fPIC -fno-omit-frame-pointer +HDRS+=$(BRPC_PATH)/output/include +LIBS+=$(BRPC_PATH)/output/lib +HDRPATHS = $(addprefix -I, $(HDRS)) +LIBPATHS = $(addprefix -L, $(LIBS)) +COMMA=, +SOPATHS=$(addprefix -Wl$(COMMA)-rpath$(COMMA), $(LIBS)) + +SOURCES = $(wildcard *.cpp) +OBJS = $(addsuffix .o, $(basename $(SOURCES))) + +ifeq ($(SYSTEM),Darwin) + ifneq ("$(LINK_SO)", "") + STATIC_LINKINGS += -lbrpc + else + # *.a must be explicitly specified in clang + STATIC_LINKINGS += $(BRPC_PATH)/output/lib/libbrpc.a + endif + LINK_OPTIONS_SO = $^ $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) + LINK_OPTIONS = $^ $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) +else ifeq ($(SYSTEM),Linux) + STATIC_LINKINGS += -lbrpc + LINK_OPTIONS_SO = -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) + LINK_OPTIONS = -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) +endif + +.PHONY:all +all: couchbase_client + +.PHONY:clean +clean: + @echo "> Cleaning" + rm -rf couchbase_client $(OBJS) + +couchbase_client:$(OBJS) + @echo "> Linking $@" +ifneq ("$(LINK_SO)", "") + $(CXX) $(LIBPATHS) $(SOPATHS) $(LINK_OPTIONS_SO) -o $@ +else + $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ +endif + +%.o:%.cpp + @echo "> Compiling $@" + $(CXX) -c $(HDRPATHS) $(CXXFLAGS) $< -o $@ + +%.o:%.cc + @echo "> Compiling $@" + $(CXX) -c $(HDRPATHS) $(CXXFLAGS) $< -o $@ From 10964aaa190bb59eab5c9b755fdf5e035e722a11 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Fri, 19 Sep 2025 15:16:32 +0530 Subject: [PATCH 09/49] fixed bugs in couchbase header files --- src/brpc/couchbase.cpp | 2 +- src/brpc/couchbase.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 3483bd27ef..54a5857030 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -27,7 +27,7 @@ namespace brpc { -static uint32_t hash_crc32(const char *key, size_t key_length) +uint32_t CouchbaseRequest::hash_crc32(const char *key, size_t key_length) { static const uint32_t crc32tab[256] = { diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index d287a2aca9..cf23727fa9 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -10,7 +10,6 @@ #include "brpc/pb_compat.h" namespace brpc { - static uint32_t hash_crc32(const char *key, size_t key_length); class CouchbaseRequest : public NonreflectableMessage { private: int _pipelined_count; @@ -26,6 +25,7 @@ namespace brpc { bool Store(uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value); + uint32_t hash_crc32(const char *key, size_t key_length); public: CouchbaseRequest(); ~CouchbaseRequest() override; From 8bc4c594d1c6f8f45bd117093d5e500ce92a1c21 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Fri, 19 Sep 2025 15:47:21 +0530 Subject: [PATCH 10/49] Added License and formatted to google c++ norms --- example/couchbase_c++/couchbase_client.cpp | 1396 +++++++------- src/brpc/couchbase.cpp | 1927 ++++++++++---------- src/brpc/couchbase.h | 464 ++--- 3 files changed, 1982 insertions(+), 1805 deletions(-) diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index 6768cbb803..be305ed580 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -1,14 +1,32 @@ -#include -#include -#include -#include +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include -#include #include +#include +#include +#include +#include +#include + #include -#include #include +#include // ANSI color codes for console output #define GREEN "\033[32m" @@ -16,15 +34,17 @@ #define RESET "\033[0m" DEFINE_string(server, "127.0.0.1:11210", "IP Address of server"); -DEFINE_string(connection_type, "single", "Connection type. Available values: single, pooled, short"); +DEFINE_string(connection_type, "single", + "Connection type. Available values: single, pooled, short"); // DEFINE_string(username, "Administrator", "Couchbase username"); // DEFINE_string(password, "password", "Couchbase password"); // DEFINE_string(bucket_name, "testing", "Couchbase bucket name"); DEFINE_bool(use_bthread, true, "Use bthread to send requests"); DEFINE_string(load_balancer, "", "The algorithm for load balancing"); -DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); +DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); DEFINE_bool(dont_fail, false, "Print fatal when some call failed"); -DEFINE_int32(exptime, 0, "The to-be-got data will be expired after so many seconds"); +DEFINE_int32(exptime, 0, + "The to-be-got data will be expired after so many seconds"); uint32_t batch_size = 10; @@ -37,9 +57,10 @@ std::vector thread_response_times; std::mutex timing_mutex; // static void* sender(void* arg) { -// google::protobuf::RpcChannel* channel = +// google::protobuf::RpcChannel* channel = // static_cast(arg); -// const int base_index = g_sender_count.fetch_add(1, butil::memory_order_relaxed); +// const int base_index = g_sender_count.fetch_add(1, +// butil::memory_order_relaxed); // std::string value; // std::string key = "test_brpc_"; @@ -49,10 +70,10 @@ std::mutex timing_mutex; // for (int i = 0; i < batch_size; ++i) { // CHECK(request.Get(butil::string_printf("%s%d", key.c_str(), i))); // } - + // // Start timing for this thread // auto start_time = std::chrono::high_resolution_clock::now(); - + // while (!brpc::IsAskedToQuit()) { // // We will receive response synchronously, safe to put variables // // on stack. @@ -69,39 +90,48 @@ std::mutex timing_mutex; // for (i = 0; i < batch_size; ++i) { // uint32_t flags; // if (!response.PopGet(&value, &flags, NULL)) { -// LOG(INFO) << "Fail to GET the key, " << response.LastError(); -// std::cout<<"thread id "<< bthread_self() << " failed to get key: " << butil::string_printf("%s%d", key.c_str(), i) << std::endl; -// break; +// LOG(INFO) << "Fail to GET the key, " << +// response.LastError(); std::cout<<"thread id "<< +// bthread_self() << " failed to get key: " << +// butil::string_printf("%s%d", key.c_str(), i) << +// std::endl; break; // } -// std::cout<<"thread id "<< bthread_self() <<"Key: " << butil::string_printf("%s%d", key.c_str(), i) << ", Value: " << value << std::endl; +// std::cout<<"thread id "<< bthread_self() <<"Key: " << +// butil::string_printf("%s%d", key.c_str(), i) << ", Value: " +// << value << std::endl; // } - + // // End timing and calculate response time for this thread // auto end_time = std::chrono::high_resolution_clock::now(); -// auto duration = std::chrono::duration_cast(end_time - start_time); -// double response_time_ms = duration.count() / 1000.0; // Convert to milliseconds - -// std::cout << "Thread " << bthread_self() << " GET request took: " << response_time_ms << " ms" << std::endl; - +// auto duration = +// std::chrono::duration_cast(end_time - +// start_time); double response_time_ms = duration.count() / 1000.0; +// // Convert to milliseconds + +// std::cout << "Thread " << bthread_self() << " GET request took: " +// << response_time_ms << " ms" << std::endl; + // // Store the response time in thread-safe manner // { // std::lock_guard lock(timing_mutex); // thread_response_times.push_back(response_time_ms); // } - + // bthread_usleep(5000000); // Sleep for 50ms before exiting // return NULL; - + // } else { -// g_error_count << 1; - +// g_error_count << 1; + // // End timing even for failed requests // auto end_time = std::chrono::high_resolution_clock::now(); -// auto duration = std::chrono::duration_cast(end_time - start_time); -// double response_time_ms = duration.count() / 1000.0; - -// std::cout << "Thread " << bthread_self() << " GET request failed after: " << response_time_ms << " ms" << std::endl; - +// auto duration = +// std::chrono::duration_cast(end_time - +// start_time); double response_time_ms = duration.count() / 1000.0; + +// std::cout << "Thread " << bthread_self() << " GET request failed +// after: " << response_time_ms << " ms" << std::endl; + // bthread_usleep(5000000); // return NULL; // } @@ -109,610 +139,714 @@ std::mutex timing_mutex; // return NULL; // } -int main(){ - - std::vector> operation_times; - - brpc::Channel channel; - - // Initialize the channel, NULL means using default options. - brpc::ChannelOptions options; - options.protocol = brpc::PROTOCOL_COUCHBASE; - options.connection_type = FLAGS_connection_type; - options.timeout_ms = 2000 /*milliseconds*/; - options.max_retry = 2; - - if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) { - LOG(ERROR) << "Fail to initialize channel"; - return -1; - } - std::cout << GREEN << "Channel initialized successfully" << RESET << std::endl; - // Couchbase Authentication packet(SASL Auth) is now present in the channel - // now a request can be sent with which auth packet will also be sent. - - - //create authrequest and authresponse - brpc::Controller cntl; - brpc::CouchbaseRequest auth_request; - brpc::CouchbaseResponse auth_response; - - // Ask username and password for authentication - std::string username; - std::string password; - while(username.empty() || password.empty()) { - std::cout << "Enter Couchbase username: "; - std::cin >> username; - if(username.empty()) { - std::cout << "Username cannot be empty. Please enter again." << std::endl; - continue; - } - std::cout << "Enter Couchbase password: "; - std::cin >> password; - if(password.empty()) { - std::cout << "Password cannot be empty. Please enter again." << std::endl; - continue; - } - } - - if(!auth_request.Authenticate(username.c_str(), password.c_str())) { - LOG(ERROR) << "Fail to create authentication request"; - return -1; - } - - // Start timing for authentication - auto auth_start_time = std::chrono::high_resolution_clock::now(); - - channel.CallMethod(NULL, &cntl, &auth_request, &auth_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check authentication response status - if (!auth_response.LastError().empty()) { - LOG(ERROR) << "Authentication failed: " << auth_response.LastError(); - return -1; - } - - // End timing for authentication - auto auth_end_time = std::chrono::high_resolution_clock::now(); - auto auth_duration = std::chrono::duration_cast(auth_end_time - auth_start_time); - double auth_time_ms = auth_duration.count() / 1000.0; - - cntl.Reset(); - - std::cout << GREEN << "Authentication successful (took " << auth_time_ms << " ms), proceeding with Couchbase operations..." << RESET << std::endl; - - // Select bucket - std::string bucket_name; - while(bucket_name.empty()) { - std::cout << "Enter Couchbase bucket name: "; - std::cin >> bucket_name; - if(bucket_name.empty()) { - std::cout << "Bucket name cannot be empty. Please enter again." << std::endl; - continue; - } - } - brpc::CouchbaseRequest select_bucket_request; - brpc::CouchbaseResponse select_bucket_response; - if (!select_bucket_request.SelectBucket(bucket_name.c_str())) { - LOG(ERROR) << "Fail to SELECT bucket request"; - return -1; - } - - // Start timing for bucket selection - auto bucket_start_time = std::chrono::high_resolution_clock::now(); - - channel.CallMethod(NULL, &cntl, &select_bucket_request, &select_bucket_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check bucket selection response status - if (!select_bucket_response.LastError().empty()) { - LOG(ERROR) << "Bucket selection failed: " << select_bucket_response.LastError(); - return -1; - } - - // End timing for bucket selection - auto bucket_end_time = std::chrono::high_resolution_clock::now(); - auto bucket_duration = std::chrono::duration_cast(bucket_end_time - bucket_start_time); - double bucket_time_ms = bucket_duration.count() / 1000.0; - - cntl.Reset(); - - if (select_bucket_response.LastError().empty()) { - std::cout << GREEN << "Bucket selected successfully: " << bucket_name << " (took " << bucket_time_ms << " ms)" << RESET << std::endl; - } else { - std::cout << RED << "Failed to select bucket: " << select_bucket_response.LastError() << RESET << std::endl; - return -1; - } - - //enter batch size - // std::cout << "Enter batch size for operations (e.g., 10): "; - // std::cin >> batch_size; - - // // Add operation - // brpc::CouchbaseRequest add_request; - // brpc::CouchbaseResponse add_response; - - // for(int i = 0;i(add_end_time - add_start_time); - // double add_time_ms = add_duration.count() / 1000.0; - - // std::cout << "Added " << batch_size << " documents (took " << add_time_ms << " ms)" << std::endl; - - // cntl.Reset(); - - // std::vector bids; - // std::vector pids; - // if (!FLAGS_use_bthread) { - // pids.resize( batch_size); - // for (int i = 0; i < batch_size; ++i) { - // if (pthread_create(&pids[i], NULL, sender, &channel) != 0) { - // LOG(ERROR) << "Fail to create pthread"; - // return -1; - // } - // } - // } else { - // bids.resize( batch_size); - // for (int i = 0; i < batch_size; ++i) { - // if (bthread_start_background( - // &bids[i], NULL, sender, &channel) != 0) { - // LOG(ERROR) << "Fail to create bthread"; - // return -1; - // } - // } - // } - - // LOG(INFO) << "couchbase_client is going to quit"; - // for (int i = 0; i < batch_size; ++i) { - // if (!FLAGS_use_bthread) { - // pthread_join(pids[i], NULL); - // } else { - // bthread_join(bids[i], NULL); - // } - // } - - // // Calculate and print average response time - // if (!thread_response_times.empty()) { - // double total_time = 0.0; - // for (double time : thread_response_times) { - // total_time += time; - // } - // double average_response_time = total_time / thread_response_times.size(); - - // std::cout << "\n=== Performance Summary ===" << std::endl; - // std::cout << "Authentication time: " << auth_time_ms << " ms" << std::endl; - // std::cout << "Bucket selection time: " << bucket_time_ms << " ms" << std::endl; - // std::cout << "ADD operations time: " << add_time_ms << " ms" << std::endl; - // std::cout << "Total threads: " << thread_response_times.size() << std::endl; - // std::cout << "Average GET response time: " << average_response_time << " ms" << std::endl; - // std::cout << "Total time for all GET requests: " << total_time << " ms" << std::endl; - // std::cout << "Total authorization time: " << (auth_time_ms + bucket_time_ms) << " ms" << std::endl; - // std::cout << "=========================" << std::endl; - // } else { - // std::cout << "\n=== Performance Summary ===" << std::endl; - // std::cout << "Authentication time: " << auth_time_ms << " ms" << std::endl; - // std::cout << "Bucket selection time: " << bucket_time_ms << " ms" << std::endl; - // std::cout << "ADD operations time: " << add_time_ms << " ms" << std::endl; - // std::cout << "Total authorization time: " << (auth_time_ms + bucket_time_ms) << " ms" << std::endl; - // std::cout << "No successful GET requests completed." << std::endl; - // std::cout << "=========================" << std::endl; - // } - - // Add operation - brpc::CouchbaseRequest add_request; - brpc::CouchbaseResponse add_response; - if (!add_request.Add(butil::string_printf("user::test_brpc_binprot"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - auto add_start_time = std::chrono::high_resolution_clock::now(); - channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); - auto add_end_time = std::chrono::high_resolution_clock::now(); - auto add_duration = std::chrono::duration_cast(add_end_time - add_start_time); - operation_times.push_back({"Add user data (first attempt) in microsecond", add_duration.count()}); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - else{ - // Check ADD operation response status - uint64_t cas_value; - if (add_response.PopAdd(&cas_value)) { - std::cout << GREEN << "ADD operation successful, CAS: " << cas_value << RESET << std::endl; - } else { - std::cout << RED << add_response.LastError() << RESET << std::endl; - } - } - - cntl.Reset(); - - if (!add_request.Add(butil::string_printf("user::test_brpc_binprot"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - add_start_time = std::chrono::high_resolution_clock::now(); - channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast(add_end_time - add_start_time); - operation_times.push_back({"Add user data (Second attempt - expected failure) in microsecond ", add_duration.count()}); - - // Check second ADD operation response status (should fail with key exists) - if (cntl.Failed()) { - LOG(ERROR) << "RPC call failed: " << cntl.ErrorText(); - } else { - uint64_t cas_value; - if (add_response.PopAdd(&cas_value)) { - std::cout << GREEN << "Second ADD operation unexpectedly successful, CAS: " << cas_value << RESET << std::endl; - } else { - std::cout << RED << "Second ADD operation failed as expected: " << add_response.LastError() << RESET << std::endl; - } - } - - cntl.Reset(); - // Get operation - brpc::CouchbaseRequest get_request; - brpc::CouchbaseResponse get_response; - if (!get_request.Get(butil::string_printf("%s", "user::test_brpc_binprot"))) { - LOG(ERROR) << "Fail to GET request"; - return -1; - } - add_start_time = std::chrono::high_resolution_clock::now(); - channel.CallMethod(NULL, &cntl, &get_request, &get_response, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast(add_end_time - add_start_time); - operation_times.push_back({"Get user data in microsecond ", add_duration.count()}); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check GET operation response status - std::string value; - uint32_t flags = 0; - uint64_t cas = 0; - if (get_response.PopGet(&value, &flags, &cas)) { - std::cout << GREEN << "GET operation successful" << RESET << std::endl; - std::cout << "GET value: " << value << std::endl; - std::cout << "Flags: " << flags << std::endl; - std::cout << "CAS: " << cas << std::endl; - } else { - std::cout << RED << "GET operation failed: " << get_response.LastError() << RESET << std::endl; - std::cout << "Raw response (hex): "; - for (char c : get_response.raw_buffer().to_string()) { - printf("%02x ", static_cast(c)); - } - std::cout << std::endl; - } - cntl.Reset(); - - // Create a new request for binprot item1 - brpc::CouchbaseRequest add_request1; - brpc::CouchbaseResponse add_response1; - if (!add_request1.Add(butil::string_printf("binprot_item1"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - std::cout << "Sending ADD request for binprot_item1, pipelined count: " << add_request1.pipelined_count() << std::endl; - add_start_time = std::chrono::high_resolution_clock::now(); - channel.CallMethod(NULL, &cntl, &add_request1, &add_response1, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast(add_end_time - add_start_time); - operation_times.push_back({"Add binprot item1 in microsecond", add_duration.count()}); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check ADD binprot item1 response status +int main() { + std::vector> operation_times; + + brpc::Channel channel; + + // Initialize the channel, NULL means using default options. + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_COUCHBASE; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = 2000 /*milliseconds*/; + options.max_retry = 2; + + if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), + &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; + } + std::cout << GREEN << "Channel initialized successfully" << RESET + << std::endl; + // Couchbase Authentication packet(SASL Auth) is now present in the channel + // now a request can be sent with which auth packet will also be sent. + + // create authrequest and authresponse + brpc::Controller cntl; + brpc::CouchbaseRequest auth_request; + brpc::CouchbaseResponse auth_response; + + // Ask username and password for authentication + std::string username; + std::string password; + while (username.empty() || password.empty()) { + std::cout << "Enter Couchbase username: "; + std::cin >> username; + if (username.empty()) { + std::cout << "Username cannot be empty. Please enter again." << std::endl; + continue; + } + std::cout << "Enter Couchbase password: "; + std::cin >> password; + if (password.empty()) { + std::cout << "Password cannot be empty. Please enter again." << std::endl; + continue; + } + } + + if (!auth_request.Authenticate(username.c_str(), password.c_str())) { + LOG(ERROR) << "Fail to create authentication request"; + return -1; + } + + // Start timing for authentication + auto auth_start_time = std::chrono::high_resolution_clock::now(); + + channel.CallMethod(NULL, &cntl, &auth_request, &auth_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check authentication response status + if (!auth_response.LastError().empty()) { + LOG(ERROR) << "Authentication failed: " << auth_response.LastError(); + return -1; + } + + // End timing for authentication + auto auth_end_time = std::chrono::high_resolution_clock::now(); + auto auth_duration = std::chrono::duration_cast( + auth_end_time - auth_start_time); + double auth_time_ms = auth_duration.count() / 1000.0; + + cntl.Reset(); + + std::cout << GREEN << "Authentication successful (took " << auth_time_ms + << " ms), proceeding with Couchbase operations..." << RESET + << std::endl; + + // Select bucket + std::string bucket_name; + while (bucket_name.empty()) { + std::cout << "Enter Couchbase bucket name: "; + std::cin >> bucket_name; + if (bucket_name.empty()) { + std::cout << "Bucket name cannot be empty. Please enter again." + << std::endl; + continue; + } + } + brpc::CouchbaseRequest select_bucket_request; + brpc::CouchbaseResponse select_bucket_response; + if (!select_bucket_request.SelectBucket(bucket_name.c_str())) { + LOG(ERROR) << "Fail to SELECT bucket request"; + return -1; + } + + // Start timing for bucket selection + auto bucket_start_time = std::chrono::high_resolution_clock::now(); + + channel.CallMethod(NULL, &cntl, &select_bucket_request, + &select_bucket_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check bucket selection response status + if (!select_bucket_response.LastError().empty()) { + LOG(ERROR) << "Bucket selection failed: " + << select_bucket_response.LastError(); + return -1; + } + + // End timing for bucket selection + auto bucket_end_time = std::chrono::high_resolution_clock::now(); + auto bucket_duration = std::chrono::duration_cast( + bucket_end_time - bucket_start_time); + double bucket_time_ms = bucket_duration.count() / 1000.0; + + cntl.Reset(); + + if (select_bucket_response.LastError().empty()) { + std::cout << GREEN << "Bucket selected successfully: " << bucket_name + << " (took " << bucket_time_ms << " ms)" << RESET << std::endl; + } else { + std::cout << RED << "Failed to select bucket: " + << select_bucket_response.LastError() << RESET << std::endl; + return -1; + } + + // enter batch size + // std::cout << "Enter batch size for operations (e.g., 10): "; + // std::cin >> batch_size; + + // // Add operation + // brpc::CouchbaseRequest add_request; + // brpc::CouchbaseResponse add_response; + + // for(int i = 0;i(add_end_time - + // add_start_time); double add_time_ms = add_duration.count() / 1000.0; + + // std::cout << "Added " << batch_size << " documents (took " << add_time_ms + // << " ms)" << std::endl; + + // cntl.Reset(); + + // std::vector bids; + // std::vector pids; + // if (!FLAGS_use_bthread) { + // pids.resize( batch_size); + // for (int i = 0; i < batch_size; ++i) { + // if (pthread_create(&pids[i], NULL, sender, &channel) != 0) { + // LOG(ERROR) << "Fail to create pthread"; + // return -1; + // } + // } + // } else { + // bids.resize( batch_size); + // for (int i = 0; i < batch_size; ++i) { + // if (bthread_start_background( + // &bids[i], NULL, sender, &channel) != 0) { + // LOG(ERROR) << "Fail to create bthread"; + // return -1; + // } + // } + // } + + // LOG(INFO) << "couchbase_client is going to quit"; + // for (int i = 0; i < batch_size; ++i) { + // if (!FLAGS_use_bthread) { + // pthread_join(pids[i], NULL); + // } else { + // bthread_join(bids[i], NULL); + // } + // } + + // // Calculate and print average response time + // if (!thread_response_times.empty()) { + // double total_time = 0.0; + // for (double time : thread_response_times) { + // total_time += time; + // } + // double average_response_time = total_time / + // thread_response_times.size(); + + // std::cout << "\n=== Performance Summary ===" << std::endl; + // std::cout << "Authentication time: " << auth_time_ms << " ms" << + // std::endl; std::cout << "Bucket selection time: " << bucket_time_ms << + // " ms" << std::endl; std::cout << "ADD operations time: " << add_time_ms + // << " ms" << std::endl; std::cout << "Total threads: " << + // thread_response_times.size() << std::endl; std::cout << "Average GET + // response time: " << average_response_time << " ms" << std::endl; + // std::cout << "Total time for all GET requests: " << total_time << " ms" + // << std::endl; std::cout << "Total authorization time: " << + // (auth_time_ms + bucket_time_ms) << " ms" << std::endl; std::cout << + // "=========================" << std::endl; + // } else { + // std::cout << "\n=== Performance Summary ===" << std::endl; + // std::cout << "Authentication time: " << auth_time_ms << " ms" << + // std::endl; std::cout << "Bucket selection time: " << bucket_time_ms << + // " ms" << std::endl; std::cout << "ADD operations time: " << add_time_ms + // << " ms" << std::endl; std::cout << "Total authorization time: " << + // (auth_time_ms + bucket_time_ms) << " ms" << std::endl; std::cout << "No + // successful GET requests completed." << std::endl; std::cout << + // "=========================" << std::endl; + // } + + // Add operation + brpc::CouchbaseRequest add_request; + brpc::CouchbaseResponse add_response; + if (!add_request.Add( + butil::string_printf("user::test_brpc_binprot"), + R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", + 0xdeadbeef, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + auto add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); + auto add_end_time = std::chrono::high_resolution_clock::now(); + auto add_duration = std::chrono::duration_cast( + add_end_time - add_start_time); + operation_times.push_back( + {"Add user data (first attempt) in microsecond", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } else { + // Check ADD operation response status uint64_t cas_value; - if (add_response1.PopAdd(&cas_value)) { - std::cout << GREEN << "ADD binprot item1 successful, CAS: " << cas_value << RESET << std::endl; - } else { - std::cout << RED << "ADD binprot item1 failed: " << add_response1.LastError() << RESET << std::endl; - } - cntl.Reset(); - - // Create a new request for binprot item2 - brpc::CouchbaseRequest add_request2; - brpc::CouchbaseResponse add_response2; - if (!add_request2.Add(butil::string_printf("binprot_item2"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - std::cout << "Sending ADD request for binprot_item2, pipelined count: " << add_request2.pipelined_count() << std::endl; - add_start_time = std::chrono::high_resolution_clock::now(); - channel.CallMethod(NULL, &cntl, &add_request2, &add_response2, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast(add_end_time - add_start_time); - operation_times.push_back({"Add binprot item2 in microsecond", add_duration.count()}); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check ADD binprot item2 response status - if (add_response2.PopAdd(&cas_value)) { - std::cout << GREEN << "ADD binprot item2 successful, CAS: " << cas_value << RESET << std::endl; - } else { - std::cout << RED << "ADD binprot item2 failed: " << add_response2.LastError() << RESET << std::endl; - } - cntl.Reset(); - - // Create a new request for binprot item3 - brpc::CouchbaseRequest add_request3; - brpc::CouchbaseResponse add_response3; - if (!add_request3.Add(butil::string_printf("binprot_item3"), R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - std::cout << "Sending ADD request for binprot_item3, pipelined count: " << add_request3.pipelined_count() << std::endl; - add_start_time = std::chrono::high_resolution_clock::now(); - channel.CallMethod(NULL, &cntl, &add_request3, &add_response3, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast(add_end_time - add_start_time); - operation_times.push_back({"Add binprot item3 in microsecond", add_duration.count()}); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check ADD binprot item3 response status - if (add_response3.PopAdd(&cas_value)) { - std::cout << GREEN << "ADD binprot item3 successful, CAS: " << cas_value << RESET << std::endl; - } else { - std::cout << RED << "ADD binprot item3 failed: " << add_response3.LastError() << RESET << std::endl; - } - cntl.Reset(); - - //pipeline ADD operation - brpc::CouchbaseRequest pipeline_request; - brpc::CouchbaseResponse pipeline_response; - for (int i = 0; i < 10; ++i) { - if (!pipeline_request.Add(butil::string_printf("pipeline_item_%d", i), R"({"name": "Pipeline User", "age": 25, "email": "pipeline@example.com"})", 0xdeadbeef , FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - } - std::cout << "Sending pipeline ADD request, pipelined count: " << pipeline_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &pipeline_request, &pipeline_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check each pipelined operation response status - std::cout << "Processing pipeline responses..." << std::endl; - int successful_operations = 0; - int failed_operations = 0; - - for (int i = 0; i < 10; ++i) { - uint64_t cas_value; - if (pipeline_response.PopAdd(&cas_value)) { - std::cout << GREEN << "Pipeline ADD operation " << i << " successful, CAS: " << cas_value << RESET << std::endl; - successful_operations++; - } else { - std::cout << RED << "Pipeline ADD operation " << i << " failed: " << pipeline_response.LastError() << RESET << std::endl; - failed_operations++; - } - } - - std::cout << "Pipeline summary: " << successful_operations << " successful, " - << failed_operations << " failed operations" << std::endl; - - cntl.Reset(); - - // Example of mixed pipeline operations (GET operations on existing and non-existing keys) - std::cout << "\n=== Testing Mixed Pipeline Operations ===" << std::endl; - brpc::CouchbaseRequest mixed_pipeline_request; - brpc::CouchbaseResponse mixed_pipeline_response; - - // Add some GET operations to the pipeline - some will succeed, some will fail - std::vector keys_to_get = { - "user::test_brpc_binprot", // Should exist - "binprot_item1", // Should exist - "nonexistent_key1", // Should fail - "binprot_item2", // Should exist - "nonexistent_key2", // Should fail - "binprot_item3" // Should exist - }; - - for (const auto& key : keys_to_get) { - if (!mixed_pipeline_request.Get(key)) { - LOG(ERROR) << "Fail to add GET request for key: " << key; - return -1; - } - } - - std::cout << "Sending mixed pipeline GET request, pipelined count: " << mixed_pipeline_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &mixed_pipeline_request, &mixed_pipeline_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Process each GET response - int successful_gets = 0; - int failed_gets = 0; - - for (size_t i = 0; i < keys_to_get.size(); ++i) { - std::string value; - uint32_t flags; - uint64_t cas; - - if (mixed_pipeline_response.PopGet(&value, &flags, &cas)) { - std::cout << GREEN << "GET '" << keys_to_get[i] << "' successful - Value length: " << value.length() - << ", Flags: " << flags << ", CAS: " << cas << RESET << std::endl; - successful_gets++; - } else { - std::cout << RED << "GET '" << keys_to_get[i] << "' failed: " << mixed_pipeline_response.LastError() << RESET << std::endl; - failed_gets++; - } - } - - std::cout << "Mixed pipeline summary: " << successful_gets << " successful, " - << failed_gets << " failed GET operations" << std::endl; - - - cntl.Reset(); - - //perform an upsert on the existing key - brpc::CouchbaseRequest upsert_request; - brpc::CouchbaseResponse upsert_response; - if (!upsert_request.Upsert(butil::string_printf("user::test_brpc_binprot"), R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})", 0, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to add UPSERT request for key: user::test_brpc_binprot"; - return -1; - } - - channel.CallMethod(NULL, &cntl, &upsert_request, &upsert_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - // Check UPSERT operation response status - if (upsert_response.PopUpsert(&cas_value)) { - std::cout << GREEN << "UPSERT operation successful when the document exists in the server, CAS: " << cas_value << RESET << std::endl; + if (add_response.PopAdd(&cas_value)) { + std::cout << GREEN << "ADD operation successful, CAS: " << cas_value + << RESET << std::endl; } else { - std::cout << RED << "UPSERT operation failed, when the document does exists in the server: " << upsert_response.LastError() << RESET << std::endl; - } - - cntl.Reset(); - //do the upsert operation - brpc::CouchbaseRequest upsert_request_new_doc; - brpc::CouchbaseResponse upsert_response_new_doc; - if (!upsert_request_new_doc.Upsert(butil::string_printf("user::test_brpc_new_upsert"), R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})", 0, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to add UPSERT request for key: user::test_brpc_new_upsert"; - return -1; - } - - channel.CallMethod(NULL, &cntl, &upsert_request_new_doc, &upsert_response_new_doc, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - // Check UPSERT operation response status - if (upsert_response_new_doc.PopUpsert(&cas_value)) { - std::cout << GREEN << "UPSERT operation successful when the document doesn't exists in the server, CAS: " << cas_value << RESET << std::endl; + std::cout << RED << add_response.LastError() << RESET << std::endl; + } + } + + cntl.Reset(); + + if (!add_request.Add( + butil::string_printf("user::test_brpc_binprot"), + R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", + 0xdeadbeef, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast( + add_end_time - add_start_time); + operation_times.push_back( + {"Add user data (Second attempt - expected failure) in microsecond ", + add_duration.count()}); + + // Check second ADD operation response status (should fail with key exists) + if (cntl.Failed()) { + LOG(ERROR) << "RPC call failed: " << cntl.ErrorText(); + } else { + uint64_t cas_value; + if (add_response.PopAdd(&cas_value)) { + std::cout << GREEN + << "Second ADD operation unexpectedly successful, CAS: " + << cas_value << RESET << std::endl; } else { - std::cout << RED << "UPSERT operation failed when document does not exists in the server: " << upsert_response_new_doc.LastError() << RESET << std::endl; - } - - cntl.Reset(); - - //check the upserted data - brpc::CouchbaseRequest check_upsert_request; - brpc::CouchbaseResponse check_upsert_response; - if (!check_upsert_request.Get(butil::string_printf("user::test_brpc_new_upsert"))) { - LOG(ERROR) << "Fail to GET request for key: user::test_brpc_new_upsert"; - return -1; - } - - std::cout << "Sending GET request for user::test_brpc_new_upsert, pipelined count: " << check_upsert_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &check_upsert_request, &check_upsert_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check GET operation response status - if (check_upsert_response.PopGet(&value, &flags, &cas)) { - std::cout << GREEN << "GET after UPSERT operation successful - Value: " << value - << ", Flags: " << flags << ", CAS: " << cas << RESET << std::endl; + std::cout << RED << "Second ADD operation failed as expected: " + << add_response.LastError() << RESET << std::endl; + } + } + + cntl.Reset(); + // Get operation + brpc::CouchbaseRequest get_request; + brpc::CouchbaseResponse get_response; + if (!get_request.Get(butil::string_printf("%s", "user::test_brpc_binprot"))) { + LOG(ERROR) << "Fail to GET request"; + return -1; + } + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &get_request, &get_response, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast( + add_end_time - add_start_time); + operation_times.push_back( + {"Get user data in microsecond ", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check GET operation response status + std::string value; + uint32_t flags = 0; + uint64_t cas = 0; + if (get_response.PopGet(&value, &flags, &cas)) { + std::cout << GREEN << "GET operation successful" << RESET << std::endl; + std::cout << "GET value: " << value << std::endl; + std::cout << "Flags: " << flags << std::endl; + std::cout << "CAS: " << cas << std::endl; + } else { + std::cout << RED << "GET operation failed: " << get_response.LastError() + << RESET << std::endl; + std::cout << "Raw response (hex): "; + for (char c : get_response.raw_buffer().to_string()) { + printf("%02x ", static_cast(c)); + } + std::cout << std::endl; + } + cntl.Reset(); + + // Create a new request for binprot item1 + brpc::CouchbaseRequest add_request1; + brpc::CouchbaseResponse add_response1; + if (!add_request1.Add( + butil::string_printf("binprot_item1"), + R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", + 0xdeadbeef, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + std::cout << "Sending ADD request for binprot_item1, pipelined count: " + << add_request1.pipelined_count() << std::endl; + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request1, &add_response1, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast( + add_end_time - add_start_time); + operation_times.push_back( + {"Add binprot item1 in microsecond", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check ADD binprot item1 response status + uint64_t cas_value; + if (add_response1.PopAdd(&cas_value)) { + std::cout << GREEN << "ADD binprot item1 successful, CAS: " << cas_value + << RESET << std::endl; + } else { + std::cout << RED + << "ADD binprot item1 failed: " << add_response1.LastError() + << RESET << std::endl; + } + cntl.Reset(); + + // Create a new request for binprot item2 + brpc::CouchbaseRequest add_request2; + brpc::CouchbaseResponse add_response2; + if (!add_request2.Add( + butil::string_printf("binprot_item2"), + R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", + 0xdeadbeef, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + std::cout << "Sending ADD request for binprot_item2, pipelined count: " + << add_request2.pipelined_count() << std::endl; + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request2, &add_response2, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast( + add_end_time - add_start_time); + operation_times.push_back( + {"Add binprot item2 in microsecond", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check ADD binprot item2 response status + if (add_response2.PopAdd(&cas_value)) { + std::cout << GREEN << "ADD binprot item2 successful, CAS: " << cas_value + << RESET << std::endl; + } else { + std::cout << RED + << "ADD binprot item2 failed: " << add_response2.LastError() + << RESET << std::endl; + } + cntl.Reset(); + + // Create a new request for binprot item3 + brpc::CouchbaseRequest add_request3; + brpc::CouchbaseResponse add_response3; + if (!add_request3.Add( + butil::string_printf("binprot_item3"), + R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", + 0xdeadbeef, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + std::cout << "Sending ADD request for binprot_item3, pipelined count: " + << add_request3.pipelined_count() << std::endl; + add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request3, &add_response3, NULL); + add_end_time = std::chrono::high_resolution_clock::now(); + add_duration = std::chrono::duration_cast( + add_end_time - add_start_time); + operation_times.push_back( + {"Add binprot item3 in microsecond", add_duration.count()}); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check ADD binprot item3 response status + if (add_response3.PopAdd(&cas_value)) { + std::cout << GREEN << "ADD binprot item3 successful, CAS: " << cas_value + << RESET << std::endl; + } else { + std::cout << RED + << "ADD binprot item3 failed: " << add_response3.LastError() + << RESET << std::endl; + } + cntl.Reset(); + + // pipeline ADD operation + brpc::CouchbaseRequest pipeline_request; + brpc::CouchbaseResponse pipeline_response; + for (int i = 0; i < 10; ++i) { + if (!pipeline_request.Add( + butil::string_printf("pipeline_item_%d", i), + R"({"name": "Pipeline User", "age": 25, "email": "pipeline@example.com"})", + 0xdeadbeef, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to ADD request"; + return -1; + } + } + std::cout << "Sending pipeline ADD request, pipelined count: " + << pipeline_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &pipeline_request, &pipeline_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check each pipelined operation response status + std::cout << "Processing pipeline responses..." << std::endl; + int successful_operations = 0; + int failed_operations = 0; + + for (int i = 0; i < 10; ++i) { + uint64_t cas_value; + if (pipeline_response.PopAdd(&cas_value)) { + std::cout << GREEN << "Pipeline ADD operation " << i + << " successful, CAS: " << cas_value << RESET << std::endl; + successful_operations++; } else { - std::cout << RED << "GET after UPSERT operation failed: " << check_upsert_response.LastError() << RESET << std::endl; - } - - cntl.Reset(); - - //delete the Non-existent Key - brpc::CouchbaseRequest delete_request; - brpc::CouchbaseResponse delete_response; - if (!delete_request.Delete(butil::string_printf("Nonexistent_key"))) { - LOG(ERROR) << "Fail to DELETE request for key: Nonexistent_key"; - return -1; - } - - std::cout << "Sending DELETE request for Nonexistent_key, pipelined count: " << delete_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &delete_request, &delete_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check DELETE operation response status - if (delete_response.PopDelete()) { - std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; + std::cout << RED << "Pipeline ADD operation " << i + << " failed: " << pipeline_response.LastError() << RESET + << std::endl; + failed_operations++; + } + } + + std::cout << "Pipeline summary: " << successful_operations << " successful, " + << failed_operations << " failed operations" << std::endl; + + cntl.Reset(); + + // Example of mixed pipeline operations (GET operations on existing and + // non-existing keys) + std::cout << "\n=== Testing Mixed Pipeline Operations ===" << std::endl; + brpc::CouchbaseRequest mixed_pipeline_request; + brpc::CouchbaseResponse mixed_pipeline_response; + + // Add some GET operations to the pipeline - some will succeed, some will fail + std::vector keys_to_get = { + "user::test_brpc_binprot", // Should exist + "binprot_item1", // Should exist + "nonexistent_key1", // Should fail + "binprot_item2", // Should exist + "nonexistent_key2", // Should fail + "binprot_item3" // Should exist + }; + + for (const auto& key : keys_to_get) { + if (!mixed_pipeline_request.Get(key)) { + LOG(ERROR) << "Fail to add GET request for key: " << key; + return -1; + } + } + + std::cout << "Sending mixed pipeline GET request, pipelined count: " + << mixed_pipeline_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &mixed_pipeline_request, + &mixed_pipeline_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Process each GET response + int successful_gets = 0; + int failed_gets = 0; + + for (size_t i = 0; i < keys_to_get.size(); ++i) { + std::string value; + uint32_t flags; + uint64_t cas; + + if (mixed_pipeline_response.PopGet(&value, &flags, &cas)) { + std::cout << GREEN << "GET '" << keys_to_get[i] + << "' successful - Value length: " << value.length() + << ", Flags: " << flags << ", CAS: " << cas << RESET + << std::endl; + successful_gets++; } else { - std::cout << RED << "DELETE operation failed: as expected" << delete_response.LastError() << RESET << std::endl; - } - - cntl.Reset(); - - //delete the existing key - brpc::CouchbaseRequest delete_existing_request; - brpc::CouchbaseResponse delete_existing_response; - if (!delete_existing_request.Delete(butil::string_printf("user::test_brpc_binprot"))) { - LOG(ERROR) << "Fail to DELETE request for key: user::test_brpc_binprot"; - return -1; - } - - std::cout << "Sending DELETE request for user::test_brpc_binprot, pipelined count: " << delete_existing_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &delete_existing_request, &delete_existing_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check DELETE operation response status - if (delete_existing_response.PopDelete()) { - std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; + std::cout << RED << "GET '" << keys_to_get[i] + << "' failed: " << mixed_pipeline_response.LastError() << RESET + << std::endl; + failed_gets++; + } + } + + std::cout << "Mixed pipeline summary: " << successful_gets << " successful, " + << failed_gets << " failed GET operations" << std::endl; + + cntl.Reset(); + + // perform an upsert on the existing key + brpc::CouchbaseRequest upsert_request; + brpc::CouchbaseResponse upsert_response; + if (!upsert_request.Upsert( + butil::string_printf("user::test_brpc_binprot"), + R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})", + 0, FLAGS_exptime, 0)) { + LOG(ERROR) << "Fail to add UPSERT request for key: user::test_brpc_binprot"; + return -1; + } + + channel.CallMethod(NULL, &cntl, &upsert_request, &upsert_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + // Check UPSERT operation response status + if (upsert_response.PopUpsert(&cas_value)) { + std::cout << GREEN + << "UPSERT operation successful when the document exists in the " + "server, CAS: " + << cas_value << RESET << std::endl; + } else { + std::cout << RED + << "UPSERT operation failed, when the document does exists in " + "the server: " + << upsert_response.LastError() << RESET << std::endl; + } + + cntl.Reset(); + // do the upsert operation + brpc::CouchbaseRequest upsert_request_new_doc; + brpc::CouchbaseResponse upsert_response_new_doc; + if (!upsert_request_new_doc.Upsert( + butil::string_printf("user::test_brpc_new_upsert"), + R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})", + 0, FLAGS_exptime, 0)) { + LOG(ERROR) + << "Fail to add UPSERT request for key: user::test_brpc_new_upsert"; + return -1; + } + + channel.CallMethod(NULL, &cntl, &upsert_request_new_doc, + &upsert_response_new_doc, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + // Check UPSERT operation response status + if (upsert_response_new_doc.PopUpsert(&cas_value)) { + std::cout << GREEN + << "UPSERT operation successful when the document doesn't exists " + "in the server, CAS: " + << cas_value << RESET << std::endl; + } else { + std::cout << RED + << "UPSERT operation failed when document does not exists in the " + "server: " + << upsert_response_new_doc.LastError() << RESET << std::endl; + } + + cntl.Reset(); + + // check the upserted data + brpc::CouchbaseRequest check_upsert_request; + brpc::CouchbaseResponse check_upsert_response; + if (!check_upsert_request.Get( + butil::string_printf("user::test_brpc_new_upsert"))) { + LOG(ERROR) << "Fail to GET request for key: user::test_brpc_new_upsert"; + return -1; + } + + std::cout + << "Sending GET request for user::test_brpc_new_upsert, pipelined count: " + << check_upsert_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &check_upsert_request, &check_upsert_response, + NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check GET operation response status + if (check_upsert_response.PopGet(&value, &flags, &cas)) { + std::cout << GREEN + << "GET after UPSERT operation successful - Value: " << value + << ", Flags: " << flags << ", CAS: " << cas << RESET << std::endl; + } else { + std::cout << RED << "GET after UPSERT operation failed: " + << check_upsert_response.LastError() << RESET << std::endl; + } + + cntl.Reset(); + + // delete the Non-existent Key + brpc::CouchbaseRequest delete_request; + brpc::CouchbaseResponse delete_response; + if (!delete_request.Delete(butil::string_printf("Nonexistent_key"))) { + LOG(ERROR) << "Fail to DELETE request for key: Nonexistent_key"; + return -1; + } + + std::cout << "Sending DELETE request for Nonexistent_key, pipelined count: " + << delete_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &delete_request, &delete_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check DELETE operation response status + if (delete_response.PopDelete()) { + std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; + } else { + std::cout << RED << "DELETE operation failed: as expected" + << delete_response.LastError() << RESET << std::endl; + } + + cntl.Reset(); + + // delete the existing key + brpc::CouchbaseRequest delete_existing_request; + brpc::CouchbaseResponse delete_existing_response; + if (!delete_existing_request.Delete( + butil::string_printf("user::test_brpc_binprot"))) { + LOG(ERROR) << "Fail to DELETE request for key: user::test_brpc_binprot"; + return -1; + } + + std::cout + << "Sending DELETE request for user::test_brpc_binprot, pipelined count: " + << delete_existing_request.pipelined_count() << std::endl; + channel.CallMethod(NULL, &cntl, &delete_existing_request, + &delete_existing_response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + return -1; + } + + // Check DELETE operation response status + if (delete_existing_response.PopDelete()) { + std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; + } else { + std::cout << RED << "DELETE operation failed: " + << delete_existing_response.LastError() << RESET << std::endl; + } + + cntl.Reset(); + + // Print operation times + long long total_time = 0; + for (const auto& op : operation_times) { + std::cout << std::left << std::setw(40) << op.first << ": "; + if (op.second >= 1000) { + std::cout << std::right << std::setw(8) << (op.second / 1000.0) << " ms" + << std::endl; } else { - std::cout << RED << "DELETE operation failed: " << delete_existing_response.LastError() << RESET << std::endl; - } - - cntl.Reset(); - - // Print operation times - long long total_time = 0; - for (const auto& op : operation_times) { - std::cout << std::left << std::setw(40) << op.first << ": "; - if (op.second >= 1000) { - std::cout << std::right << std::setw(8) << (op.second / 1000.0) << " ms" << std::endl; - } else { - std::cout << std::right << std::setw(8) << op.second << " μs" << std::endl; - } - total_time += op.second; + std::cout << std::right << std::setw(8) << op.second << " μs" + << std::endl; } - return 0; + total_time += op.second; + } + return 0; } - - - - diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 54a5857030..89712a3c7c 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -17,8 +17,9 @@ #include "brpc/couchbase.h" +#include //for crc32 Vbucket_id + #include "brpc/policy/couchbase_protocol.h" -#include //for crc32 Vbucket_id #include "brpc/proto_base.pb.h" #include "butil/logging.h" #include "butil/macros.h" @@ -27,605 +28,612 @@ namespace brpc { -uint32_t CouchbaseRequest::hash_crc32(const char *key, size_t key_length) -{ - - static const uint32_t crc32tab[256] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, - 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, - 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 0x646ba8c0, 0xfd62f97a, - 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, - 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, - 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, - 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, 0x6b6b51f4, - 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, - 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, - 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, - 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, - 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, - 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 0xfed41b76, - 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, - 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, - 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, - 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, - 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, - 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, - 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, - 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 0xbdbdf21c, 0xcabac28a, 0x53b39330, - 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, - }; - - - uint64_t x; - uint32_t crc = UINT32_MAX; - - for (x = 0; x < key_length; x++) - crc = (crc >> 8) ^ crc32tab[(crc ^ (uint64_t)key[x]) & 0xff]; - - - #ifdef __APPLE__ - return ((~crc) >> 16) % 64; - #else - return ((~crc) >> 16) % 1024; - #endif -} - -//redefinition +uint32_t CouchbaseRequest::hash_crc32(const char* key, size_t key_length) { + static const uint32_t crc32tab[256] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, + }; + + uint64_t x; + uint32_t crc = UINT32_MAX; + + for (x = 0; x < key_length; x++) + crc = (crc >> 8) ^ crc32tab[(crc ^ (uint64_t)key[x]) & 0xff]; + +#ifdef __APPLE__ + return ((~crc) >> 16) % 64; +#else + return ((~crc) >> 16) % 1024; +#endif +} + +// redefinition CouchbaseRequest::CouchbaseRequest() : NonreflectableMessage() { - SharedCtor(); + SharedCtor(); } CouchbaseRequest::CouchbaseRequest(const CouchbaseRequest& from) : NonreflectableMessage(from) { - SharedCtor(); - MergeFrom(from); + SharedCtor(); + MergeFrom(from); } void CouchbaseRequest::SharedCtor() { - _pipelined_count = 0; - _cached_size_ = 0; + _pipelined_count = 0; + _cached_size_ = 0; } -CouchbaseRequest::~CouchbaseRequest() { - SharedDtor(); -} +CouchbaseRequest::~CouchbaseRequest() { SharedDtor(); } -void CouchbaseRequest::SharedDtor() { -} +void CouchbaseRequest::SharedDtor() {} -void CouchbaseRequest::SetCachedSize(int size) const { - _cached_size_ = size; -} +void CouchbaseRequest::SetCachedSize(int size) const { _cached_size_ = size; } void CouchbaseRequest::Clear() { - _buf.clear(); - _pipelined_count = 0; + _buf.clear(); + _pipelined_count = 0; } // Get the Scope ID for a given scope name bool CouchbaseRequest::GetScopeId(const butil::StringPiece& scope_name) { - if (scope_name.empty()) { - LOG(ERROR) << "Empty scope name"; - return false; - } - // Opcode 0xBC for Get Scope ID (see Collections.md) - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - policy::CB_GET_SCOPE_ID, - butil::HostToNet16(scope_name.size()), - 0, // no extras - policy::CB_BINARY_RAW_BYTES, - 0, // no vbucket - butil::HostToNet32(scope_name.size()), - 0, // opaque - 0 // no CAS - }; - if (_buf.append(&header, sizeof(header))) { - return false; - } - if (_buf.append(scope_name.data(), scope_name.size())) { - return false; - } - ++_pipelined_count; - return true; -} - -bool CouchbaseRequest::SelectBucket(const butil::StringPiece &bucket_name) { - if (bucket_name.empty()) { - LOG(ERROR) << "Empty bucket name"; - return false; - } - //construct the request header - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - policy::CB_SELECT_BUCKET, - butil::HostToNet16(bucket_name.size()), - 0, - policy::CB_BINARY_RAW_BYTES, - 0, - butil::HostToNet32(bucket_name.size()), - 0, - 0 - }; - if (_buf.append(&header, sizeof(header))) { - std::cout<<"Failed to append header to buffer"<(features), value_len)) { - std::cout << "Failed to append Hello features to buffer" << std::endl; - return false; - } - ++_pipelined_count; - return true; +bool CouchbaseRequest::SelectBucket(const butil::StringPiece& bucket_name) { + if (bucket_name.empty()) { + LOG(ERROR) << "Empty bucket name"; + return false; + } + // construct the request header + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_SELECT_BUCKET, + butil::HostToNet16(bucket_name.size()), + 0, + policy::CB_BINARY_RAW_BYTES, + 0, + butil::HostToNet32(bucket_name.size()), + 0, + 0}; + if (_buf.append(&header, sizeof(header))) { + std::cout << "Failed to append header to buffer" << std::endl; + return false; + } + if (_buf.append(bucket_name.data(), bucket_name.size())) { + std::cout << "Failed to append bucket name to buffer" << std::endl; + return false; + } + ++_pipelined_count; + return true; +} + +// HelloRequest sends a Hello request to the Couchbase server, which specifies +// the client features and capabilities. +// This is typically the first request sent after connecting to the server. +// It includes the agent name and a randomly generated connection ID in JSON +// format. +bool CouchbaseRequest::HelloRequest() { + std::string agent = "brpc/1.0.0 ("; +#ifdef __APPLE__ + agent += "Darwin/"; +#elif defined(__linux__) + agent += "Linux/"; +#else + agent += "UnknownOS/"; +#endif +#if defined(__x86_64__) + agent += "x86_64"; +#elif defined(__aarch64__) + agent += "arm64"; +#else + agent += "unknown"; +#endif + agent += ";bssl/0x1010107f)"; + + // Generate a random 33-byte connection id as hex string (66 hex chars) + unsigned char raw_id[33]; + FILE* urandom = fopen("/dev/urandom", "rb"); + if (!urandom || fread(raw_id, 1, 33, urandom) != 33) { + if (urandom) fclose(urandom); + std::cout << "Failed to generate random connection id" << std::endl; + return false; + } + fclose(urandom); + char hex_id[67] = {0}; + for (int i = 0; i < 33; ++i) { + sprintf(hex_id + i * 2, "%02x", raw_id[i]); + } + + // Format key as JSON: {"a":"agent","i":"hex_id"} + std::string key = + std::string("{\"a\":\"") + agent + "\",\"i\":\"" + hex_id + "\"}"; + + const uint16_t key_len = key.size(); + uint16_t features[] = { + butil::HostToNet16(0x0001), // Datatype + butil::HostToNet16(0x0006), // XError + butil::HostToNet16(0x0007), // SelectBucket + butil::HostToNet16(0x000b), // Snappy + butil::HostToNet16(0x0012) // Collections + }; + + const uint32_t value_len = sizeof(features); + const uint32_t total_body_len = key_len + value_len; + + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_HELLO_SELECT_FEATURES, + butil::HostToNet16(key_len), // key length + 0, // extras length + policy::CB_BINARY_RAW_BYTES, // data type + 0, // vbucket id + butil::HostToNet32(total_body_len), // total body length + 0, // opaque + 0 // cas value + }; + + if (_buf.append(&header, sizeof(header))) { + std::cout << "Failed to append Hello header to buffer" << std::endl; + return false; + } + if (_buf.append(key.data(), key_len)) { + std::cout << "Failed to append Hello JSON key to buffer" << std::endl; + return false; + } + if (_buf.append(reinterpret_cast(features), value_len)) { + std::cout << "Failed to append Hello features to buffer" << std::endl; + return false; + } + ++_pipelined_count; + return true; } -bool CouchbaseRequest::Authenticate(const butil::StringPiece &username, - const butil::StringPiece &password) { - if (username.empty() || password.empty()) { - LOG(ERROR) << "Empty username or password"; - return false; - } - //insert the features to get enabled, calling function HelloRequest() will do this. - if (!HelloRequest()) { - LOG(ERROR) << "Failed to send HelloRequest for authentication"; - return false; - } - // Construct the request header - constexpr char kPlainAuthCommand[] = "PLAIN"; - constexpr char kPadding[1] = {'\0'}; - const brpc::policy::CouchbaseRequestHeader header = { - brpc::policy::CB_MAGIC_REQUEST, brpc::policy::CB_BINARY_SASL_AUTH, - butil::HostToNet16(sizeof(kPlainAuthCommand) - 1), 0, 0, 0, - butil::HostToNet32(sizeof(kPlainAuthCommand) + 1 + - username.length() * 2 + password.length()), - 0, 0}; - std::string* auth_str = new std::string(); - auth_str->clear(); - auth_str->append(reinterpret_cast(&header), sizeof(header)); - auth_str->append(kPlainAuthCommand, sizeof(kPlainAuthCommand) - 1); - auth_str->append(username.data()); - auth_str->append(kPadding, sizeof(kPadding)); - auth_str->append(username.data()); - auth_str->append(kPadding, sizeof(kPadding)); - auth_str->append(password.data()); - if(_buf.append(auth_str->data(), auth_str->size())) { - std::cout<<"Failed to append auth string to buffer"<clear(); + auth_str->append(reinterpret_cast(&header), sizeof(header)); + auth_str->append(kPlainAuthCommand, sizeof(kPlainAuthCommand) - 1); + auth_str->append(username.data()); + auth_str->append(kPadding, sizeof(kPadding)); + auth_str->append(username.data()); + auth_str->append(kPadding, sizeof(kPadding)); + auth_str->append(password.data()); + if (_buf.append(auth_str->data(), auth_str->size())) { + std::cout << "Failed to append auth string to buffer" << std::endl; + delete auth_str; + return false; + } + ++_pipelined_count; + return true; } bool CouchbaseRequest::MergePartialFromCodedStream( ::google::protobuf::io::CodedInputStream* input) { - LOG(WARNING) << "You're not supposed to parse a CouchbaseRequest"; - - // simple approach just making it work. - butil::IOBuf tmp; - const void* data = NULL; - int size = 0; - while (input->GetDirectBufferPointer(&data, &size)) { - tmp.append(data, size); - input->Skip(size); - } - const butil::IOBuf saved = tmp; - int count = 0; - for (; !tmp.empty(); ++count) { - char aux_buf[sizeof(policy::CouchbaseRequestHeader)]; - const policy::CouchbaseRequestHeader* header = - (const policy::CouchbaseRequestHeader*)tmp.fetch(aux_buf, sizeof(aux_buf)); - if (header == NULL) { - return false; - } - if (header->magic != (uint8_t)policy::CB_MAGIC_REQUEST) { - return false; - } - uint32_t total_body_length = butil::NetToHost32(header->total_body_length); - if (tmp.size() < sizeof(*header) + total_body_length) { - return false; - } - tmp.pop_front(sizeof(*header) + total_body_length); - } - _buf.append(saved); - _pipelined_count += count; - return true; + LOG(WARNING) << "You're not supposed to parse a CouchbaseRequest"; + + // simple approach just making it work. + butil::IOBuf tmp; + const void* data = NULL; + int size = 0; + while (input->GetDirectBufferPointer(&data, &size)) { + tmp.append(data, size); + input->Skip(size); + } + const butil::IOBuf saved = tmp; + int count = 0; + for (; !tmp.empty(); ++count) { + char aux_buf[sizeof(policy::CouchbaseRequestHeader)]; + const policy::CouchbaseRequestHeader* header = + (const policy::CouchbaseRequestHeader*)tmp.fetch(aux_buf, + sizeof(aux_buf)); + if (header == NULL) { + return false; + } + if (header->magic != (uint8_t)policy::CB_MAGIC_REQUEST) { + return false; + } + uint32_t total_body_length = butil::NetToHost32(header->total_body_length); + if (tmp.size() < sizeof(*header) + total_body_length) { + return false; + } + tmp.pop_front(sizeof(*header) + total_body_length); + } + _buf.append(saved); + _pipelined_count += count; + return true; } void CouchbaseRequest::SerializeWithCachedSizes( ::google::protobuf::io::CodedOutputStream* output) const { - LOG(WARNING) << "You're not supposed to serialize a CouchbaseRequest"; - - // simple approach just making it work. - butil::IOBufAsZeroCopyInputStream wrapper(_buf); - const void* data = NULL; - int size = 0; - while (wrapper.Next(&data, &size)) { - output->WriteRaw(data, size); - } + LOG(WARNING) << "You're not supposed to serialize a CouchbaseRequest"; + + // simple approach just making it work. + butil::IOBufAsZeroCopyInputStream wrapper(_buf); + const void* data = NULL; + int size = 0; + while (wrapper.Next(&data, &size)) { + output->WriteRaw(data, size); + } } size_t CouchbaseRequest::ByteSizeLong() const { - int total_size = static_cast(_buf.size()); - _cached_size_ = total_size; - return total_size; + int total_size = static_cast(_buf.size()); + _cached_size_ = total_size; + return total_size; } void CouchbaseRequest::MergeFrom(const CouchbaseRequest& from) { - CHECK_NE(&from, this); - _buf.append(from._buf); - _pipelined_count += from._pipelined_count; + CHECK_NE(&from, this); + _buf.append(from._buf); + _pipelined_count += from._pipelined_count; } -bool CouchbaseRequest::IsInitialized() const { - return _pipelined_count != 0; -} +bool CouchbaseRequest::IsInitialized() const { return _pipelined_count != 0; } void CouchbaseRequest::Swap(CouchbaseRequest* other) { - if (other != this) { - _buf.swap(other->_buf); - std::swap(_pipelined_count, other->_pipelined_count); - std::swap(_cached_size_, other->_cached_size_); - } + if (other != this) { + _buf.swap(other->_buf); + std::swap(_pipelined_count, other->_pipelined_count); + std::swap(_cached_size_, other->_cached_size_); + } } ::google::protobuf::Metadata CouchbaseRequest::GetMetadata() const { - ::google::protobuf::Metadata metadata{}; - metadata.descriptor = CouchbaseRequestBase::descriptor(); - metadata.reflection = nullptr; - return metadata; + ::google::protobuf::Metadata metadata{}; + metadata.descriptor = CouchbaseRequestBase::descriptor(); + metadata.reflection = nullptr; + return metadata; } -//redifinition +// redifinition CouchbaseResponse::CouchbaseResponse() : NonreflectableMessage() { - SharedCtor(); + SharedCtor(); } -//redifinition +// redifinition CouchbaseResponse::CouchbaseResponse(const CouchbaseResponse& from) : NonreflectableMessage(from) { - SharedCtor(); - MergeFrom(from); + SharedCtor(); + MergeFrom(from); } -void CouchbaseResponse::SharedCtor() { - _cached_size_ = 0; -} +void CouchbaseResponse::SharedCtor() { _cached_size_ = 0; } -//redifinition -CouchbaseResponse::~CouchbaseResponse() { - SharedDtor(); -} +// redifinition +CouchbaseResponse::~CouchbaseResponse() { SharedDtor(); } -void CouchbaseResponse::SharedDtor() { -} +void CouchbaseResponse::SharedDtor() {} -void CouchbaseResponse::SetCachedSize(int size) const { - _cached_size_ = size; -} +void CouchbaseResponse::SetCachedSize(int size) const { _cached_size_ = size; } -void CouchbaseResponse::Clear() { -} +void CouchbaseResponse::Clear() {} bool CouchbaseResponse::MergePartialFromCodedStream( ::google::protobuf::io::CodedInputStream* input) { - LOG(WARNING) << "You're not supposed to parse a CouchbaseResponse"; - - // simple approach just making it work. - const void* data = NULL; - int size = 0; - while (input->GetDirectBufferPointer(&data, &size)) { - _buf.append(data, size); - input->Skip(size); - } - return true; + LOG(WARNING) << "You're not supposed to parse a CouchbaseResponse"; + + // simple approach just making it work. + const void* data = NULL; + int size = 0; + while (input->GetDirectBufferPointer(&data, &size)) { + _buf.append(data, size); + input->Skip(size); + } + return true; } void CouchbaseResponse::SerializeWithCachedSizes( ::google::protobuf::io::CodedOutputStream* output) const { - LOG(WARNING) << "You're not supposed to serialize a CouchbaseResponse"; - - // simple approach just making it work. - butil::IOBufAsZeroCopyInputStream wrapper(_buf); - const void* data = NULL; - int size = 0; - while (wrapper.Next(&data, &size)) { - output->WriteRaw(data, size); - } + LOG(WARNING) << "You're not supposed to serialize a CouchbaseResponse"; + + // simple approach just making it work. + butil::IOBufAsZeroCopyInputStream wrapper(_buf); + const void* data = NULL; + int size = 0; + while (wrapper.Next(&data, &size)) { + output->WriteRaw(data, size); + } } size_t CouchbaseResponse::ByteSizeLong() const { - int total_size = static_cast(_buf.size()); - _cached_size_ = total_size; - return total_size; + int total_size = static_cast(_buf.size()); + _cached_size_ = total_size; + return total_size; } void CouchbaseResponse::MergeFrom(const CouchbaseResponse& from) { - CHECK_NE(&from, this); - _err = from._err; - // responses of memcached according to their binary layout, should be - // directly concatenatible. - _buf.append(from._buf); + CHECK_NE(&from, this); + _err = from._err; + // responses of memcached according to their binary layout, should be + // directly concatenatible. + _buf.append(from._buf); } -bool CouchbaseResponse::IsInitialized() const { - return !_buf.empty(); -} +bool CouchbaseResponse::IsInitialized() const { return !_buf.empty(); } void CouchbaseResponse::Swap(CouchbaseResponse* other) { - if (other != this) { - _buf.swap(other->_buf); - std::swap(_cached_size_, other->_cached_size_); - } + if (other != this) { + _buf.swap(other->_buf); + std::swap(_cached_size_, other->_cached_size_); + } } ::google::protobuf::Metadata CouchbaseResponse::GetMetadata() const { - ::google::protobuf::Metadata metadata{}; - metadata.descriptor = CouchbaseResponseBase::descriptor(); - metadata.reflection = nullptr; - return metadata; + ::google::protobuf::Metadata metadata{}; + metadata.descriptor = CouchbaseResponseBase::descriptor(); + metadata.reflection = nullptr; + return metadata; } // =================================================================== const char* CouchbaseResponse::status_str(Status st) { - switch (st) { + switch (st) { case STATUS_SUCCESS: - return "SUCCESS"; + return "SUCCESS"; case STATUS_KEY_ENOENT: - return "Key not found"; + return "Key not found"; case STATUS_KEY_EEXISTS: - return "Key already exists"; + return "Key already exists"; case STATUS_E2BIG: - return "Value too large"; + return "Value too large"; case STATUS_EINVAL: - return "Invalid arguments"; + return "Invalid arguments"; case STATUS_NOT_STORED: - return "Item not stored"; + return "Item not stored"; case STATUS_DELTA_BADVAL: - return "Invalid delta value for increment/decrement"; + return "Invalid delta value for increment/decrement"; case STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER: - return "VBucket belongs to another server"; + return "VBucket belongs to another server"; case STATUS_AUTH_ERROR: - return "Authentication failed"; + return "Authentication failed"; case STATUS_AUTH_CONTINUE: - return "Authentication continue"; + return "Authentication continue"; case STATUS_ERANGE: - return "Range error"; + return "Range error"; case STATUS_ROLLBACK: - return "Rollback required"; + return "Rollback required"; case STATUS_EACCESS: - return "Access denied"; + return "Access denied"; case STATUS_NOT_INITIALIZED: - return "Not initialized"; + return "Not initialized"; case STATUS_UNKNOWN_COMMAND: - return "Unknown command"; + return "Unknown command"; case STATUS_ENOMEM: - return "Out of memory"; + return "Out of memory"; case STATUS_NOT_SUPPORTED: - return "Operation not supported"; + return "Operation not supported"; case STATUS_EINTERNAL: - return "Internal server error"; + return "Internal server error"; case STATUS_EBUSY: - return "Server busy"; + return "Server busy"; case STATUS_ETMPFAIL: - return "Temporary failure"; + return "Temporary failure"; case STATUS_UNKNOWN_COLLECTION: - return "Unknown collection"; + return "Unknown collection"; case STATUS_NO_COLLECTIONS_MANIFEST: - return "No collections manifest"; + return "No collections manifest"; case STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST: - return "Cannot apply collections manifest"; + return "Cannot apply collections manifest"; case STATUS_COLLECTIONS_MANIFEST_IS_AHEAD: - return "Collections manifest is ahead"; + return "Collections manifest is ahead"; case STATUS_UNKNOWN_SCOPE: - return "Unknown scope"; + return "Unknown scope"; case STATUS_DCP_STREAM_ID_INVALID: - return "Invalid DCP stream ID"; + return "Invalid DCP stream ID"; case STATUS_DURABILITY_INVALID_LEVEL: - return "Invalid durability level"; + return "Invalid durability level"; case STATUS_DURABILITY_IMPOSSIBLE: - return "Durability requirements impossible"; + return "Durability requirements impossible"; case STATUS_SYNC_WRITE_IN_PROGRESS: - return "Synchronous write in progress"; + return "Synchronous write in progress"; case STATUS_SYNC_WRITE_AMBIGUOUS: - return "Synchronous write result ambiguous"; + return "Synchronous write result ambiguous"; case STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS: - return "Synchronous write re-commit in progress"; + return "Synchronous write re-commit in progress"; case STATUS_SUBDOC_PATH_NOT_FOUND: - return "Sub-document path not found"; + return "Sub-document path not found"; case STATUS_SUBDOC_PATH_MISMATCH: - return "Sub-document path mismatch"; + return "Sub-document path mismatch"; case STATUS_SUBDOC_PATH_EINVAL: - return "Invalid sub-document path"; + return "Invalid sub-document path"; case STATUS_SUBDOC_PATH_E2BIG: - return "Sub-document path too deep"; + return "Sub-document path too deep"; case STATUS_SUBDOC_DOC_E2DEEP: - return "Sub-document too deep"; + return "Sub-document too deep"; case STATUS_SUBDOC_VALUE_CANTINSERT: - return "Cannot insert sub-document value"; + return "Cannot insert sub-document value"; case STATUS_SUBDOC_DOC_NOT_JSON: - return "Document is not JSON"; + return "Document is not JSON"; case STATUS_SUBDOC_NUM_E2BIG: - return "Sub-document number too large"; + return "Sub-document number too large"; case STATUS_SUBDOC_DELTA_E2BIG: - return "Sub-document delta too large"; + return "Sub-document delta too large"; case STATUS_SUBDOC_PATH_EEXISTS: - return "Sub-document path already exists"; + return "Sub-document path already exists"; case STATUS_SUBDOC_VALUE_E2DEEP: - return "Sub-document value too deep"; + return "Sub-document value too deep"; case STATUS_SUBDOC_INVALID_COMBO: - return "Invalid sub-document operation combination"; + return "Invalid sub-document operation combination"; case STATUS_SUBDOC_MULTI_PATH_FAILURE: - return "Sub-document multi-path operation failed"; + return "Sub-document multi-path operation failed"; case STATUS_SUBDOC_SUCCESS_DELETED: - return "Sub-document operation succeeded on deleted document"; + return "Sub-document operation succeeded on deleted document"; case STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO: - return "Invalid extended attribute flag combination"; + return "Invalid extended attribute flag combination"; case STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO: - return "Invalid extended attribute key combination"; + return "Invalid extended attribute key combination"; case STATUS_SUBDOC_XATTR_UNKNOWN_MACRO: - return "Unknown extended attribute macro"; + return "Unknown extended attribute macro"; case STATUS_SUBDOC_XATTR_UNKNOWN_VATTR: - return "Unknown virtual extended attribute"; + return "Unknown virtual extended attribute"; case STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR: - return "Cannot modify virtual extended attribute"; + return "Cannot modify virtual extended attribute"; case STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED: - return "Sub-document multi-path operation failed on deleted document"; + return "Sub-document multi-path operation failed on deleted document"; case STATUS_SUBDOC_INVALID_XATTR_ORDER: - return "Invalid extended attribute order"; + return "Invalid extended attribute order"; case STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO: - return "Unknown virtual extended attribute macro"; + return "Unknown virtual extended attribute macro"; case STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS: - return "Can only revive deleted documents"; + return "Can only revive deleted documents"; case STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE: - return "Deleted document cannot have a value"; + return "Deleted document cannot have a value"; case STATUS_XATTR_EINVAL: - return "Invalid extended attributes"; - } - return "Unknown status"; + return "Invalid extended attributes"; + } + return "Unknown status"; } // Helper method to format error messages with status codes -std::string CouchbaseResponse::format_error_message(uint16_t status_code, const std::string& operation, const std::string& error_msg) { - if (error_msg.empty()) { - return butil::string_printf("%s failed with status 0x%02x (%s)", - operation.c_str(), status_code, status_str((Status)status_code)); - } else { - return butil::string_printf("%s failed with status 0x%02x (%s): %s", - operation.c_str(), status_code, status_str((Status)status_code), error_msg.c_str()); - } +std::string CouchbaseResponse::format_error_message( + uint16_t status_code, const std::string& operation, + const std::string& error_msg) { + if (error_msg.empty()) { + return butil::string_printf("%s failed with status 0x%02x (%s)", + operation.c_str(), status_code, + status_str((Status)status_code)); + } else { + return butil::string_printf( + "%s failed with status 0x%02x (%s): %s", operation.c_str(), status_code, + status_str((Status)status_code), error_msg.c_str()); + } } // MUST NOT have extras. // MUST have key. // MUST NOT have value. -bool CouchbaseRequest::GetOrDelete(uint8_t command, const butil::StringPiece& key) { - //Collection ID - uint8_t collection_id = 0; - uint16_t VBucket_id = hash_crc32(key.data(), key.size()); - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - command, - butil::HostToNet16(key.size() + 1), //collection id is part of key, so adding it in the key length - 0, // extras length - policy::CB_BINARY_RAW_BYTES, // data type - butil::HostToNet16(VBucket_id), - butil::HostToNet32(key.size()+sizeof(collection_id)), // total body length includes key and collection id - 0, - 0 - }; - if (_buf.append(&header, sizeof(header))) { - return false; - } - if( _buf.append(&collection_id, sizeof(collection_id))) { - return false; - } - if (_buf.append(key.data(), key.size())) { - return false; - } - ++_pipelined_count; - return true; +bool CouchbaseRequest::GetOrDelete(uint8_t command, + const butil::StringPiece& key) { + // Collection ID + uint8_t collection_id = 0; + uint16_t VBucket_id = hash_crc32(key.data(), key.size()); + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, command, + butil::HostToNet16( + key.size() + + 1), // collection id is part of key, so adding it in the key length + 0, // extras length + policy::CB_BINARY_RAW_BYTES, // data type + butil::HostToNet16(VBucket_id), + butil::HostToNet32(key.size() + + sizeof(collection_id)), // total body length includes + // key and collection id + 0, 0}; + if (_buf.append(&header, sizeof(header))) { + return false; + } + if (_buf.append(&collection_id, sizeof(collection_id))) { + return false; + } + if (_buf.append(key.data(), key.size())) { + return false; + } + ++_pipelined_count; + return true; } bool CouchbaseRequest::Get(const butil::StringPiece& key) { - return GetOrDelete(policy::CB_BINARY_GET, key); + return GetOrDelete(policy::CB_BINARY_GET, key); } bool CouchbaseRequest::Delete(const butil::StringPiece& key) { - return GetOrDelete(policy::CB_BINARY_DELETE, key); + return GetOrDelete(policy::CB_BINARY_DELETE, key); } struct FlushHeaderWithExtras { - policy::CouchbaseRequestHeader header; - uint32_t exptime; + policy::CouchbaseRequestHeader header; + uint32_t exptime; } __attribute__((packed)); BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); @@ -641,29 +649,23 @@ BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); // +---------------+---------------+---------------+---------------+ // Total 4 bytes bool CouchbaseRequest::Flush(uint32_t timeout) { - const uint8_t FLUSH_EXTRAS = (timeout == 0 ? 0 : 4); - FlushHeaderWithExtras header_with_extras = {{ - policy::CB_MAGIC_REQUEST, - policy::CB_BINARY_FLUSH, - 0, - FLUSH_EXTRAS, - policy::CB_BINARY_RAW_BYTES, - 0, - butil::HostToNet32(FLUSH_EXTRAS), - 0, - 0 }, butil::HostToNet32(timeout) }; - if (FLUSH_EXTRAS == 0) { - if (_buf.append(&header_with_extras.header, - sizeof(policy::CouchbaseRequestHeader))) { - return false; - } - } else { - if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { - return false; - } + const uint8_t FLUSH_EXTRAS = (timeout == 0 ? 0 : 4); + FlushHeaderWithExtras header_with_extras = { + {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_FLUSH, 0, FLUSH_EXTRAS, + policy::CB_BINARY_RAW_BYTES, 0, butil::HostToNet32(FLUSH_EXTRAS), 0, 0}, + butil::HostToNet32(timeout)}; + if (FLUSH_EXTRAS == 0) { + if (_buf.append(&header_with_extras.header, + sizeof(policy::CouchbaseRequestHeader))) { + return false; + } + } else { + if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { + return false; } - ++_pipelined_count; - return true; + } + ++_pipelined_count; + return true; } // (if found): @@ -678,104 +680,106 @@ bool CouchbaseRequest::Flush(uint32_t timeout) { // 0| Flags | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseResponse::PopGet( - butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value) { - const size_t n = _buf.size(); - policy::CouchbaseResponseHeader header; - if (n < sizeof(header)) { - butil::string_printf(&_err, "buffer is too small to contain a header"); - return false; - } - _buf.copy_to(&header, sizeof(header)); - if (header.command != (uint8_t)policy::CB_BINARY_GET) { - butil::string_printf(&_err, "not a GET response"); - return false; - } - if (n < sizeof(header) + header.total_body_length) { - butil::string_printf(&_err, "response=%u < header=%u + body=%u", - (unsigned)n, (unsigned)sizeof(header), header.total_body_length); - return false; - } - if (header.status != (uint16_t)STATUS_SUCCESS) { - LOG_IF(ERROR, header.extras_length != 0) << "GET response must not have flags"; - LOG_IF(ERROR, header.key_length != 0) << "GET response must not have key"; - const int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; - if (value_size < 0) { - butil::string_printf(&_err, "value_size=%d is non-negative", value_size); - return false; - } - _buf.pop_front(sizeof(header) + header.extras_length + - header.key_length); - if (value_size > 0) { - std::string error_msg; - _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "GET operation", error_msg); - } else { - _err = format_error_message(header.status, "GET operation"); - } - return false; - } - if (header.extras_length != 4u) { - butil::string_printf(&_err, "GET response must have flags as extras, actual length=%u", - header.extras_length); - return false; - } - if (header.key_length != 0) { - butil::string_printf(&_err, "GET response must not have key"); - return false; - } - const int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; +bool CouchbaseResponse::PopGet(butil::IOBuf* value, uint32_t* flags, + uint64_t* cas_value) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != (uint8_t)policy::CB_BINARY_GET) { + butil::string_printf(&_err, "not a GET response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "response=%u < header=%u + body=%u", + (unsigned)n, (unsigned)sizeof(header), + header.total_body_length); + return false; + } + if (header.status != (uint16_t)STATUS_SUCCESS) { + LOG_IF(ERROR, header.extras_length != 0) + << "GET response must not have flags"; + LOG_IF(ERROR, header.key_length != 0) << "GET response must not have key"; + const int value_size = (int)header.total_body_length - + (int)header.extras_length - (int)header.key_length; if (value_size < 0) { - butil::string_printf(&_err, "value_size=%d is non-negative", value_size); - return false; - } - _buf.pop_front(sizeof(header)); - uint32_t raw_flags = 0; - _buf.cutn(&raw_flags, sizeof(raw_flags)); - if (flags) { - *flags = butil::NetToHost32(raw_flags); + butil::string_printf(&_err, "value_size=%d is non-negative", value_size); + return false; } - if (value) { - value->clear(); - _buf.cutn(value, value_size); - } - if (cas_value) { - *cas_value = header.cas_value; - } - _err.clear(); - return true; -} - -bool CouchbaseResponse::PopGet( - std::string* value, uint32_t* flags, uint64_t* cas_value) { - butil::IOBuf tmp; - if (PopGet(&tmp, flags, cas_value)) { - tmp.copy_to(value); - return true; + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "GET operation", error_msg); + } else { + _err = format_error_message(header.status, "GET operation"); } return false; + } + if (header.extras_length != 4u) { + butil::string_printf( + &_err, "GET response must have flags as extras, actual length=%u", + header.extras_length); + return false; + } + if (header.key_length != 0) { + butil::string_printf(&_err, "GET response must not have key"); + return false; + } + const int value_size = (int)header.total_body_length - + (int)header.extras_length - (int)header.key_length; + if (value_size < 0) { + butil::string_printf(&_err, "value_size=%d is non-negative", value_size); + return false; + } + _buf.pop_front(sizeof(header)); + uint32_t raw_flags = 0; + _buf.cutn(&raw_flags, sizeof(raw_flags)); + if (flags) { + *flags = butil::NetToHost32(raw_flags); + } + if (value) { + value->clear(); + _buf.cutn(value, value_size); + } + if (cas_value) { + *cas_value = header.cas_value; + } + _err.clear(); + return true; +} + +bool CouchbaseResponse::PopGet(std::string* value, uint32_t* flags, + uint64_t* cas_value) { + butil::IOBuf tmp; + if (PopGet(&tmp, flags, cas_value)) { + tmp.copy_to(value); + return true; + } + return false; } // MUST NOT have extras // MUST NOT have key // MUST NOT have value bool CouchbaseResponse::PopDelete() { - return PopStore(policy::CB_BINARY_DELETE, NULL); + return PopStore(policy::CB_BINARY_DELETE, NULL); } bool CouchbaseResponse::PopFlush() { - return PopStore(policy::CB_BINARY_FLUSH, NULL); + return PopStore(policy::CB_BINARY_FLUSH, NULL); } struct StoreHeaderWithExtras { - policy::CouchbaseRequestHeader header; - uint32_t flags; - uint32_t exptime; + policy::CouchbaseRequestHeader header; + uint32_t flags; + uint32_t exptime; } __attribute__((packed)); BAIDU_CASSERT(sizeof(StoreHeaderWithExtras) == 32, must_match); -const size_t STORE_EXTRAS = sizeof(StoreHeaderWithExtras) - - sizeof(policy::CouchbaseRequestHeader); +const size_t STORE_EXTRAS = + sizeof(StoreHeaderWithExtras) - sizeof(policy::CouchbaseRequestHeader); // MUST have extras. // MUST have key. // MAY have value. @@ -789,38 +793,38 @@ const size_t STORE_EXTRAS = sizeof(StoreHeaderWithExtras) - // 4| Expiration | // +---------------+---------------+---------------+---------------+ // Total 8 bytes -bool CouchbaseRequest::Store( - uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value) { - //add collection id - // uint16_t collection_id = 0x00; - uint8_t collection_id = 0x00; - uint16_t vBucket_id = hash_crc32(key.data(),key.size()); - StoreHeaderWithExtras header_with_extras = {{ - policy::CB_MAGIC_REQUEST, - command, - butil::HostToNet16(key.size()+1), //collection id is part of key, so including it in key length. - STORE_EXTRAS, - policy::CB_JSON, - butil::HostToNet16(vBucket_id), - butil::HostToNet32(STORE_EXTRAS +sizeof(uint8_t) + key.size() + value.size()), // total body length - 0, - butil::HostToNet64(cas_value) - }, butil::HostToNet32(flags), butil::HostToNet32(exptime)}; - if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { - return false; - } - if( _buf.append(&collection_id, sizeof(uint8_t))) { - return false; - } - if (_buf.append(key.data(), key.size())) { - return false; - } - if (_buf.append(value.data(), value.size())) { - return false; - } - ++_pipelined_count; - return true; +bool CouchbaseRequest::Store(uint8_t command, const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value) { + // add collection id + // uint16_t collection_id = 0x00; + uint8_t collection_id = 0x00; + uint16_t vBucket_id = hash_crc32(key.data(), key.size()); + StoreHeaderWithExtras header_with_extras = { + {policy::CB_MAGIC_REQUEST, command, + butil::HostToNet16( + key.size() + + 1), // collection id is part of key, so including it in key length. + STORE_EXTRAS, policy::CB_JSON, butil::HostToNet16(vBucket_id), + butil::HostToNet32(STORE_EXTRAS + sizeof(uint8_t) + key.size() + + value.size()), // total body length + 0, butil::HostToNet64(cas_value)}, + butil::HostToNet32(flags), + butil::HostToNet32(exptime)}; + if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { + return false; + } + if (_buf.append(&collection_id, sizeof(uint8_t))) { + return false; + } + if (_buf.append(key.data(), key.size())) { + return false; + } + if (_buf.append(value.data(), value.size())) { + return false; + } + ++_pipelined_count; + return true; } // MUST have CAS @@ -828,279 +832,294 @@ bool CouchbaseRequest::Store( // MUST NOT have key // MUST NOT have value bool CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { - const size_t n = _buf.size(); - policy::CouchbaseResponseHeader header; - if (n < sizeof(header)) { - butil::string_printf(&_err, "buffer is too small to contain a header"); - return false; - } - _buf.copy_to(&header, sizeof(header)); - if (header.command != command) { - butil::string_printf(&_err, "Not a STORE response"); - return false; - } - if (n < sizeof(header) + header.total_body_length) { - butil::string_printf(&_err, "Not enough data"); - return false; - } - LOG_IF(ERROR, header.extras_length != 0) << "STORE response must not have flags"; - LOG_IF(ERROR, header.key_length != 0) << "STORE response must not have key"; - int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; - if (header.status != (uint16_t)STATUS_SUCCESS) { - _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - if (value_size > 0) { - std::string error_msg; - _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "STORE operation", error_msg); - } else { - _err = format_error_message(header.status, "STORE operation"); - } - return false; - } - LOG_IF(ERROR, value_size != 0) << "STORE response must not have value, actually=" - << value_size; - _buf.pop_front(sizeof(header) + header.total_body_length); - if (cas_value) { - CHECK(header.cas_value); - *cas_value = header.cas_value; + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != command) { + butil::string_printf(&_err, "Not a STORE response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "Not enough data"); + return false; + } + LOG_IF(ERROR, header.extras_length != 0) + << "STORE response must not have flags"; + LOG_IF(ERROR, header.key_length != 0) << "STORE response must not have key"; + int value_size = (int)header.total_body_length - (int)header.extras_length - + (int)header.key_length; + if (header.status != (uint16_t)STATUS_SUCCESS) { + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "STORE operation", error_msg); + } else { + _err = format_error_message(header.status, "STORE operation"); } - _err.clear(); - return true; + return false; + } + LOG_IF(ERROR, value_size != 0) + << "STORE response must not have value, actually=" << value_size; + _buf.pop_front(sizeof(header) + header.total_body_length); + if (cas_value) { + CHECK(header.cas_value); + *cas_value = header.cas_value; + } + _err.clear(); + return true; } -bool CouchbaseRequest::Upsert( - const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value) { - return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value); +bool CouchbaseRequest::Upsert(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value) { + return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value); } // Collection Management Methods bool CouchbaseRequest::GetCollectionsManifest() { - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - policy::CB_GET_COLLECTIONS_MANIFEST, - 0, // no key - 0, // no extras - policy::CB_BINARY_RAW_BYTES, - 0, // no vbucket - 0, // no body - 0, // opaque - 0 // no CAS - }; - if (_buf.append(&header, sizeof(header))) { - return false; - } - ++_pipelined_count; - return true; -} - -bool CouchbaseRequest::GetCollectionId(const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name) { - // Format the collection path as "scope.collection" - std::string collection_path = scope_name.as_string() + "." + collection_name.as_string(); - - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - policy::CB_COLLECTIONS_GET_CID, - butil::HostToNet16(collection_path.size()), - 0, // no extras - policy::CB_BINARY_RAW_BYTES, - 0, // no vbucket - butil::HostToNet32(collection_path.size()), - 0, // opaque - 0 // no CAS - }; - if (_buf.append(&header, sizeof(header))) { - return false; - } - if (_buf.append(collection_path.data(), collection_path.size())) { - return false; - } - ++_pipelined_count; - return true; + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_GET_COLLECTIONS_MANIFEST, + 0, // no key + 0, // no extras + policy::CB_BINARY_RAW_BYTES, + 0, // no vbucket + 0, // no body + 0, // opaque + 0 // no CAS + }; + if (_buf.append(&header, sizeof(header))) { + return false; + } + ++_pipelined_count; + return true; +} + +bool CouchbaseRequest::GetCollectionId( + const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name) { + // Format the collection path as "scope.collection" + std::string collection_path = + scope_name.as_string() + "." + collection_name.as_string(); + + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_COLLECTIONS_GET_CID, + butil::HostToNet16(collection_path.size()), + 0, // no extras + policy::CB_BINARY_RAW_BYTES, + 0, // no vbucket + butil::HostToNet32(collection_path.size()), + 0, // opaque + 0 // no CAS + }; + if (_buf.append(&header, sizeof(header))) { + return false; + } + if (_buf.append(collection_path.data(), collection_path.size())) { + return false; + } + ++_pipelined_count; + return true; } // Collection-aware document operations -bool CouchbaseRequest::GetFromCollection(const butil::StringPiece& key, - const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name) { - // Construct a collection-aware key: collection::key - std::string scoped_key = collection_name.as_string() + "::" + key.as_string(); - return GetOrDelete(policy::CB_BINARY_GET, butil::StringPiece(scoped_key)); +bool CouchbaseRequest::GetFromCollection( + const butil::StringPiece& key, const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name) { + // Construct a collection-aware key: collection::key + std::string scoped_key = collection_name.as_string() + "::" + key.as_string(); + return GetOrDelete(policy::CB_BINARY_GET, butil::StringPiece(scoped_key)); } -bool CouchbaseRequest::UpsertToCollection(const butil::StringPiece& key, - const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name) { - // Construct a collection-aware key: collection::key - std::string scoped_key = collection_name.as_string() + "::" + key.as_string(); - return Store(policy::CB_BINARY_SET, butil::StringPiece(scoped_key), value, flags, exptime, cas_value); +bool CouchbaseRequest::UpsertToCollection( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name) { + // Construct a collection-aware key: collection::key + std::string scoped_key = collection_name.as_string() + "::" + key.as_string(); + return Store(policy::CB_BINARY_SET, butil::StringPiece(scoped_key), value, + flags, exptime, cas_value); } -bool CouchbaseRequest::Add( - const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value) { - return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value); +bool CouchbaseRequest::Add(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value) { + return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value); } -bool CouchbaseRequest::Replace( - const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value) { - return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, cas_value); +bool CouchbaseRequest::Replace(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value) { + return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, + cas_value); } - -bool CouchbaseRequest::Append( - const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value) { - if (value.empty()) { - LOG(ERROR) << "value to append must be non-empty"; - return false; - } - return Store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value); + +bool CouchbaseRequest::Append(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value) { + if (value.empty()) { + LOG(ERROR) << "value to append must be non-empty"; + return false; + } + return Store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value); } -bool CouchbaseRequest::Prepend( - const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value) { - if (value.empty()) { - LOG(ERROR) << "value to prepend must be non-empty"; - return false; - } - return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, cas_value); +bool CouchbaseRequest::Prepend(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value) { + if (value.empty()) { + LOG(ERROR) << "value to prepend must be non-empty"; + return false; + } + return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, + cas_value); } bool CouchbaseResponse::PopUpsert(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_SET, cas_value); + return PopStore(policy::CB_BINARY_SET, cas_value); } bool CouchbaseResponse::PopAdd(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_ADD, cas_value); + return PopStore(policy::CB_BINARY_ADD, cas_value); } bool CouchbaseResponse::PopReplace(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_REPLACE, cas_value); + return PopStore(policy::CB_BINARY_REPLACE, cas_value); } bool CouchbaseResponse::PopAppend(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_APPEND, cas_value); + return PopStore(policy::CB_BINARY_APPEND, cas_value); } bool CouchbaseResponse::PopPrepend(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_PREPEND, cas_value); + return PopStore(policy::CB_BINARY_PREPEND, cas_value); } // Collection-related response methods bool CouchbaseResponse::PopCollectionsManifest(std::string* manifest_json) { - const size_t n = _buf.size(); - policy::CouchbaseResponseHeader header; - if (n < sizeof(header)) { - butil::string_printf(&_err, "buffer is too small to contain a header"); - return false; - } - _buf.copy_to(&header, sizeof(header)); - if (header.command != policy::CB_GET_COLLECTIONS_MANIFEST) { - butil::string_printf(&_err, "Not a collections manifest response"); - return false; - } - if (n < sizeof(header) + header.total_body_length) { - butil::string_printf(&_err, "Not enough data"); - return false; - } - if (header.status != 0) { - _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; - if (value_size > 0) { - std::string error_msg; - _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "Collections manifest request", error_msg); - } else { - _err = format_error_message(header.status, "Collections manifest request"); - } - return false; - } - - // Skip header and extras/key if any + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != policy::CB_GET_COLLECTIONS_MANIFEST) { + butil::string_printf(&_err, "Not a collections manifest response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "Not enough data"); + return false; + } + if (header.status != 0) { _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - - // Get the manifest JSON - int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; - if (manifest_json && value_size > 0) { - _buf.cutn(manifest_json, value_size); + int value_size = (int)header.total_body_length - (int)header.extras_length - + (int)header.key_length; + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "Collections manifest request", + error_msg); } else { - _buf.pop_front(value_size); + _err = + format_error_message(header.status, "Collections manifest request"); } - - _err.clear(); - return true; + return false; + } + + // Skip header and extras/key if any + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + + // Get the manifest JSON + int value_size = (int)header.total_body_length - (int)header.extras_length - + (int)header.key_length; + if (manifest_json && value_size > 0) { + _buf.cutn(manifest_json, value_size); + } else { + _buf.pop_front(value_size); + } + + _err.clear(); + return true; } bool CouchbaseResponse::PopCollectionId(uint32_t* collection_id) { - const size_t n = _buf.size(); - policy::CouchbaseResponseHeader header; - if (n < sizeof(header)) { - butil::string_printf(&_err, "buffer is too small to contain a header"); - return false; - } - _buf.copy_to(&header, sizeof(header)); - if (header.command != policy::CB_COLLECTIONS_GET_CID) { - butil::string_printf(&_err, "Not a collection ID response"); - return false; - } - if (n < sizeof(header) + header.total_body_length) { - butil::string_printf(&_err, "Not enough data"); - return false; - } - if (header.status != 0) { - _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; - if (value_size > 0) { - std::string error_msg; - _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "Collection ID request", error_msg); - } else { - _err = format_error_message(header.status, "Collection ID request"); - } - return false; - } - - // Skip header and extras/key if any + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != policy::CB_COLLECTIONS_GET_CID) { + butil::string_printf(&_err, "Not a collection ID response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "Not enough data"); + return false; + } + if (header.status != 0) { _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - - // Get the collection ID (typically 4 bytes) - int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; - if (collection_id && value_size >= 4) { - uint32_t cid; - _buf.copy_to(&cid, 4); - *collection_id = butil::NetToHost32(cid); - _buf.pop_front(value_size); + int value_size = (int)header.total_body_length - (int)header.extras_length - + (int)header.key_length; + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "Collection ID request", + error_msg); } else { - _buf.pop_front(value_size); + _err = format_error_message(header.status, "Collection ID request"); } - - _err.clear(); - return true; + return false; + } + + // Skip header and extras/key if any + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + + // Get the collection ID (typically 4 bytes) + int value_size = (int)header.total_body_length - (int)header.extras_length - + (int)header.key_length; + if (collection_id && value_size >= 4) { + uint32_t cid; + _buf.copy_to(&cid, 4); + *collection_id = butil::NetToHost32(cid); + _buf.pop_front(value_size); + } else { + _buf.pop_front(value_size); + } + + _err.clear(); + return true; } // Collection-aware response methods -bool CouchbaseResponse::PopGetFromCollection(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value) { - // Same implementation as PopGet, just aliased for clarity - return PopGet(value, flags, cas_value); +bool CouchbaseResponse::PopGetFromCollection(butil::IOBuf* value, + uint32_t* flags, + uint64_t* cas_value) { + // Same implementation as PopGet, just aliased for clarity + return PopGet(value, flags, cas_value); } bool CouchbaseResponse::PopUpsertToCollection(uint64_t* cas_value) { - // Same implementation as PopSet, just aliased for clarity - return PopUpsert(cas_value); + // Same implementation as PopSet, just aliased for clarity + return PopUpsert(cas_value); } struct IncrHeaderWithExtras { - policy::CouchbaseRequestHeader header; - uint64_t delta; - uint64_t initial_value; - uint32_t exptime; + policy::CouchbaseRequestHeader header; + uint64_t delta; + uint64_t initial_value; + uint32_t exptime; } __attribute__((packed)); BAIDU_CASSERT(sizeof(IncrHeaderWithExtras) == 44, must_match); -const size_t INCR_EXTRAS = sizeof(IncrHeaderWithExtras) - - sizeof(policy::CouchbaseRequestHeader); +const size_t INCR_EXTRAS = + sizeof(IncrHeaderWithExtras) - sizeof(policy::CouchbaseRequestHeader); // MUST have extras. // MUST have key. @@ -1119,37 +1138,36 @@ const size_t INCR_EXTRAS = sizeof(IncrHeaderWithExtras) - // 16| Expiration | // +---------------+---------------+---------------+---------------+ // Total 20 bytes -bool CouchbaseRequest::Counter( - uint8_t command, const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime) { - IncrHeaderWithExtras header_with_extras = {{ - policy::CB_MAGIC_REQUEST, - command, - butil::HostToNet16(key.size()), - INCR_EXTRAS, - policy::CB_BINARY_RAW_BYTES, - 0, - butil::HostToNet32(INCR_EXTRAS + key.size()), - 0, - 0 }, butil::HostToNet64(delta), butil::HostToNet64(initial_value), butil::HostToNet32(exptime) }; - if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { - return false; - } - if (_buf.append(key.data(), key.size())) { - return false; - } - ++_pipelined_count; - return true; +bool CouchbaseRequest::Counter(uint8_t command, const butil::StringPiece& key, + uint64_t delta, uint64_t initial_value, + uint32_t exptime) { + IncrHeaderWithExtras header_with_extras = { + {policy::CB_MAGIC_REQUEST, command, butil::HostToNet16(key.size()), + INCR_EXTRAS, policy::CB_BINARY_RAW_BYTES, 0, + butil::HostToNet32(INCR_EXTRAS + key.size()), 0, 0}, + butil::HostToNet64(delta), + butil::HostToNet64(initial_value), + butil::HostToNet32(exptime)}; + if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { + return false; + } + if (_buf.append(key.data(), key.size())) { + return false; + } + ++_pipelined_count; + return true; } bool CouchbaseRequest::Increment(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime) { - return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, exptime); + uint64_t initial_value, uint32_t exptime) { + return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, + exptime); } bool CouchbaseRequest::Decrement(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime) { - return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, exptime); + uint64_t initial_value, uint32_t exptime) { + return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, + exptime); } // MUST NOT have extras. @@ -1163,74 +1181,79 @@ bool CouchbaseRequest::Decrement(const butil::StringPiece& key, uint64_t delta, // | | // +---------------+---------------+---------------+---------------+ // Total 8 bytes -bool CouchbaseResponse::PopCounter( - uint8_t command, uint64_t* new_value, uint64_t* cas_value) { - const size_t n = _buf.size(); - policy::CouchbaseResponseHeader header; - if (n < sizeof(header)) { - butil::string_printf(&_err, "buffer is too small to contain a header"); - return false; - } - _buf.copy_to(&header, sizeof(header)); - if (header.command != command) { - butil::string_printf(&_err, "not a INCR/DECR response"); - return false; - } - if (n < sizeof(header) + header.total_body_length) { - butil::string_printf(&_err, "response=%u < header=%u + body=%u", - (unsigned)n, (unsigned)sizeof(header), header.total_body_length); - return false; - } - LOG_IF(ERROR, header.extras_length != 0) << "INCR/DECR response must not have flags"; - LOG_IF(ERROR, header.key_length != 0) << "INCR/DECR response must not have key"; - const int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; - _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - - if (header.status != (uint16_t)STATUS_SUCCESS) { - if (value_size < 0) { - butil::string_printf(&_err, "value_size=%d is negative", value_size); - } else { - if (value_size > 0) { - std::string error_msg; - _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "Counter operation", error_msg); - } else { - _err = format_error_message(header.status, "Counter operation"); - } - } - return false; - } - if (value_size != 8) { - butil::string_printf(&_err, "value_size=%d is not 8", value_size); - return false; - } - uint64_t raw_value = 0; - _buf.cutn(&raw_value, sizeof(raw_value)); - *new_value = butil::NetToHost64(raw_value); - if (cas_value) { - *cas_value = header.cas_value; +bool CouchbaseResponse::PopCounter(uint8_t command, uint64_t* new_value, + uint64_t* cas_value) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != command) { + butil::string_printf(&_err, "not a INCR/DECR response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "response=%u < header=%u + body=%u", + (unsigned)n, (unsigned)sizeof(header), + header.total_body_length); + return false; + } + LOG_IF(ERROR, header.extras_length != 0) + << "INCR/DECR response must not have flags"; + LOG_IF(ERROR, header.key_length != 0) + << "INCR/DECR response must not have key"; + const int value_size = (int)header.total_body_length - + (int)header.extras_length - (int)header.key_length; + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + + if (header.status != (uint16_t)STATUS_SUCCESS) { + if (value_size < 0) { + butil::string_printf(&_err, "value_size=%d is negative", value_size); + } else { + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = + format_error_message(header.status, "Counter operation", error_msg); + } else { + _err = format_error_message(header.status, "Counter operation"); + } } - _err.clear(); - return true; + return false; + } + if (value_size != 8) { + butil::string_printf(&_err, "value_size=%d is not 8", value_size); + return false; + } + uint64_t raw_value = 0; + _buf.cutn(&raw_value, sizeof(raw_value)); + *new_value = butil::NetToHost64(raw_value); + if (cas_value) { + *cas_value = header.cas_value; + } + _err.clear(); + return true; } bool CouchbaseResponse::PopIncrement(uint64_t* new_value, uint64_t* cas_value) { - return PopCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); + return PopCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); } bool CouchbaseResponse::PopDecrement(uint64_t* new_value, uint64_t* cas_value) { - return PopCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); + return PopCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); } // MUST have extras. // MUST have key. // MUST NOT have value. struct TouchHeaderWithExtras { - policy::CouchbaseRequestHeader header; - uint32_t exptime; + policy::CouchbaseRequestHeader header; + uint32_t exptime; } __attribute__((packed)); BAIDU_CASSERT(sizeof(TouchHeaderWithExtras) == 28, must_match); -const size_t TOUCH_EXTRAS = sizeof(TouchHeaderWithExtras) - sizeof(policy::CouchbaseRequestHeader); +const size_t TOUCH_EXTRAS = + sizeof(TouchHeaderWithExtras) - sizeof(policy::CouchbaseRequestHeader); // MAY have extras. // MUST NOT have key. @@ -1244,97 +1267,93 @@ const size_t TOUCH_EXTRAS = sizeof(TouchHeaderWithExtras) - sizeof(policy::Couch // +---------------+---------------+---------------+---------------+ // Total 4 bytes bool CouchbaseRequest::Touch(const butil::StringPiece& key, uint32_t exptime) { - TouchHeaderWithExtras header_with_extras = {{ - policy::CB_MAGIC_REQUEST, - policy::CB_BINARY_TOUCH, - butil::HostToNet16(key.size()), - TOUCH_EXTRAS, - policy::CB_BINARY_RAW_BYTES, - 0, - butil::HostToNet32(TOUCH_EXTRAS + key.size()), - 0, - 0 }, butil::HostToNet32(exptime) }; - if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { - return false; - } - if (_buf.append(key.data(), key.size())) { - return false; - } - ++_pipelined_count; - return true; + TouchHeaderWithExtras header_with_extras = { + {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_TOUCH, + butil::HostToNet16(key.size()), TOUCH_EXTRAS, + policy::CB_BINARY_RAW_BYTES, 0, + butil::HostToNet32(TOUCH_EXTRAS + key.size()), 0, 0}, + butil::HostToNet32(exptime)}; + if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { + return false; + } + if (_buf.append(key.data(), key.size())) { + return false; + } + ++_pipelined_count; + return true; } // MUST NOT have extras. // MUST NOT have key. // MUST NOT have value. bool CouchbaseRequest::Version() { - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - policy::CB_BINARY_VERSION, - 0, - 0, - policy::CB_BINARY_RAW_BYTES, - 0, - 0, - 0, - 0 - }; - if (_buf.append(&header, sizeof(header))) { - return false; - } - ++_pipelined_count; - return true; + const policy::CouchbaseRequestHeader header = {policy::CB_MAGIC_REQUEST, + policy::CB_BINARY_VERSION, + 0, + 0, + policy::CB_BINARY_RAW_BYTES, + 0, + 0, + 0, + 0}; + if (_buf.append(&header, sizeof(header))) { + return false; + } + ++_pipelined_count; + return true; } // MUST NOT have extras. // MUST NOT have key. // MUST have value. bool CouchbaseResponse::PopTouch() { - return PopStore(policy::CB_BINARY_TOUCH, NULL); + return PopStore(policy::CB_BINARY_TOUCH, NULL); } bool CouchbaseResponse::PopVersion(std::string* version) { - const size_t n = _buf.size(); - policy::CouchbaseResponseHeader header; - if (n < sizeof(header)) { - butil::string_printf(&_err, "buffer is too small to contain a header"); - return false; - } - _buf.copy_to(&header, sizeof(header)); - if (header.command != policy::CB_BINARY_VERSION) { - butil::string_printf(&_err, "not a VERSION response"); - return false; - } - if (n < sizeof(header) + header.total_body_length) { - butil::string_printf(&_err, "response=%u < header=%u + body=%u", - (unsigned)n, (unsigned)sizeof(header), header.total_body_length); - return false; - } - LOG_IF(ERROR, header.extras_length != 0) << "VERSION response must not have flags"; - LOG_IF(ERROR, header.key_length != 0) << "VERSION response must not have key"; - const int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; - _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - if (value_size < 0) { - butil::string_printf(&_err, "value_size=%d is negative", value_size); - return false; - } - if (header.status != (uint16_t)STATUS_SUCCESS) { - if (value_size > 0) { - std::string error_msg; - _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "Version request", error_msg); - } else { - _err = format_error_message(header.status, "Version request"); - } - return false; - } - if (version) { - version->clear(); - _buf.cutn(version, value_size); + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + if (header.command != policy::CB_BINARY_VERSION) { + butil::string_printf(&_err, "not a VERSION response"); + return false; + } + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "response=%u < header=%u + body=%u", + (unsigned)n, (unsigned)sizeof(header), + header.total_body_length); + return false; + } + LOG_IF(ERROR, header.extras_length != 0) + << "VERSION response must not have flags"; + LOG_IF(ERROR, header.key_length != 0) << "VERSION response must not have key"; + const int value_size = (int)header.total_body_length - + (int)header.extras_length - (int)header.key_length; + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + if (value_size < 0) { + butil::string_printf(&_err, "value_size=%d is negative", value_size); + return false; + } + if (header.status != (uint16_t)STATUS_SUCCESS) { + if (value_size > 0) { + std::string error_msg; + _buf.cutn(&error_msg, value_size); + _err = format_error_message(header.status, "Version request", error_msg); + } else { + _err = format_error_message(header.status, "Version request"); } - _err.clear(); - return true; + return false; + } + if (version) { + version->clear(); + _buf.cutn(version, value_size); + } + _err.clear(); + return true; } - -} // namespace brpc + +} // namespace brpc diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index cf23727fa9..392e5b438d 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -1,229 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #ifndef BRPC_COUCHBASE_H #define BRPC_COUCHBASE_H -#endif // COUCHBASE_MEMCACHE_H +#endif #include -#include "butil/iobuf.h" -#include "butil/strings/string_piece.h" + #include "brpc/nonreflectable_message.h" #include "brpc/pb_compat.h" +#include "butil/iobuf.h" +#include "butil/strings/string_piece.h" namespace brpc { - class CouchbaseRequest : public NonreflectableMessage { - private: - int _pipelined_count; - butil::IOBuf _buf; - mutable int _cached_size_; - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - bool GetOrDelete(uint8_t command, const butil::StringPiece& key); - bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime); - - bool Store(uint8_t command, const butil::StringPiece& key, - const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); - uint32_t hash_crc32(const char *key, size_t key_length); - public: - CouchbaseRequest(); - ~CouchbaseRequest() override; - CouchbaseRequest(const CouchbaseRequest& from); - inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { - CopyFrom(from); - return *this; - } - - bool SelectBucket(const butil::StringPiece &bucket_name); - bool Authenticate(const butil::StringPiece &username, - const butil::StringPiece &password); - bool HelloRequest(); - - // Collection Management Methods - bool GetCollectionsManifest(); - bool GetCollectionId(const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name); - - bool GetScopeId(const butil::StringPiece& scope_name); - - // Collection-aware document operations - bool Get(const butil::StringPiece& key); - bool GetFromCollection(const butil::StringPiece& key, - const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name); - - bool Upsert(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); - bool UpsertToCollection(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - const butil::StringPiece& scope_name, const butil::StringPiece& collection_name); - - bool Add(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); - - bool Replace(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); - - bool Append(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); - - bool Prepend(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); - - bool Delete(const butil::StringPiece& key); - bool Flush(uint32_t timeout); - - bool Increment(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime); - bool Decrement(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime); - - bool Touch(const butil::StringPiece& key, uint32_t exptime); - - bool Version(); - - int pipelined_count() const { return _pipelined_count; } - - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - void Swap(CouchbaseRequest* other); - void MergeFrom(const CouchbaseRequest& from) override; - void Clear() override; - bool IsInitialized() const PB_527_OVERRIDE; - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - - }; - - class CouchbaseResponse : public NonreflectableMessage { - private: - std::string _err; - butil::IOBuf _buf; - mutable int _cached_size_; - bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); - bool PopStore(uint8_t command, uint64_t* cas_value); - - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - public: - CouchbaseResponse(); - ~CouchbaseResponse() override; - CouchbaseResponse(const CouchbaseResponse& from); - inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { - CopyFrom(from); - return *this; - } - enum Status { - STATUS_SUCCESS = 0x00, - STATUS_KEY_ENOENT = 0x01, - STATUS_KEY_EEXISTS = 0x02, - STATUS_E2BIG = 0x03, - STATUS_EINVAL = 0x04, - STATUS_NOT_STORED = 0x05, - STATUS_DELTA_BADVAL = 0x06, - STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, - STATUS_AUTH_ERROR = 0x20, - STATUS_AUTH_CONTINUE = 0x21, - STATUS_ERANGE = 0x22, - STATUS_ROLLBACK = 0x23, - STATUS_EACCESS = 0x24, - STATUS_NOT_INITIALIZED = 0x25, - STATUS_UNKNOWN_COMMAND = 0x81, - STATUS_ENOMEM = 0x82, - STATUS_NOT_SUPPORTED = 0x83, - STATUS_EINTERNAL = 0x84, - STATUS_EBUSY = 0x85, - STATUS_ETMPFAIL = 0x86, - STATUS_UNKNOWN_COLLECTION = 0x88, - STATUS_NO_COLLECTIONS_MANIFEST = 0x89, - STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, - STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, - STATUS_UNKNOWN_SCOPE = 0x8c, - STATUS_DCP_STREAM_ID_INVALID = 0x8d, - STATUS_DURABILITY_INVALID_LEVEL = 0xa0, - STATUS_DURABILITY_IMPOSSIBLE = 0xa1, - STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, - STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, - STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, - STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, - STATUS_SUBDOC_PATH_MISMATCH = 0xc1, - STATUS_SUBDOC_PATH_EINVAL = 0xc2, - STATUS_SUBDOC_PATH_E2BIG = 0xc3, - STATUS_SUBDOC_DOC_E2DEEP = 0xc4, - STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, - STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, - STATUS_SUBDOC_NUM_E2BIG = 0xc7, - STATUS_SUBDOC_DELTA_E2BIG = 0xc8, - STATUS_SUBDOC_PATH_EEXISTS = 0xc9, - STATUS_SUBDOC_VALUE_E2DEEP = 0xca, - STATUS_SUBDOC_INVALID_COMBO = 0xcb, - STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, - STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, - STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, - STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, - STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, - STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, - STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, - STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, - STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, - STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, - STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, - STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, - STATUS_XATTR_EINVAL = 0xe0 - }; - - void MergeFrom(const CouchbaseResponse& from) override; - void Clear() override; - bool IsInitialized() const PB_527_OVERRIDE; - - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - static const char* status_str(Status); - - // Helper method to format error messages with status codes - static std::string format_error_message(uint16_t status_code, const std::string& operation, const std::string& error_msg = ""); - - // Add methods to handle response parsing - void Swap(CouchbaseResponse* other); - bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool PopGet(std::string* value, uint32_t* flags, uint64_t* cas_value); - const std::string& LastError() const { return _err; } - bool PopUpsert(uint64_t* cas_value); - bool PopAdd(uint64_t* cas_value); - bool PopReplace(uint64_t* cas_value); - bool PopAppend(uint64_t* cas_value); - bool PopPrepend(uint64_t* cas_value); - - // Collection-related response methods - bool PopCollectionsManifest(std::string* manifest_json); - bool PopCollectionId(uint32_t* collection_id); - - // Collection-aware response methods (same as regular but for documentation) - bool PopGetFromCollection(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool PopUpsertToCollection(uint64_t* cas_value); - - bool PopDelete(); - bool PopFlush(); - bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); - bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); - bool PopTouch(); - bool PopVersion(std::string* version); - }; -} \ No newline at end of file +class CouchbaseRequest : public NonreflectableMessage { + private: + int _pipelined_count; + butil::IOBuf _buf; + mutable int _cached_size_; + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + bool GetOrDelete(uint8_t command, const butil::StringPiece& key); + bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + + bool Store(uint8_t command, const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, uint32_t exptime, + uint64_t cas_value); + uint32_t hash_crc32(const char* key, size_t key_length); + + public: + CouchbaseRequest(); + ~CouchbaseRequest() override; + CouchbaseRequest(const CouchbaseRequest& from); + inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { + CopyFrom(from); + return *this; + } + + bool SelectBucket(const butil::StringPiece& bucket_name); + bool Authenticate(const butil::StringPiece& username, + const butil::StringPiece& password); + bool HelloRequest(); + + // Collection Management Methods + bool GetCollectionsManifest(); + bool GetCollectionId(const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name); + + bool GetScopeId(const butil::StringPiece& scope_name); + + // Collection-aware document operations + bool Get(const butil::StringPiece& key); + bool GetFromCollection(const butil::StringPiece& key, + const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name); + + bool Upsert(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + bool UpsertToCollection(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value, + const butil::StringPiece& scope_name, + const butil::StringPiece& collection_name); + + bool Add(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + + bool Replace(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + + bool Append(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + + bool Prepend(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value); + + bool Delete(const butil::StringPiece& key); + bool Flush(uint32_t timeout); + + bool Increment(const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + bool Decrement(const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + + bool Touch(const butil::StringPiece& key, uint32_t exptime); + + bool Version(); + + int pipelined_count() const { return _pipelined_count; } + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + void Swap(CouchbaseRequest* other); + void MergeFrom(const CouchbaseRequest& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; +}; + +class CouchbaseResponse : public NonreflectableMessage { + private: + std::string _err; + butil::IOBuf _buf; + mutable int _cached_size_; + bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); + bool PopStore(uint8_t command, uint64_t* cas_value); + + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + + public: + CouchbaseResponse(); + ~CouchbaseResponse() override; + CouchbaseResponse(const CouchbaseResponse& from); + inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { + CopyFrom(from); + return *this; + } + enum Status { + STATUS_SUCCESS = 0x00, + STATUS_KEY_ENOENT = 0x01, + STATUS_KEY_EEXISTS = 0x02, + STATUS_E2BIG = 0x03, + STATUS_EINVAL = 0x04, + STATUS_NOT_STORED = 0x05, + STATUS_DELTA_BADVAL = 0x06, + STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, + STATUS_AUTH_ERROR = 0x20, + STATUS_AUTH_CONTINUE = 0x21, + STATUS_ERANGE = 0x22, + STATUS_ROLLBACK = 0x23, + STATUS_EACCESS = 0x24, + STATUS_NOT_INITIALIZED = 0x25, + STATUS_UNKNOWN_COMMAND = 0x81, + STATUS_ENOMEM = 0x82, + STATUS_NOT_SUPPORTED = 0x83, + STATUS_EINTERNAL = 0x84, + STATUS_EBUSY = 0x85, + STATUS_ETMPFAIL = 0x86, + STATUS_UNKNOWN_COLLECTION = 0x88, + STATUS_NO_COLLECTIONS_MANIFEST = 0x89, + STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, + STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, + STATUS_UNKNOWN_SCOPE = 0x8c, + STATUS_DCP_STREAM_ID_INVALID = 0x8d, + STATUS_DURABILITY_INVALID_LEVEL = 0xa0, + STATUS_DURABILITY_IMPOSSIBLE = 0xa1, + STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, + STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, + STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, + STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, + STATUS_SUBDOC_PATH_MISMATCH = 0xc1, + STATUS_SUBDOC_PATH_EINVAL = 0xc2, + STATUS_SUBDOC_PATH_E2BIG = 0xc3, + STATUS_SUBDOC_DOC_E2DEEP = 0xc4, + STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, + STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, + STATUS_SUBDOC_NUM_E2BIG = 0xc7, + STATUS_SUBDOC_DELTA_E2BIG = 0xc8, + STATUS_SUBDOC_PATH_EEXISTS = 0xc9, + STATUS_SUBDOC_VALUE_E2DEEP = 0xca, + STATUS_SUBDOC_INVALID_COMBO = 0xcb, + STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, + STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, + STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, + STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, + STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, + STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, + STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, + STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, + STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, + STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, + STATUS_XATTR_EINVAL = 0xe0 + }; + + void MergeFrom(const CouchbaseResponse& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + static const char* status_str(Status); + + // Helper method to format error messages with status codes + static std::string format_error_message(uint16_t status_code, + const std::string& operation, + const std::string& error_msg = ""); + + // Add methods to handle response parsing + void Swap(CouchbaseResponse* other); + bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); + bool PopGet(std::string* value, uint32_t* flags, uint64_t* cas_value); + const std::string& LastError() const { return _err; } + bool PopUpsert(uint64_t* cas_value); + bool PopAdd(uint64_t* cas_value); + bool PopReplace(uint64_t* cas_value); + bool PopAppend(uint64_t* cas_value); + bool PopPrepend(uint64_t* cas_value); + + // Collection-related response methods + bool PopCollectionsManifest(std::string* manifest_json); + bool PopCollectionId(uint32_t* collection_id); + + // Collection-aware response methods (same as regular but for documentation) + bool PopGetFromCollection(butil::IOBuf* value, uint32_t* flags, + uint64_t* cas_value); + bool PopUpsertToCollection(uint64_t* cas_value); + + bool PopDelete(); + bool PopFlush(); + bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); + bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); + bool PopTouch(); + bool PopVersion(std::string* version); +}; +} // namespace brpc \ No newline at end of file From b38bb2de88ad809b931031a87ad9156ed9eacad7 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Sat, 20 Sep 2025 06:11:31 +0530 Subject: [PATCH 11/49] fixed bugs, added support for collections and added couchbase_client.md --- docs/en/couchbase_example.md | 213 ++++++++++ example/couchbase_c++/couchbase_client.cpp | 467 +++++++++------------ src/brpc/couchbase.cpp | 347 ++++++++------- src/brpc/couchbase.h | 53 +-- src/brpc/policy/couchbase_protocol.cpp | 371 ++++++++-------- 5 files changed, 800 insertions(+), 651 deletions(-) create mode 100644 docs/en/couchbase_example.md diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md new file mode 100644 index 0000000000..5f4cd83a86 --- /dev/null +++ b/docs/en/couchbase_example.md @@ -0,0 +1,213 @@ +## Couchbase bRPC Binary Protocol Integration + +This document explains the implementation of Couchbase Binary Protocol support added to this branch of bRPC, the available request/response operations, collection support, and how to run the provided example client against either a local Couchbase Server cluster or a Couchbase Capella (cloud) deployment. + +--- +### 1. Overview + +The integration adds a new protocol handler (`PROTOCOL_COUCHBASE`) that allows using bRPC's asynchronous / pipelined request machinery to talk directly to Couchbase Server using its Couchbase Binary Protocol. + +The core pieces are: +* `src/brpc/policy/couchbase_protocol.[h|cpp]` – framing + parse loop for binary responses, and request serialization pass‑through. +* `src/brpc/couchbase.[h|cpp]` – high level request/response builders (`CouchbaseRequest`), parsers (`CouchbaseResponse`) and error-handlers. +* `example/couchbase_c++/couchbase_client.cpp` – an end‑to‑end example performing authentication, bucket selection, CRUD operations, pipelining, and collection‑scoped operations. + +Design goals: +* Keep wire structs identical to the binary protocol (24‑byte header, network order numeric fields). +* Allow batching multiple operations in one TCP write using bRPC's pipelining. +* Provide ergonomic helpers (Add, Get, Upsert, Delete, Increment/Decrement, SelectBucket, GetCollectionId, etc.). +* Future extensions. + +--- +### 2. Features + +| Category | Supported Operations | Notes | +|----------|----------------------|-------| +| Authentication | SASL `PLAIN` (`CB_BINARY_SASL_AUTH`) | Sent automatically when you enqueue an `Authenticate` request before others. | +| Bucket selection | `SelectBucket` (`CB_SELECT_BUCKET`) | Required before document operations (unless default bucket context). | +| Basic KV | Add / Set(Upsert) / Delete / Get | Flags + Exptime handled. CAS values returned. | +| Pipelining | Yes | Multiple independent binary requests in one buffer, responses drained in order. | +| Collections | `GetCollectionId`, collection‑scoped CRUD (key + collection id) | Parsing currently truncates to 8 bits in public API (upgrade path below). | +| Error Handling | Status → string mapping + formatted error message | Unsupported codes produce generic fallback. | + +Missing / Future candidates: Sub‑Document operations, Durability requirements, DCP, Collections Manifest retrieval, Extended Attributes (XATTR), Hello feature negotiation, TLS bootstrap. + +--- +### 3. Binary Protocol Mapping + +Couchbase binary protcol header +``` +Byte/ 0 | 1 | 2 | 3 | + / | | | | + |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| + +---------------+---------------+---------------+---------------+ + 0| Magic | Opcode | Key length | + +---------------+---------------+---------------+---------------+ + 4| Extras length | Data type | vbucket id | + +---------------+---------------+---------------+---------------+ + 8| Total body length | + +---------------+---------------+---------------+---------------+ + 12| Opaque | + +---------------+---------------+---------------+---------------+ + 16| CAS | + | | + +---------------+---------------+---------------+---------------+ + Total 24 bytes +``` + +Overall packet structure:- +``` + Byte/ 0 | 1 | 2 | 3 | + / | | | | + |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| + +---------------+---------------+---------------+---------------+ + 0| HEADER | + | | + | | + | | + +---------------+---------------+---------------+---------------+ + 24| COMMAND-SPECIFIC EXTRAS (as needed) | + | (note length in the extras length header field) | + +---------------+---------------+---------------+---------------+ + m| Key (as needed) | + | (note length in key length header field) | + +---------------+---------------+---------------+---------------+ + n| Value (as needed) | + | (note length is total body length header field, minus | + | sum of the extras and key length body fields) | + +---------------+---------------+---------------+---------------+ + Total 24 + x bytes (24 byte header, and x byte body) +``` + +--- +### 4. Request Building (`CouchbaseRequest`) + +High‑level helpers append one or more wire messages onto an internal `IOBuf`. After you schedule all operations, bRPC sends the accumulated buffer over the channel. + +Examples: +```cpp +CouchbaseRequest req; +req.Authenticate(user, pass); // SASL PLAIN +req.SelectBucket("travel-sample"); +req.Add("doc::1", json_body, flags, exptime, /*cas*/0); +req.Get("doc::1"); // Pipeline GET after ADD + +channel.CallMethod(nullptr, &cntl, &req, &resp, nullptr); +``` + +Pipelining count (`pipelined_count()`) is tracked so the parser knows when the full response group is collected. + +Collection ID retrieval: +```cpp +CouchbaseRequest coll_req; +coll_req.GetCollectionId("_default", "my_collection"); +``` +This builds a key of the form `scope.collection` with opcode `0xbb`. + +Collection‑scoped CRUD reuse existing helpers with the final `coll_id` byte parameter. + +--- +### 5. Response Parsing (`CouchbaseResponse`) + +Each `Pop*` method consumes the front of the internal response buffer, validating: +1. Header present. +2. Opcode matches expected operation. +3. Status == success (otherwise `_err` filled with formatted message). +4. Body length sufficient. + +Common patterns: +```cpp +uint64_t cas; +if (resp.PopAdd(&cas)) { /* success */ } else { LOG(ERROR) << resp.LastError(); } + +std::string val; uint32_t flags; uint64_t get_cas; +if (resp.PopGet(&val, &flags, &get_cas)) { /* use val */ } +``` + +Collection ID parsing (current state): +* Reads 8‑byte Manifest UID + 4‑byte Collection ID extras. +* Truncates Collection ID to `uint8_t` for API compatibility. + +The raw response buffer can be inspected via `raw_buffer()` for debug / future operations. + +--- +### 6. Example Client Walkthrough + +`example/couchbase_c++/couchbase_client.cpp` flow: +1. Build channel with `PROTOCOL_COUCHBASE`. +2. Prompt for username/password (SASL PLAIN auth). +3. Select bucket. +4. Perform Add, duplicate Add (expected failure), Get. +5. Add several documents (pipelined) and read responses sequentially. +6. Mixed pipelined GET operations (existing + missing keys) to showcase error handling. +7. Upsert existing and new documents; verify with subsequent Get. +8. Delete existing and missing keys. +9. Retrieve Collection ID for a target collection, then perform collection‑scoped Add/Get/Upsert/Get/Delete if ID retrieved. + +Removed instrumentation: The example originally timed operations; those statements were stripped per request to keep output concise. + +--- +### 7. Building and Running the Example + +Build (Make): +```bash +cd example/couchbase_c++/ +make +./couchbase_client +``` + +You will be prompted for: +``` +Enter Couchbase username: Administrator +Enter Couchbase password: ******** +Enter Couchbase bucket name: travel-sample +``` + +Ensure buckets/collections you test exist (or create them via UI/CLI) before collection‑scoped CRUD. + +--- +### 8. Setting Up Couchbase + +#### A. Local Install (Non‑Docker) +Download from: https://www.couchbase.com/downloads/ (Community or Enterprise). Install and repeat the same initialization steps through the Web Console at `http://localhost:8091`. +- Open http://localhost:8091 in a browser and follow setup wizard: +- Set admin credentials (Administrator / password) +- Accept terms, choose services (Data, Query, Index at minimum) +- Initialize cluster +- Create a bucket (e.g. travel-sample or custom) + +Create a collection (7.0+): + +- In the Web Console navigate: Buckets → Your Bucket → Scopes & Collections. +- Add a Scope (optional) or use `_default`. +- Add a Collection (e.g. `testing_collection`). + +#### B. Couchbase Capella (Cloud) +1. Sign up / log in: https://cloud.couchbase.com/ +2. Create a Free Trial or a Hosted Cluster. +3. Create a bucket (or load a sample dataset). +4. Create a database access credential (API key or user with appropriate RBAC roles – Data Reader/Writer, Bucket Admin as needed). +5. Get the connection string (choose the internal or public endpoint). +6. Update `--server` flag (or code) to point to the KV endpoint host:port. + +--- +### 9. Error Handling Patterns + +Each `Pop*` method returns `false` on: +* Mismatched opcode +* Incomplete buffer +* Non‑zero status (error) – `_err` stores a human readable string like: `STATUS_KEY_EEXISTS: Add operation failed: Key already exists` + +Recommended usage: +```cpp +uint64_t cas; +if (!resp.PopAdd(&cas)) { + LOG(ERROR) << resp.LastError(); +} +``` + +For pipelined batches, call the matching `Pop*` in the same sequence you enqueued the operations. + +--- +### 10. Summary +This implementation provides a foundation for high‑performance Couchbase KV and collection operations through bRPC's pipelined framework. Extending to sub‑doc, durability, TLS, and richer collection metadata are natural next steps. Contributions and issue reports are welcome. \ No newline at end of file diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index be305ed580..d43ca313a3 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -24,7 +24,6 @@ #include #include -#include #include #include @@ -56,92 +55,7 @@ butil::static_atomic g_sender_count = BUTIL_STATIC_ATOMIC_INIT(0); std::vector thread_response_times; std::mutex timing_mutex; -// static void* sender(void* arg) { -// google::protobuf::RpcChannel* channel = -// static_cast(arg); -// const int base_index = g_sender_count.fetch_add(1, -// butil::memory_order_relaxed); - -// std::string value; -// std::string key = "test_brpc_"; -// // Example batch size, can be parameterized - -// brpc::CouchbaseRequest request; -// for (int i = 0; i < batch_size; ++i) { -// CHECK(request.Get(butil::string_printf("%s%d", key.c_str(), i))); -// } - -// // Start timing for this thread -// auto start_time = std::chrono::high_resolution_clock::now(); - -// while (!brpc::IsAskedToQuit()) { -// // We will receive response synchronously, safe to put variables -// // on stack. -// brpc::CouchbaseResponse response; -// brpc::Controller cntl; - -// // Because `done'(last parameter) is NULL, this function waits until -// // the response comes back or error occurs(including timedout). -// channel->CallMethod(NULL, &cntl, &request, &response, NULL); -// const int64_t elp = cntl.latency_us(); -// if (!cntl.Failed()) { -// int i = 0; -// g_latency_recorder << cntl.latency_us(); -// for (i = 0; i < batch_size; ++i) { -// uint32_t flags; -// if (!response.PopGet(&value, &flags, NULL)) { -// LOG(INFO) << "Fail to GET the key, " << -// response.LastError(); std::cout<<"thread id "<< -// bthread_self() << " failed to get key: " << -// butil::string_printf("%s%d", key.c_str(), i) << -// std::endl; break; -// } -// std::cout<<"thread id "<< bthread_self() <<"Key: " << -// butil::string_printf("%s%d", key.c_str(), i) << ", Value: " -// << value << std::endl; -// } - -// // End timing and calculate response time for this thread -// auto end_time = std::chrono::high_resolution_clock::now(); -// auto duration = -// std::chrono::duration_cast(end_time - -// start_time); double response_time_ms = duration.count() / 1000.0; -// // Convert to milliseconds - -// std::cout << "Thread " << bthread_self() << " GET request took: " -// << response_time_ms << " ms" << std::endl; - -// // Store the response time in thread-safe manner -// { -// std::lock_guard lock(timing_mutex); -// thread_response_times.push_back(response_time_ms); -// } - -// bthread_usleep(5000000); // Sleep for 50ms before exiting -// return NULL; - -// } else { -// g_error_count << 1; - -// // End timing even for failed requests -// auto end_time = std::chrono::high_resolution_clock::now(); -// auto duration = -// std::chrono::duration_cast(end_time - -// start_time); double response_time_ms = duration.count() / 1000.0; - -// std::cout << "Thread " << bthread_self() << " GET request failed -// after: " << response_time_ms << " ms" << std::endl; - -// bthread_usleep(5000000); -// return NULL; -// } -// } -// return NULL; -// } - int main() { - std::vector> operation_times; - brpc::Channel channel; // Initialize the channel, NULL means using default options. @@ -189,32 +103,19 @@ int main() { return -1; } - // Start timing for authentication - auto auth_start_time = std::chrono::high_resolution_clock::now(); - channel.CallMethod(NULL, &cntl, &auth_request, &auth_response, NULL); + if (cntl.Failed()) { LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); return -1; } - // Check authentication response status - if (!auth_response.LastError().empty()) { - LOG(ERROR) << "Authentication failed: " << auth_response.LastError(); - return -1; - } - - // End timing for authentication - auto auth_end_time = std::chrono::high_resolution_clock::now(); - auto auth_duration = std::chrono::duration_cast( - auth_end_time - auth_start_time); - double auth_time_ms = auth_duration.count() / 1000.0; - cntl.Reset(); - std::cout << GREEN << "Authentication successful (took " << auth_time_ms - << " ms), proceeding with Couchbase operations..." << RESET - << std::endl; + std::cout + << GREEN + << "Authentication successful, proceeding with Couchbase operations..." + << RESET << std::endl; // Select bucket std::string bucket_name; @@ -234,143 +135,25 @@ int main() { return -1; } - // Start timing for bucket selection - auto bucket_start_time = std::chrono::high_resolution_clock::now(); - channel.CallMethod(NULL, &cntl, &select_bucket_request, &select_bucket_response, NULL); + if (cntl.Failed()) { LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); return -1; - } - - // Check bucket selection response status - if (!select_bucket_response.LastError().empty()) { - LOG(ERROR) << "Bucket selection failed: " - << select_bucket_response.LastError(); - return -1; - } - - // End timing for bucket selection - auto bucket_end_time = std::chrono::high_resolution_clock::now(); - auto bucket_duration = std::chrono::duration_cast( - bucket_end_time - bucket_start_time); - double bucket_time_ms = bucket_duration.count() / 1000.0; - - cntl.Reset(); - - if (select_bucket_response.LastError().empty()) { - std::cout << GREEN << "Bucket selected successfully: " << bucket_name - << " (took " << bucket_time_ms << " ms)" << RESET << std::endl; } else { - std::cout << RED << "Failed to select bucket: " - << select_bucket_response.LastError() << RESET << std::endl; - return -1; + // Check ADD operation response status + uint64_t cas_value; + if (select_bucket_response.PopSelectBucket(&cas_value)) { + std::cout << GREEN << "Bucket Selection Successful, CAS: " << cas_value + << RESET << std::endl; + } else { + std::cout << RED << select_bucket_response.LastError() << RESET + << std::endl; + } } - // enter batch size - // std::cout << "Enter batch size for operations (e.g., 10): "; - // std::cin >> batch_size; - - // // Add operation - // brpc::CouchbaseRequest add_request; - // brpc::CouchbaseResponse add_response; - - // for(int i = 0;i(add_end_time - - // add_start_time); double add_time_ms = add_duration.count() / 1000.0; - - // std::cout << "Added " << batch_size << " documents (took " << add_time_ms - // << " ms)" << std::endl; - - // cntl.Reset(); - - // std::vector bids; - // std::vector pids; - // if (!FLAGS_use_bthread) { - // pids.resize( batch_size); - // for (int i = 0; i < batch_size; ++i) { - // if (pthread_create(&pids[i], NULL, sender, &channel) != 0) { - // LOG(ERROR) << "Fail to create pthread"; - // return -1; - // } - // } - // } else { - // bids.resize( batch_size); - // for (int i = 0; i < batch_size; ++i) { - // if (bthread_start_background( - // &bids[i], NULL, sender, &channel) != 0) { - // LOG(ERROR) << "Fail to create bthread"; - // return -1; - // } - // } - // } - - // LOG(INFO) << "couchbase_client is going to quit"; - // for (int i = 0; i < batch_size; ++i) { - // if (!FLAGS_use_bthread) { - // pthread_join(pids[i], NULL); - // } else { - // bthread_join(bids[i], NULL); - // } - // } - - // // Calculate and print average response time - // if (!thread_response_times.empty()) { - // double total_time = 0.0; - // for (double time : thread_response_times) { - // total_time += time; - // } - // double average_response_time = total_time / - // thread_response_times.size(); - - // std::cout << "\n=== Performance Summary ===" << std::endl; - // std::cout << "Authentication time: " << auth_time_ms << " ms" << - // std::endl; std::cout << "Bucket selection time: " << bucket_time_ms << - // " ms" << std::endl; std::cout << "ADD operations time: " << add_time_ms - // << " ms" << std::endl; std::cout << "Total threads: " << - // thread_response_times.size() << std::endl; std::cout << "Average GET - // response time: " << average_response_time << " ms" << std::endl; - // std::cout << "Total time for all GET requests: " << total_time << " ms" - // << std::endl; std::cout << "Total authorization time: " << - // (auth_time_ms + bucket_time_ms) << " ms" << std::endl; std::cout << - // "=========================" << std::endl; - // } else { - // std::cout << "\n=== Performance Summary ===" << std::endl; - // std::cout << "Authentication time: " << auth_time_ms << " ms" << - // std::endl; std::cout << "Bucket selection time: " << bucket_time_ms << - // " ms" << std::endl; std::cout << "ADD operations time: " << add_time_ms - // << " ms" << std::endl; std::cout << "Total authorization time: " << - // (auth_time_ms + bucket_time_ms) << " ms" << std::endl; std::cout << "No - // successful GET requests completed." << std::endl; std::cout << - // "=========================" << std::endl; - // } + cntl.Reset(); // Add operation brpc::CouchbaseRequest add_request; @@ -382,13 +165,9 @@ int main() { LOG(ERROR) << "Fail to ADD request"; return -1; } - auto add_start_time = std::chrono::high_resolution_clock::now(); + channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); - auto add_end_time = std::chrono::high_resolution_clock::now(); - auto add_duration = std::chrono::duration_cast( - add_end_time - add_start_time); - operation_times.push_back( - {"Add user data (first attempt) in microsecond", add_duration.count()}); + if (cntl.Failed()) { LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); return -1; @@ -412,14 +191,7 @@ int main() { LOG(ERROR) << "Fail to ADD request"; return -1; } - add_start_time = std::chrono::high_resolution_clock::now(); channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast( - add_end_time - add_start_time); - operation_times.push_back( - {"Add user data (Second attempt - expected failure) in microsecond ", - add_duration.count()}); // Check second ADD operation response status (should fail with key exists) if (cntl.Failed()) { @@ -444,13 +216,7 @@ int main() { LOG(ERROR) << "Fail to GET request"; return -1; } - add_start_time = std::chrono::high_resolution_clock::now(); channel.CallMethod(NULL, &cntl, &get_request, &get_response, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast( - add_end_time - add_start_time); - operation_times.push_back( - {"Get user data in microsecond ", add_duration.count()}); if (cntl.Failed()) { LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); return -1; @@ -488,13 +254,7 @@ int main() { } std::cout << "Sending ADD request for binprot_item1, pipelined count: " << add_request1.pipelined_count() << std::endl; - add_start_time = std::chrono::high_resolution_clock::now(); channel.CallMethod(NULL, &cntl, &add_request1, &add_response1, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast( - add_end_time - add_start_time); - operation_times.push_back( - {"Add binprot item1 in microsecond", add_duration.count()}); if (cntl.Failed()) { LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); return -1; @@ -524,13 +284,7 @@ int main() { } std::cout << "Sending ADD request for binprot_item2, pipelined count: " << add_request2.pipelined_count() << std::endl; - add_start_time = std::chrono::high_resolution_clock::now(); channel.CallMethod(NULL, &cntl, &add_request2, &add_response2, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast( - add_end_time - add_start_time); - operation_times.push_back( - {"Add binprot item2 in microsecond", add_duration.count()}); if (cntl.Failed()) { LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); return -1; @@ -559,13 +313,7 @@ int main() { } std::cout << "Sending ADD request for binprot_item3, pipelined count: " << add_request3.pipelined_count() << std::endl; - add_start_time = std::chrono::high_resolution_clock::now(); channel.CallMethod(NULL, &cntl, &add_request3, &add_response3, NULL); - add_end_time = std::chrono::high_resolution_clock::now(); - add_duration = std::chrono::duration_cast( - add_end_time - add_start_time); - operation_times.push_back( - {"Add binprot item3 in microsecond", add_duration.count()}); if (cntl.Failed()) { LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); return -1; @@ -835,18 +583,177 @@ int main() { cntl.Reset(); - // Print operation times - long long total_time = 0; - for (const auto& op : operation_times) { - std::cout << std::left << std::setw(40) << op.first << ": "; - if (op.second >= 1000) { - std::cout << std::right << std::setw(8) << (op.second / 1000.0) << " ms" - << std::endl; + // Retrieve Collection ID for scope `_default` and collection + // `testing_collection` + + brpc::CouchbaseRequest get_collection_request; + brpc::CouchbaseResponse get_collection_response; + uint8_t testing_collection_id = 0; // will hold the numeric collection id + const std::string scope_name = "_default"; // default scope + std::string collection_name = "testing_collection"; // target collection + // enter collection name as user input + std::cout << "Enter collection name (default 'testing_collection'): "; + std::string user_input; + std::cin >> user_input; + if (!user_input.empty()) { + collection_name = user_input; + } + + if (!get_collection_request.GetCollectionId(scope_name.c_str(), + collection_name.c_str())) { + LOG(ERROR) << "Fail to build GetCollectionId request for scope '" + << scope_name << "' collection '" << collection_name << "'"; + return -1; + } + + channel.CallMethod(NULL, &cntl, &get_collection_request, + &get_collection_response, NULL); + + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access Couchbase for GetCollectionId, " + << cntl.ErrorText(); + return -1; + } + + if (get_collection_response.PopCollectionId(&testing_collection_id)) { + std::cout << GREEN << "Retrieved collection id for _default." + << collection_name << " = " + << static_cast(testing_collection_id) + << " dec=" << static_cast(testing_collection_id) + << ", hex=0x" << std::hex + << static_cast(testing_collection_id) << RESET + << std::endl; + } else { + std::cout << RED << "Failed to retrieve collection id for _default." + << collection_name << ": " << get_collection_response.LastError() + << RESET << std::endl; + // We continue, but subsequent collection operations will skip if id=0 + } + cntl.Reset(); + // ------------------------------------------------------------------ + // Collection-scoped CRUD operations (only if collection id was retrieved) + // ------------------------------------------------------------------ + if (testing_collection_id != 0) { + // 1. ADD in collection + brpc::CouchbaseRequest coll_add_req; + brpc::CouchbaseResponse coll_add_resp; + const std::string coll_key = "user::collection_doc"; + if (!coll_add_req.Add( + coll_key.c_str(), R"({"type":"collection","op":"add","v":1})", + 0xabcddcba, FLAGS_exptime, 0, (uint8_t)testing_collection_id)) { + LOG(ERROR) << "Fail to build collection ADD request"; } else { - std::cout << std::right << std::setw(8) << op.second << " μs" - << std::endl; + channel.CallMethod(NULL, &cntl, &coll_add_req, &coll_add_resp, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Collection ADD RPC failed: " << cntl.ErrorText(); + } else { + uint64_t ccas; + if (coll_add_resp.PopAdd(&ccas)) { + std::cout << GREEN << "Collection ADD success, CAS=" << ccas << RESET + << std::endl; + } else { + std::cout << RED + << "Collection ADD failed: " << coll_add_resp.LastError() + << RESET << std::endl; + } + } + cntl.Reset(); } - total_time += op.second; + + // 2. GET from collection + brpc::CouchbaseRequest coll_get_req; + brpc::CouchbaseResponse coll_get_resp; + if (!coll_get_req.Get(coll_key.c_str(), (uint8_t)testing_collection_id)) { + LOG(ERROR) << "Fail to build collection GET request"; + } else { + channel.CallMethod(NULL, &cntl, &coll_get_req, &coll_get_resp, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Collection GET RPC failed: " << cntl.ErrorText(); + } else { + std::string v; + uint32_t f = 0; + uint64_t ccas = 0; + if (coll_get_resp.PopGet(&v, &f, &ccas)) { + std::cout << GREEN << "Collection GET success value=" << v + << ", CAS=" << ccas << RESET << std::endl; + } else { + std::cout << RED + << "Collection GET failed: " << coll_get_resp.LastError() + << RESET << std::endl; + } + } + cntl.Reset(); + } + + // 3. UPSERT in collection + brpc::CouchbaseRequest coll_upsert_req; + brpc::CouchbaseResponse coll_upsert_resp; + if (!coll_upsert_req.Upsert( + coll_key.c_str(), R"({"type":"collection","op":"upsert","v":2})", 0, + FLAGS_exptime, 0, (uint8_t)testing_collection_id)) { + LOG(ERROR) << "Fail to build collection UPSERT request"; + } else { + channel.CallMethod(NULL, &cntl, &coll_upsert_req, &coll_upsert_resp, + NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Collection UPSERT RPC failed: " << cntl.ErrorText(); + } else { + uint64_t ccas; + if (coll_upsert_resp.PopUpsert(&ccas)) { + std::cout << GREEN << "Collection UPSERT success, CAS=" << ccas + << RESET << std::endl; + } else { + std::cout << RED << "Collection UPSERT failed: " + << coll_upsert_resp.LastError() << RESET << std::endl; + } + } + cntl.Reset(); + } + + // 4. GET again to verify upsert + brpc::CouchbaseRequest coll_get2_req; + brpc::CouchbaseResponse coll_get2_resp; + if (coll_get2_req.Get(coll_key.c_str(), (uint8_t)testing_collection_id)) { + channel.CallMethod(NULL, &cntl, &coll_get2_req, &coll_get2_resp, NULL); + if (!cntl.Failed()) { + std::string v; + uint32_t f = 0; + uint64_t ccas = 0; + if (coll_get2_resp.PopGet(&v, &f, &ccas)) { + std::cout << GREEN << "Collection GET(after upsert) value=" << v + << RESET << std::endl; + } + } + cntl.Reset(); + } + + // 5. DELETE from collection + brpc::CouchbaseRequest coll_del_req; + brpc::CouchbaseResponse coll_del_resp; + if (!coll_del_req.Delete(coll_key.c_str(), + (uint8_t)testing_collection_id)) { + LOG(ERROR) << "Fail to build collection DELETE request"; + } else { + channel.CallMethod(NULL, &cntl, &coll_del_req, &coll_del_resp, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Collection DELETE RPC failed: " << cntl.ErrorText(); + } else { + if (coll_del_resp.PopDelete()) { + std::cout << GREEN << "Collection DELETE success" << RESET + << std::endl; + } else { + std::cout << RED + << "Collection DELETE failed: " << coll_del_resp.LastError() + << RESET << std::endl; + } + } + cntl.Reset(); + } + } else { + std::cout << RED + << "Skipping collection-scoped CRUD operations (collection id " + "not available)" + << RESET << std::endl; } return 0; } diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 89712a3c7c..64f7fc4721 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -434,8 +434,6 @@ size_t CouchbaseResponse::ByteSizeLong() const { void CouchbaseResponse::MergeFrom(const CouchbaseResponse& from) { CHECK_NE(&from, this); _err = from._err; - // responses of memcached according to their binary layout, should be - // directly concatenatible. _buf.append(from._buf); } @@ -594,9 +592,10 @@ std::string CouchbaseResponse::format_error_message( // MUST have key. // MUST NOT have value. bool CouchbaseRequest::GetOrDelete(uint8_t command, - const butil::StringPiece& key) { + const butil::StringPiece& key, + uint8_t coll_id) { // Collection ID - uint8_t collection_id = 0; + uint8_t collection_id = coll_id; uint16_t VBucket_id = hash_crc32(key.data(), key.size()); const policy::CouchbaseRequestHeader header = { policy::CB_MAGIC_REQUEST, command, @@ -623,12 +622,12 @@ bool CouchbaseRequest::GetOrDelete(uint8_t command, return true; } -bool CouchbaseRequest::Get(const butil::StringPiece& key) { - return GetOrDelete(policy::CB_BINARY_GET, key); +bool CouchbaseRequest::Get(const butil::StringPiece& key, uint8_t coll_id) { + return GetOrDelete(policy::CB_BINARY_GET, key, coll_id); } -bool CouchbaseRequest::Delete(const butil::StringPiece& key) { - return GetOrDelete(policy::CB_BINARY_DELETE, key); +bool CouchbaseRequest::Delete(const butil::StringPiece& key, uint8_t coll_id) { + return GetOrDelete(policy::CB_BINARY_DELETE, key, coll_id); } struct FlushHeaderWithExtras { @@ -795,18 +794,19 @@ const size_t STORE_EXTRAS = // Total 8 bytes bool CouchbaseRequest::Store(uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value) { + uint32_t exptime, uint64_t cas_value, + uint8_t coll_id) { // add collection id // uint16_t collection_id = 0x00; - uint8_t collection_id = 0x00; + uint8_t collection_id = coll_id; uint16_t vBucket_id = hash_crc32(key.data(), key.size()); StoreHeaderWithExtras header_with_extras = { {policy::CB_MAGIC_REQUEST, command, - butil::HostToNet16( - key.size() + - 1), // collection id is part of key, so including it in key length. + butil::HostToNet16(key.size() + + 1), // collection id is not included in part of key, + // so not including it in key length. STORE_EXTRAS, policy::CB_JSON, butil::HostToNet16(vBucket_id), - butil::HostToNet32(STORE_EXTRAS + sizeof(uint8_t) + key.size() + + butil::HostToNet32(STORE_EXTRAS + sizeof(collection_id) + key.size() + value.size()), // total body length 0, butil::HostToNet64(cas_value)}, butil::HostToNet32(flags), @@ -814,7 +814,7 @@ bool CouchbaseRequest::Store(uint8_t command, const butil::StringPiece& key, if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { return false; } - if (_buf.append(&collection_id, sizeof(uint8_t))) { + if (_buf.append(&collection_id, sizeof(collection_id))) { return false; } if (_buf.append(key.data(), key.size())) { @@ -857,9 +857,12 @@ bool CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { if (value_size > 0) { std::string error_msg; _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "STORE operation", error_msg); + _err = format_error_message(header.status, + couchbase_binary_command_to_string(command), + error_msg); } else { - _err = format_error_message(header.status, "STORE operation"); + _err = format_error_message(header.status, + couchbase_binary_command_to_string(command)); } return false; } @@ -867,39 +870,110 @@ bool CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { << "STORE response must not have value, actually=" << value_size; _buf.pop_front(sizeof(header) + header.total_body_length); if (cas_value) { - CHECK(header.cas_value); *cas_value = header.cas_value; } _err.clear(); return true; } -bool CouchbaseRequest::Upsert(const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value) { - return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value); +const char* CouchbaseResponse::couchbase_binary_command_to_string(uint8_t cmd) { + switch (cmd) { + case 0x1f: + return "CB_HELLO_SELECT_FEATURES"; + case 0x89: + return "CB_SELECT_BUCKET"; + case 0xBC: + return "CB_GET_SCOPE_ID"; + case 0x00: + return "CB_BINARY_GET"; + case 0x01: + return "CB_BINARY_SET"; + case 0x02: + return "CB_BINARY_ADD"; + case 0x03: + return "CB_BINARY_REPLACE"; + case 0x04: + return "CB_BINARY_DELETE"; + case 0x05: + return "CB_BINARY_INCREMENT"; + case 0x06: + return "CB_BINARY_DECREMENT"; + case 0x07: + return "CB_BINARY_QUIT"; + case 0x08: + return "CB_BINARY_FLUSH"; + case 0x09: + return "CB_BINARY_GETQ"; + case 0x0a: + return "CB_BINARY_NOOP"; + case 0x0b: + return "CB_BINARY_VERSION"; + case 0x0c: + return "CB_BINARY_GETK"; + case 0x0d: + return "CB_BINARY_GETKQ"; + case 0x0e: + return "CB_BINARY_APPEND"; + case 0x0f: + return "CB_BINARY_PREPEND"; + case 0x10: + return "CB_BINARY_STAT"; + case 0x11: + return "CB_BINARY_SETQ"; + case 0x12: + return "CB_BINARY_ADDQ"; + case 0x13: + return "CB_BINARY_REPLACEQ"; + case 0x14: + return "CB_BINARY_DELETEQ"; + case 0x15: + return "CB_BINARY_INCREMENTQ"; + case 0x16: + return "CB_BINARY_DECREMENTQ"; + case 0x17: + return "CB_BINARY_QUITQ"; + case 0x18: + return "CB_BINARY_FLUSHQ"; + case 0x19: + return "CB_BINARY_APPENDQ"; + case 0x1a: + return "CB_BINARY_PREPENDQ"; + case 0x1c: + return "CB_BINARY_TOUCH"; + case 0x1d: + return "CB_BINARY_GAT"; + case 0x1e: + return "CB_BINARY_GATQ"; + case 0x23: + return "CB_BINARY_GATK"; + case 0x24: + return "CB_BINARY_GATKQ"; + case 0x20: + return "CB_BINARY_SASL_LIST_MECHS"; + case 0x21: + return "CB_BINARY_SASL_AUTH"; + case 0x22: + return "CB_BINARY_SASL_STEP"; + case 0xb5: + return "CB_GET_CLUSTER_CONFIG"; + case 0xba: + return "CB_GET_COLLECTIONS_MANIFEST"; + case 0xbb: + return "CB_COLLECTIONS_GET_CID"; + default: + return "UNKNOWN_COMMAND"; + } } -// Collection Management Methods -bool CouchbaseRequest::GetCollectionsManifest() { - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - policy::CB_GET_COLLECTIONS_MANIFEST, - 0, // no key - 0, // no extras - policy::CB_BINARY_RAW_BYTES, - 0, // no vbucket - 0, // no body - 0, // opaque - 0 // no CAS - }; - if (_buf.append(&header, sizeof(header))) { - return false; - } - ++_pipelined_count; - return true; +bool CouchbaseRequest::Upsert(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value, + uint8_t coll_id) { + return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value, + coll_id); } +// collection id and manifest are both returned when you call GetCollectionId bool CouchbaseRequest::GetCollectionId( const butil::StringPiece& scope_name, const butil::StringPiece& collection_name) { @@ -928,58 +1002,44 @@ bool CouchbaseRequest::GetCollectionId( return true; } -// Collection-aware document operations -bool CouchbaseRequest::GetFromCollection( - const butil::StringPiece& key, const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name) { - // Construct a collection-aware key: collection::key - std::string scoped_key = collection_name.as_string() + "::" + key.as_string(); - return GetOrDelete(policy::CB_BINARY_GET, butil::StringPiece(scoped_key)); -} - -bool CouchbaseRequest::UpsertToCollection( - const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name) { - // Construct a collection-aware key: collection::key - std::string scoped_key = collection_name.as_string() + "::" + key.as_string(); - return Store(policy::CB_BINARY_SET, butil::StringPiece(scoped_key), value, - flags, exptime, cas_value); -} - bool CouchbaseRequest::Add(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value) { - return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value); + uint32_t exptime, uint64_t cas_value, + uint8_t coll_id) { + return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, + coll_id); } bool CouchbaseRequest::Replace(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value) { - return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, - cas_value); + uint32_t exptime, uint64_t cas_value, + uint8_t coll_id) { + return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, cas_value, + coll_id); } bool CouchbaseRequest::Append(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value) { + uint32_t exptime, uint64_t cas_value, + uint8_t coll_id) { if (value.empty()) { LOG(ERROR) << "value to append must be non-empty"; return false; } - return Store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value); + return Store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value, + coll_id); } bool CouchbaseRequest::Prepend(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value) { + uint32_t exptime, uint64_t cas_value, + uint8_t coll_id) { if (value.empty()) { LOG(ERROR) << "value to prepend must be non-empty"; return false; } - return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, - cas_value); + return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, cas_value, + coll_id); } bool CouchbaseResponse::PopUpsert(uint64_t* cas_value) { @@ -997,57 +1057,11 @@ bool CouchbaseResponse::PopAppend(uint64_t* cas_value) { bool CouchbaseResponse::PopPrepend(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_PREPEND, cas_value); } - -// Collection-related response methods -bool CouchbaseResponse::PopCollectionsManifest(std::string* manifest_json) { - const size_t n = _buf.size(); - policy::CouchbaseResponseHeader header; - if (n < sizeof(header)) { - butil::string_printf(&_err, "buffer is too small to contain a header"); - return false; - } - _buf.copy_to(&header, sizeof(header)); - if (header.command != policy::CB_GET_COLLECTIONS_MANIFEST) { - butil::string_printf(&_err, "Not a collections manifest response"); - return false; - } - if (n < sizeof(header) + header.total_body_length) { - butil::string_printf(&_err, "Not enough data"); - return false; - } - if (header.status != 0) { - _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; - if (value_size > 0) { - std::string error_msg; - _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "Collections manifest request", - error_msg); - } else { - _err = - format_error_message(header.status, "Collections manifest request"); - } - return false; - } - - // Skip header and extras/key if any - _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - - // Get the manifest JSON - int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; - if (manifest_json && value_size > 0) { - _buf.cutn(manifest_json, value_size); - } else { - _buf.pop_front(value_size); - } - - _err.clear(); - return true; +bool CouchbaseResponse::PopSelectBucket(uint64_t* cas_value) { + return PopStore(policy::CB_SELECT_BUCKET, cas_value); } - -bool CouchbaseResponse::PopCollectionId(uint32_t* collection_id) { +// Collection-related response method +bool CouchbaseResponse::PopCollectionId(uint8_t* collection_id) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1055,61 +1069,81 @@ bool CouchbaseResponse::PopCollectionId(uint32_t* collection_id) { return false; } _buf.copy_to(&header, sizeof(header)); + if (header.command != policy::CB_COLLECTIONS_GET_CID) { butil::string_printf(&_err, "Not a collection ID response"); return false; } + + // Making sure buffer has the whole body (extras + key + value) if (n < sizeof(header) + header.total_body_length) { butil::string_printf(&_err, "Not enough data"); return false; } + if (header.status != 0) { + // handle error case _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; + // Possibly read error message from value if present + size_t value_size = + header.total_body_length - header.extras_length - header.key_length; if (value_size > 0) { - std::string error_msg; - _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "Collection ID request", - error_msg); + std::string err_msg; + _buf.cutn(&err_msg, value_size); + _err = + format_error_message(header.status, "Collection ID request", err_msg); } else { _err = format_error_message(header.status, "Collection ID request"); } return false; } - // Skip header and extras/key if any - _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); - - // Get the collection ID (typically 4 bytes) - int value_size = (int)header.total_body_length - (int)header.extras_length - - (int)header.key_length; - if (collection_id && value_size >= 4) { - uint32_t cid; - _buf.copy_to(&cid, 4); - *collection_id = butil::NetToHost32(cid); - _buf.pop_front(value_size); - } else { - _buf.pop_front(value_size); + // Success case: we expect extras_length >= 12 (8 bytes manifest + 4 bytes + // collection id) + if (header.extras_length < 12) { + butil::string_printf(&_err, "Extras too small to contain collection ID"); + // remove the response from buffer so you don't re‐process + _buf.pop_front(sizeof(header) + header.total_body_length); + return false; } + // Skip header + _buf.pop_front(sizeof(header)); + + // LOG(INFO) << "Total body length: " << header.total_body_length; + // LOG(INFO) << "Extras length: " << header.extras_length; + + // size_t remaining_size = _buf.size(); + // std::vector remaining_data(remaining_size); + // _buf.copy_to(remaining_data.data(), remaining_size); + + // // Print the remaining data + // for (size_t i = 0; i < remaining_size; ++i) { + // LOG(INFO) << "Byte " << i << ": " << + // static_cast(remaining_data[i]); + // } + + // return true; + uint64_t manifest_id_net = 0; + _buf.copy_to(reinterpret_cast(&manifest_id_net), + sizeof(manifest_id_net)); + // You may convert this if needed: + uint64_t manifest_id = butil::NetToHost64(manifest_id_net); + LOG(INFO) << "Manifest ID: " << manifest_id; + _buf.pop_front(sizeof(manifest_id_net)); + + // Next 1 bytes → collection ID (u8) + uint32_t cid_net = 0; + _buf.copy_to(reinterpret_cast(&cid_net), sizeof(cid_net)); + uint8_t cid_host = butil::NetToHost32(cid_net); + *collection_id = static_cast(cid_host); + _buf.pop_front(sizeof(cid_net)); + + _buf.pop_front(header.total_body_length); _err.clear(); return true; } -// Collection-aware response methods -bool CouchbaseResponse::PopGetFromCollection(butil::IOBuf* value, - uint32_t* flags, - uint64_t* cas_value) { - // Same implementation as PopGet, just aliased for clarity - return PopGet(value, flags, cas_value); -} - -bool CouchbaseResponse::PopUpsertToCollection(uint64_t* cas_value) { - // Same implementation as PopSet, just aliased for clarity - return PopUpsert(cas_value); -} - struct IncrHeaderWithExtras { policy::CouchbaseRequestHeader header; uint64_t delta; @@ -1159,13 +1193,15 @@ bool CouchbaseRequest::Counter(uint8_t command, const butil::StringPiece& key, } bool CouchbaseRequest::Increment(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime) { + uint64_t initial_value, uint32_t exptime, + uint8_t coll_id) { return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, exptime); } bool CouchbaseRequest::Decrement(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime) { + uint64_t initial_value, uint32_t exptime, + uint8_t coll_id) { return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, exptime); } @@ -1266,7 +1302,8 @@ const size_t TOUCH_EXTRAS = // 0| Expiration | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseRequest::Touch(const butil::StringPiece& key, uint32_t exptime) { +bool CouchbaseRequest::Touch(const butil::StringPiece& key, uint32_t exptime, + uint8_t coll_id) { TouchHeaderWithExtras header_with_extras = { {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_TOUCH, butil::HostToNet16(key.size()), TOUCH_EXTRAS, diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 392e5b438d..2a05d5f015 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -36,13 +36,14 @@ class CouchbaseRequest : public NonreflectableMessage { void SharedCtor(); void SharedDtor(); void SetCachedSize(int size) const PB_425_OVERRIDE; - bool GetOrDelete(uint8_t command, const butil::StringPiece& key); + bool GetOrDelete(uint8_t command, const butil::StringPiece& key, + uint8_t coll_id = 0); bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, uint64_t initial_value, uint32_t exptime); bool Store(uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, - uint64_t cas_value); + uint64_t cas_value, uint8_t coll_id = 0); uint32_t hash_crc32(const char* key, size_t key_length); public: @@ -59,48 +60,45 @@ class CouchbaseRequest : public NonreflectableMessage { const butil::StringPiece& password); bool HelloRequest(); - // Collection Management Methods - bool GetCollectionsManifest(); + // Collection Management Method bool GetCollectionId(const butil::StringPiece& scope_name, const butil::StringPiece& collection_name); bool GetScopeId(const butil::StringPiece& scope_name); // Collection-aware document operations - bool Get(const butil::StringPiece& key); - bool GetFromCollection(const butil::StringPiece& key, - const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name); + bool Get(const butil::StringPiece& key, uint8_t coll_id = 0); bool Upsert(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); - bool UpsertToCollection(const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value, - const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name); + uint32_t flags, uint32_t exptime, uint64_t cas_value, + uint8_t coll_id = 0); bool Add(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); + uint32_t flags, uint32_t exptime, uint64_t cas_value, + uint8_t coll_id = 0); bool Replace(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); + uint32_t flags, uint32_t exptime, uint64_t cas_value, + uint8_t coll_id = 0); bool Append(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); + uint32_t flags, uint32_t exptime, uint64_t cas_value, + uint8_t coll_id = 0); bool Prepend(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value); + uint32_t flags, uint32_t exptime, uint64_t cas_value, + uint8_t coll_id = 0); - bool Delete(const butil::StringPiece& key); + bool Delete(const butil::StringPiece& key, uint8_t coll_id = 0); bool Flush(uint32_t timeout); bool Increment(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime); + uint64_t initial_value, uint32_t exptime, uint8_t coll_id = 0); bool Decrement(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime); + uint64_t initial_value, uint32_t exptime, uint8_t coll_id = 0); - bool Touch(const butil::StringPiece& key, uint32_t exptime); + bool Touch(const butil::StringPiece& key, uint32_t exptime, + uint8_t coll_id = 0); bool Version(); @@ -200,7 +198,7 @@ class CouchbaseResponse : public NonreflectableMessage { STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, STATUS_XATTR_EINVAL = 0xe0 }; - + const char* couchbase_binary_command_to_string(uint8_t cmd); void MergeFrom(const CouchbaseResponse& from) override; void Clear() override; bool IsInitialized() const PB_527_OVERRIDE; @@ -233,15 +231,10 @@ class CouchbaseResponse : public NonreflectableMessage { bool PopReplace(uint64_t* cas_value); bool PopAppend(uint64_t* cas_value); bool PopPrepend(uint64_t* cas_value); + bool PopSelectBucket(uint64_t* cas_value); // Collection-related response methods - bool PopCollectionsManifest(std::string* manifest_json); - bool PopCollectionId(uint32_t* collection_id); - - // Collection-aware response methods (same as regular but for documentation) - bool PopGetFromCollection(butil::IOBuf* value, uint32_t* flags, - uint64_t* cas_value); - bool PopUpsertToCollection(uint64_t* cas_value); + bool PopCollectionId(uint8_t* collection_id); bool PopDelete(); bool PopFlush(); diff --git a/src/brpc/policy/couchbase_protocol.cpp b/src/brpc/policy/couchbase_protocol.cpp index 695f7927c1..f559df7cb5 100644 --- a/src/brpc/policy/couchbase_protocol.cpp +++ b/src/brpc/policy/couchbase_protocol.cpp @@ -15,26 +15,26 @@ // specific language governing permissions and limitations // under the License. +#include "brpc/policy/couchbase_protocol.h" -#include // MethodDescriptor -#include // Message #include -#include "butil/logging.h" // LOG() -#include "butil/time.h" -#include "butil/iobuf.h" // butil::IOBuf -#include "butil/sys_byteorder.h" -#include "brpc/controller.h" // Controller +#include // MethodDescriptor +#include // Message + +#include "brpc/compress.h" // ParseFromCompressedData +#include "brpc/controller.h" // Controller +#include "brpc/couchbase.h" // CouchbaseRequest, CouchbaseResponse #include "brpc/details/controller_private_accessor.h" -#include "brpc/socket.h" // Socket -#include "brpc/server.h" // Server #include "brpc/details/server_private_accessor.h" -#include "brpc/span.h" -#include "brpc/compress.h" // ParseFromCompressedData -#include "brpc/policy/couchbase_protocol.h" -#include "brpc/couchbase.h" // CouchbaseRequest, CouchbaseResponse #include "brpc/policy/most_common_message.h" +#include "brpc/server.h" // Server +#include "brpc/socket.h" // Socket +#include "brpc/span.h" #include "butil/containers/flat_map.h" - +#include "butil/iobuf.h" // butil::IOBuf +#include "butil/logging.h" // LOG() +#include "butil/sys_byteorder.h" +#include "butil/time.h" namespace brpc { @@ -49,199 +49,198 @@ static uint64_t supported_cmd_map[8]; static pthread_once_t supported_cmd_map_once = PTHREAD_ONCE_INIT; static void InitSupportedCommandMap() { - butil::bit_array_clear(supported_cmd_map, 256); - butil::bit_array_set(supported_cmd_map, CB_BINARY_GET); - butil::bit_array_set(supported_cmd_map, CB_HELLO_SELECT_FEATURES); - butil::bit_array_set(supported_cmd_map, CB_SELECT_BUCKET); - butil::bit_array_set(supported_cmd_map, CB_GET_SCOPE_ID); - butil::bit_array_set(supported_cmd_map, CB_BINARY_SET); - butil::bit_array_set(supported_cmd_map, CB_BINARY_ADD); - butil::bit_array_set(supported_cmd_map, CB_BINARY_REPLACE); - butil::bit_array_set(supported_cmd_map, CB_BINARY_DELETE); - butil::bit_array_set(supported_cmd_map, CB_BINARY_INCREMENT); - butil::bit_array_set(supported_cmd_map, CB_BINARY_DECREMENT); - butil::bit_array_set(supported_cmd_map, CB_BINARY_FLUSH); - butil::bit_array_set(supported_cmd_map, CB_BINARY_VERSION); - butil::bit_array_set(supported_cmd_map, CB_BINARY_NOOP); - butil::bit_array_set(supported_cmd_map, CB_BINARY_APPEND); - butil::bit_array_set(supported_cmd_map, CB_BINARY_PREPEND); - butil::bit_array_set(supported_cmd_map, CB_BINARY_STAT); - butil::bit_array_set(supported_cmd_map, CB_BINARY_TOUCH); - butil::bit_array_set(supported_cmd_map, CB_BINARY_SASL_AUTH); - // Collection management commands - butil::bit_array_set(supported_cmd_map, CB_GET_COLLECTIONS_MANIFEST); - butil::bit_array_set(supported_cmd_map, CB_COLLECTIONS_GET_CID); - butil::bit_array_set(supported_cmd_map, CB_COLLECTIONS_GET_SCOPE_ID); + butil::bit_array_clear(supported_cmd_map, 256); + butil::bit_array_set(supported_cmd_map, CB_BINARY_GET); + butil::bit_array_set(supported_cmd_map, CB_HELLO_SELECT_FEATURES); + butil::bit_array_set(supported_cmd_map, CB_SELECT_BUCKET); + butil::bit_array_set(supported_cmd_map, CB_GET_SCOPE_ID); + butil::bit_array_set(supported_cmd_map, CB_BINARY_SET); + butil::bit_array_set(supported_cmd_map, CB_BINARY_ADD); + butil::bit_array_set(supported_cmd_map, CB_BINARY_REPLACE); + butil::bit_array_set(supported_cmd_map, CB_BINARY_DELETE); + butil::bit_array_set(supported_cmd_map, CB_BINARY_INCREMENT); + butil::bit_array_set(supported_cmd_map, CB_BINARY_DECREMENT); + butil::bit_array_set(supported_cmd_map, CB_BINARY_FLUSH); + butil::bit_array_set(supported_cmd_map, CB_BINARY_VERSION); + butil::bit_array_set(supported_cmd_map, CB_BINARY_NOOP); + butil::bit_array_set(supported_cmd_map, CB_BINARY_APPEND); + butil::bit_array_set(supported_cmd_map, CB_BINARY_PREPEND); + butil::bit_array_set(supported_cmd_map, CB_BINARY_STAT); + butil::bit_array_set(supported_cmd_map, CB_BINARY_TOUCH); + butil::bit_array_set(supported_cmd_map, CB_BINARY_SASL_AUTH); + // Collection management commands + butil::bit_array_set(supported_cmd_map, CB_GET_COLLECTIONS_MANIFEST); + butil::bit_array_set(supported_cmd_map, CB_COLLECTIONS_GET_CID); + butil::bit_array_set(supported_cmd_map, CB_COLLECTIONS_GET_SCOPE_ID); } inline bool IsSupportedCommand(uint8_t command) { - pthread_once(&supported_cmd_map_once, InitSupportedCommandMap); - return butil::bit_array_get(supported_cmd_map, command); + pthread_once(&supported_cmd_map_once, InitSupportedCommandMap); + return butil::bit_array_get(supported_cmd_map, command); } -ParseResult ParseCouchbaseMessage(butil::IOBuf* source, - Socket* socket, bool /*read_eof*/, const void */*arg*/) { - while (1) { - const uint8_t* p_mcmagic = (const uint8_t*)source->fetch1(); - if (NULL == p_mcmagic) { - return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); - } - if (*p_mcmagic != (uint8_t)CB_MAGIC_RESPONSE) { - return MakeParseError(PARSE_ERROR_TRY_OTHERS); - } - char buf[24]; - const uint8_t* p = (const uint8_t*)source->fetch(buf, sizeof(buf)); - if (NULL == p) { - return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); - } - const CouchbaseResponseHeader* header = (const CouchbaseResponseHeader*)p; - uint32_t total_body_length = butil::NetToHost32(header->total_body_length); - if (source->size() < sizeof(*header) + total_body_length) { - return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); - } - - if (!IsSupportedCommand(header->command)) { - LOG(WARNING) << "Not support command=" << header->command; - source->pop_front(sizeof(*header) + total_body_length); - return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); - } - - PipelinedInfo pi; - if (!socket->PopPipelinedInfo(&pi)) { - LOG(WARNING) << "No corresponding PipelinedInfo in socket, drop"; - source->pop_front(sizeof(*header) + total_body_length); - return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); - } - MostCommonMessage* msg = - static_cast(socket->parsing_context()); - if (msg == NULL) { - msg = MostCommonMessage::Get(); - socket->reset_parsing_context(msg); - } - - // endianness conversions. - const CouchbaseResponseHeader local_header = { - header->magic, - header->command, - butil::NetToHost16(header->key_length), - header->extras_length, - header->data_type, - butil::NetToHost16(header->status), - total_body_length, - butil::NetToHost32(header->opaque), - butil::NetToHost64(header->cas_value), - }; - msg->meta.append(&local_header, sizeof(local_header)); - source->pop_front(sizeof(*header)); - source->cutn(&msg->meta, total_body_length); - if (header->command == CB_BINARY_SASL_AUTH) { - if (header->status != 0) { - LOG(ERROR) << "Failed to authenticate the couchbase Server."; - return MakeParseError(PARSE_ERROR_NO_RESOURCE, - "Fail to authenticate with the couchbase Server"); - } - msg = static_cast(socket->release_parsing_context()); - msg->pi = pi; - return MakeMessage(msg); - } else { - if (++msg->pi.count >= pi.count) { - CHECK_EQ(msg->pi.count, pi.count); - msg = static_cast(socket->release_parsing_context()); - msg->pi = pi; - return MakeMessage(msg); - } else { - socket->GivebackPipelinedInfo(pi); - } - } +ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket* socket, + bool /*read_eof*/, const void* /*arg*/) { + while (1) { + const uint8_t* p_cbmagic = (const uint8_t*)source->fetch1(); + if (NULL == p_cbmagic) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + if (*p_cbmagic != (uint8_t)CB_MAGIC_RESPONSE) { + return MakeParseError(PARSE_ERROR_TRY_OTHERS); + } + char buf[24]; + const uint8_t* p = (const uint8_t*)source->fetch(buf, sizeof(buf)); + if (NULL == p) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + const CouchbaseResponseHeader* header = (const CouchbaseResponseHeader*)p; + uint32_t total_body_length = butil::NetToHost32(header->total_body_length); + if (source->size() < sizeof(*header) + total_body_length) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); } -} -void ProcessCouchbaseResponse(InputMessageBase* msg_base) { - const int64_t start_parse_us = butil::cpuwide_time_us(); - DestroyingPtr msg(static_cast(msg_base)); - - const bthread_id_t cid = msg->pi.id_wait; - Controller* cntl = NULL; - const int rc = bthread_id_lock(cid, (void**)&cntl); - if (rc != 0) { - LOG_IF(ERROR, rc != EINVAL && rc != EPERM) - << "Fail to lock correlation_id=" << cid << ": " << berror(rc); - return; + if (!IsSupportedCommand(header->command)) { + LOG(WARNING) << "Not support command=" << header->command; + source->pop_front(sizeof(*header) + total_body_length); + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + + PipelinedInfo pi; + if (!socket->PopPipelinedInfo(&pi)) { + LOG(WARNING) << "No corresponding PipelinedInfo in socket, drop"; + source->pop_front(sizeof(*header) + total_body_length); + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); } - - ControllerPrivateAccessor accessor(cntl); - Span* span = accessor.span(); - if (span) { - span->set_base_real_us(msg->base_real_us()); - span->set_received_us(msg->received_us()); - span->set_response_size(msg->meta.length()); - span->set_start_parse_us(start_parse_us); + MostCommonMessage* msg = + static_cast(socket->parsing_context()); + if (msg == NULL) { + msg = MostCommonMessage::Get(); + socket->reset_parsing_context(msg); } - const int saved_error = cntl->ErrorCode(); - if (cntl->response() == NULL) { - cntl->SetFailed(ERESPONSE, "response is NULL!"); - } else if (cntl->response()->GetDescriptor() != CouchbaseResponse::descriptor()) { - cntl->SetFailed(ERESPONSE, "Must be CouchbaseResponse"); + + // endianness conversions. + const CouchbaseResponseHeader local_header = { + header->magic, + header->command, + butil::NetToHost16(header->key_length), + header->extras_length, + header->data_type, + butil::NetToHost16(header->status), + total_body_length, + butil::NetToHost32(header->opaque), + butil::NetToHost64(header->cas_value), + }; + msg->meta.append(&local_header, sizeof(local_header)); + source->pop_front(sizeof(*header)); + source->cutn(&msg->meta, total_body_length); + if (header->command == CB_BINARY_SASL_AUTH) { + if (header->status != 0) { + LOG(ERROR) << "Failed to authenticate the couchbase Server."; + return MakeParseError(PARSE_ERROR_NO_RESOURCE, + "Fail to authenticate with the couchbase Server"); + } + msg = static_cast(socket->release_parsing_context()); + msg->pi = pi; + return MakeMessage(msg); } else { - // We work around ParseFrom of pb which is just a placeholder. - ((CouchbaseResponse*)cntl->response())->raw_buffer() = msg->meta.movable(); - if (msg->pi.count != accessor.pipelined_count()) { - cntl->SetFailed(ERESPONSE, "pipelined_count=%d of response does " - "not equal request's=%d", - msg->pi.count, accessor.pipelined_count()); - } + if (++msg->pi.count >= pi.count) { + CHECK_EQ(msg->pi.count, pi.count); + msg = + static_cast(socket->release_parsing_context()); + msg->pi = pi; + return MakeMessage(msg); + } else { + socket->GivebackPipelinedInfo(pi); + } } - // Unlocks correlation_id inside. Revert controller's - // error code if it version check of `cid' fails - msg.reset(); // optional, just release resource ASAP - accessor.OnResponse(cid, saved_error); + } } -void SerializeCouchbaseRequest(butil::IOBuf* buf, - Controller* cntl, - const google::protobuf::Message* request) { - if (request == NULL) { - return cntl->SetFailed(EREQUEST, "request is NULL"); - } - if (request->GetDescriptor() != CouchbaseRequest::descriptor()) { - return cntl->SetFailed(EREQUEST, "Must be CouchbaseRequest"); +void ProcessCouchbaseResponse(InputMessageBase* msg_base) { + const int64_t start_parse_us = butil::cpuwide_time_us(); + DestroyingPtr msg( + static_cast(msg_base)); + + const bthread_id_t cid = msg->pi.id_wait; + Controller* cntl = NULL; + const int rc = bthread_id_lock(cid, (void**)&cntl); + if (rc != 0) { + LOG_IF(ERROR, rc != EINVAL && rc != EPERM) + << "Fail to lock correlation_id=" << cid << ": " << berror(rc); + return; + } + + ControllerPrivateAccessor accessor(cntl); + Span* span = accessor.span(); + if (span) { + span->set_base_real_us(msg->base_real_us()); + span->set_received_us(msg->received_us()); + span->set_response_size(msg->meta.length()); + span->set_start_parse_us(start_parse_us); + } + const int saved_error = cntl->ErrorCode(); + if (cntl->response() == NULL) { + cntl->SetFailed(ERESPONSE, "response is NULL!"); + } else if (cntl->response()->GetDescriptor() != + CouchbaseResponse::descriptor()) { + cntl->SetFailed(ERESPONSE, "Must be CouchbaseResponse"); + } else { + // We work around ParseFrom of pb which is just a placeholder. + ((CouchbaseResponse*)cntl->response())->raw_buffer() = msg->meta.movable(); + if (msg->pi.count != accessor.pipelined_count()) { + cntl->SetFailed(ERESPONSE, + "pipelined_count=%d of response does " + "not equal request's=%d", + msg->pi.count, accessor.pipelined_count()); } - const CouchbaseRequest* mr = (const CouchbaseRequest*)request; - // We work around SerializeTo of pb which is just a placeholder. - *buf = mr->raw_buffer(); - ControllerPrivateAccessor(cntl).set_pipelined_count(mr->pipelined_count()); + } + // Unlocks correlation_id inside. Revert controller's + // error code if it version check of `cid' fails + msg.reset(); // optional, just release resource ASAP + accessor.OnResponse(cid, saved_error); +} + +void SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, + const google::protobuf::Message* request) { + if (request == NULL) { + return cntl->SetFailed(EREQUEST, "request is NULL"); + } + if (request->GetDescriptor() != CouchbaseRequest::descriptor()) { + return cntl->SetFailed(EREQUEST, "Must be CouchbaseRequest"); + } + const CouchbaseRequest* mr = (const CouchbaseRequest*)request; + // We work around SerializeTo of pb which is just a placeholder. + *buf = mr->raw_buffer(); + ControllerPrivateAccessor(cntl).set_pipelined_count(mr->pipelined_count()); } -void PackCouchbaseRequest(butil::IOBuf* buf, - SocketMessage**, - uint64_t /*correlation_id*/, - const google::protobuf::MethodDescriptor*, - Controller* cntl, - const butil::IOBuf& request, - const Authenticator* auth) { - if (auth) { - std::cout<<"Appending authentication data to request"<GenerateCredential(&auth_str) != 0) { - return cntl->SetFailed(EREQUEST, "Fail to generate credential"); - } - if (auth_str.empty()) { - return cntl->SetFailed(EREQUEST, "Empty auth_str"); - } - buf->append(auth_str); - // pipelined_count(); +void PackCouchbaseRequest(butil::IOBuf* buf, SocketMessage**, + uint64_t /*correlation_id*/, + const google::protobuf::MethodDescriptor*, + Controller* cntl, const butil::IOBuf& request, + const Authenticator* auth) { + if (auth) { + std::cout << "Appending authentication data to request" << std::endl; + std::string auth_str; + if (auth->GenerateCredential(&auth_str) != 0) { + return cntl->SetFailed(EREQUEST, "Fail to generate credential"); } - else{ - buf->append(request); + if (auth_str.empty()) { + return cntl->SetFailed(EREQUEST, "Empty auth_str"); } + buf->append(auth_str); + // pipelined_count(); + } else { + buf->append(request); + } } const std::string& GetCouchbaseMethodName( - const google::protobuf::MethodDescriptor*, - const Controller*) { - const static std::string CouchbaseD_STR = "Couchbase"; - return CouchbaseD_STR; + const google::protobuf::MethodDescriptor*, const Controller*) { + const static std::string CouchbaseD_STR = "Couchbase"; + return CouchbaseD_STR; } } // namespace policy -} // namespace brpc +} // namespace brpc From 07d1b0c02ba01fc9ca4dee086189080fd15fee96 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Sat, 20 Sep 2025 06:24:30 +0530 Subject: [PATCH 12/49] fixed license issue --- src/brpc/policy/couchbase_protocol.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/brpc/policy/couchbase_protocol.h b/src/brpc/policy/couchbase_protocol.h index 3d15fb6d70..b9caee11f2 100644 --- a/src/brpc/policy/couchbase_protocol.h +++ b/src/brpc/policy/couchbase_protocol.h @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #ifndef BRPC_POLICY_COUCHBASE_BINARY_PROTOCOL_H #define BRPC_POLICY_COUCHBASE_BINARY_PROTOCOL_H From f8defb1bb80debc1c37eb0e398af2f81d4f6a9ae Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Tue, 30 Sep 2025 14:38:34 +0530 Subject: [PATCH 13/49] added custom logic for caching collectionIDs --- .../multithreaded_couchbase_client.cpp | 230 ++++++++++++ src/brpc/couchbase.cpp | 339 ++++++++++++++---- src/brpc/couchbase.h | 72 +++- 3 files changed, 558 insertions(+), 83 deletions(-) create mode 100644 example/couchbase_c++/multithreaded_couchbase_client.cpp diff --git a/example/couchbase_c++/multithreaded_couchbase_client.cpp b/example/couchbase_c++/multithreaded_couchbase_client.cpp new file mode 100644 index 0000000000..db919adb11 --- /dev/null +++ b/example/couchbase_c++/multithreaded_couchbase_client.cpp @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +// ANSI color codes +#define GREEN "\033[32m" +#define RED "\033[31m" +#define BLUE "\033[34m" +#define YELLOW "\033[33m" +#define CYAN "\033[36m" +#define RESET "\033[0m" + +DEFINE_string(server, "127.0.0.1:11210", "IP Address of server"); +DEFINE_string(connection_type, "single", "Connection type"); +DEFINE_int32(operations_per_thread, 50, "Number of operations each thread should perform"); +DEFINE_int32(sleep_ms, 100, "Sleep time between operations in milliseconds"); + +const int NUM_THREADS = 16; +const int THREADS_PER_BUCKET = 4; + +// Simple global config +struct { + std::string username; + std::string password; + std::vector bucket_names; + std::vector collection_names; +} g_config; + +// Thread arguments +struct ThreadArgs { + int thread_id; + int bucket_id; + std::string bucket_name; +}; + +// Simple operation function +bool perform_operation(brpc::Channel& channel, const std::string& key, const std::string& collection = "") { + brpc::Controller cntl; + brpc::CouchbaseRequest request; + brpc::CouchbaseResponse response; + + // Simple ADD operation + std::string value = butil::string_printf(R"({"thread_id": %d, "timestamp": %lld})", + bthread_self(), butil::gettimeofday_us()); + + bool success = collection.empty() ? + request.Add(key.c_str(), value.c_str(), 0xdeadbeef, 300, 0) : + request.Add(key.c_str(), value.c_str(), 0xdeadbeef, 300, 0, collection); + + if (!success) return false; + + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) return false; + + uint64_t cas_value; + return response.PopAdd(&cas_value); +} + +// Thread worker function - very simple +void* thread_worker(void* arg) { + ThreadArgs* args = static_cast(arg); + + LOG(INFO) << "Thread " << args->thread_id << " starting on bucket " << args->bucket_name; + + // One channel per thread as requested + brpc::Channel channel; + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_COUCHBASE; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = 5000; + + if (channel.Init(FLAGS_server.c_str(), "", &options) != 0) { + LOG(ERROR) << "Thread " << args->thread_id << ": Failed to init channel"; + return NULL; + } + + // Simple authentication + brpc::Controller auth_cntl; + brpc::CouchbaseRequest auth_request; + brpc::CouchbaseResponse auth_response; + + if (!auth_request.Authenticate(g_config.username.c_str(), g_config.password.c_str(), &channel, FLAGS_server)) { + LOG(ERROR) << "Thread " << args->thread_id << ": Auth request failed"; + return NULL; + } + + channel.CallMethod(NULL, &auth_cntl, &auth_request, &auth_response, NULL); + if (auth_cntl.Failed()) { + LOG(ERROR) << "Thread " << args->thread_id << ": Auth failed"; + return NULL; + } + + // Select bucket + brpc::Controller bucket_cntl; + brpc::CouchbaseRequest bucket_request; + brpc::CouchbaseResponse bucket_response; + + if (!bucket_request.SelectBucket(args->bucket_name.c_str())) { + LOG(ERROR) << "Thread " << args->thread_id << ": Bucket request failed"; + return NULL; + } + + channel.CallMethod(NULL, &bucket_cntl, &bucket_request, &bucket_response, NULL); + if (bucket_cntl.Failed()) { + LOG(ERROR) << "Thread " << args->thread_id << ": Bucket selection failed"; + return NULL; + } + + if (!bucket_response.PopSelectBucket(NULL, args->bucket_name.c_str())) { + LOG(ERROR) << "Thread " << args->thread_id << ": Bucket selection failed"; + LOG(ERROR) << bucket_response.LastError(); + return NULL; + } + + LOG(INFO) << "Thread " << args->thread_id << " connected to bucket " << args->bucket_name; + + // Perform operations + int success_count = 0; + for (int i = 0; i < FLAGS_operations_per_thread; ++i) { + std::string key = butil::string_printf("thread_%d_op_%d", args->thread_id, i); + + // Choose collection randomly if available + std::string collection = ""; + if (!g_config.collection_names.empty()) { + collection = g_config.collection_names[i % g_config.collection_names.size()]; + } + + if (perform_operation(channel, key, collection)) { + success_count++; + } + + if (FLAGS_sleep_ms > 0) { + bthread_usleep(FLAGS_sleep_ms * 1000); + } + } + + LOG(INFO) << "Thread " << args->thread_id << " completed: " << success_count + << "/" << FLAGS_operations_per_thread << " operations successful"; + + return NULL; +} + +// Simple config function +void get_config() { + std::cout << CYAN << "=== Simple Multithreaded Couchbase Client ===" << RESET << std::endl; + + std::cout << "Username: "; + std::cin >> g_config.username; + std::cout << "Password: "; + std::cin >> g_config.password; + + std::cout << "Enter 4 bucket names:" << std::endl; + for (int i = 0; i < 4; ++i) { + std::string bucket; + std::cout << "Bucket " << (i+1) << ": "; + std::cin >> bucket; + g_config.bucket_names.push_back(bucket); + } + + int num_collections; + std::cout << "Number of collections (0 for none): "; + std::cin >> num_collections; + + for (int i = 0; i < num_collections; ++i) { + std::string collection; + std::cout << "Collection " << (i+1) << ": "; + std::cin >> collection; + g_config.collection_names.push_back(collection); + } +} + +int main(int argc, char* argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + + std::cout << GREEN << "Starting 16 bthreads (4 per bucket)" << RESET << std::endl; + + get_config(); + + // Create bthreads + std::vector threads(NUM_THREADS); + std::vector args(NUM_THREADS); + + for (int i = 0; i < NUM_THREADS; ++i) { + args[i].thread_id = i; + args[i].bucket_id = i / THREADS_PER_BUCKET; + args[i].bucket_name = g_config.bucket_names[args[i].bucket_id]; + + if (bthread_start_background(&threads[i], NULL, thread_worker, &args[i]) != 0) { + LOG(ERROR) << "Failed to create thread " << i; + return -1; + } + + bthread_usleep(50000); // 50ms delay between thread starts + } + + std::cout << "All 16 threads started!" << std::endl; + + // Wait for all threads + for (int i = 0; i < NUM_THREADS; ++i) { + bthread_join(threads[i], NULL); + } + + std::cout << GREEN << "All threads completed!" << RESET << std::endl; + return 0; +} diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 64f7fc4721..541ea1ad37 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -28,6 +28,103 @@ namespace brpc { +// Couchbase protocol constants +namespace { + constexpr uint32_t APPLE_VBUCKET_COUNT = 64; + constexpr uint32_t DEFAULT_VBUCKET_COUNT = 1024; + constexpr uint8_t COLLECTION_ID_SIZE = 1; + constexpr size_t MANIFEST_ID_SIZE = 8; + constexpr size_t CONNECTION_ID_SIZE = 33; + constexpr size_t RANDOM_ID_HEX_SIZE = 67; // 33 bytes * 2 + null terminator +} + +// Static member definitions +CouchbaseMetadataTracking* CouchbaseRequest::metadata_tracking = &common_metadata_tracking; +CouchbaseMetadataTracking* CouchbaseResponse::metadata_tracking = &common_metadata_tracking; + +bool CouchbaseMetadataTracking::set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string &server) { + std::unique_lock write_lock(rw_thread_to_channel_info_mutex); + auto it = thread_to_channel_info.find(thread_id); + if (it != thread_to_channel_info.end()) { + LOG(WARNING) << "Channel already exists for thread_id: " << thread_id; + return false; + } + ChannelInfo ch_info; + ch_info.channel = channel; + ch_info.server = server; + thread_to_channel_info[thread_id] = ch_info; + return true; +} + +bool CouchbaseMetadataTracking::set_current_bucket_for_thread(uint64_t thread_id, const string& bucket){ + std::unique_lock write_lock(rw_thread_to_channel_info_mutex); + auto it = thread_to_channel_info.find(thread_id); + if(it == thread_to_channel_info.end()){ + LOG(ERROR) << "No channel exists for thread_id: " << thread_id << ", establish a connection(with authentication) first."; + return false; + } + it->second.selected_bucket = bucket; + return true; +} + +bool CouchbaseMetadataTracking::set_bucket_to_collection_map(uint64_t thread_id, const string& collection, uint8_t collection_id){ + string server, selected_bucket; + + // First, get the server and bucket info with proper locking + { + std::shared_lock read_lock(rw_thread_to_channel_info_mutex); + auto it = thread_to_channel_info.find(thread_id); + if(it == thread_to_channel_info.end()){ + LOG(ERROR) << "No channel exists for thread_id: " << thread_id << ", establish a connection(with authentication) first."; + return false; + } + server = it->second.server; + selected_bucket = it->second.selected_bucket; + if(selected_bucket.empty()){ + LOG(ERROR) << "No bucket selected for thread_id: " << thread_id << ", select a bucket first."; + return false; + } + } + + // key for inner map is bucket.scope.collection + // for now scope is always _default + string bucket_scope_collection = selected_bucket+"._default."+collection; + + // Then update the collection map with proper locking + { + std::unique_lock write_lock(rw_bucket_to_collection_map_mutex); + bucket_to_collection_map[server][bucket_scope_collection] = collection_id; + } + + return true; +} + +bool CouchbaseMetadataTracking::get_channel_info_for_thread(uint64_t thread_id, ChannelInfo *channel_info){ + std::shared_lock read_lock(rw_thread_to_channel_info_mutex); + auto it = thread_to_channel_info.find(thread_id); + if(it == thread_to_channel_info.end()){ + LOG(WARNING) << "No channel info exists for thread_id: " << thread_id; + return false; + } + *channel_info = it->second; + return true; +} + +bool CouchbaseMetadataTracking::get_bucket_to_collection_map(uint64_t thread_id, string server, string bucket, string scope, string collection, uint8_t *collection_id){ + string bucket_scope_collection = bucket+"."+scope+"."+collection; + std::shared_lock read_lock(rw_bucket_to_collection_map_mutex); + auto it1 = bucket_to_collection_map.find(server); + if(it1 == bucket_to_collection_map.end()){ + return false; + } + auto it2 = it1->second.find(bucket_scope_collection); + if(it2 == it1->second.end()){ + return false; + } + *collection_id = it2->second; + return true; +} + uint32_t CouchbaseRequest::hash_crc32(const char* key, size_t key_length) { static const uint32_t crc32tab[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, @@ -82,15 +179,16 @@ uint32_t CouchbaseRequest::hash_crc32(const char* key, size_t key_length) { crc = (crc >> 8) ^ crc32tab[(crc ^ (uint64_t)key[x]) & 0xff]; #ifdef __APPLE__ - return ((~crc) >> 16) % 64; + return ((~crc) >> 16) % APPLE_VBUCKET_COUNT; #else - return ((~crc) >> 16) % 1024; + return ((~crc) >> 16) % DEFAULT_VBUCKET_COUNT; #endif } // redefinition CouchbaseRequest::CouchbaseRequest() : NonreflectableMessage() { + metadata_tracking = &common_metadata_tracking; SharedCtor(); } @@ -116,33 +214,34 @@ void CouchbaseRequest::Clear() { _pipelined_count = 0; } +// Support for scope level collections will be added in future. // Get the Scope ID for a given scope name -bool CouchbaseRequest::GetScopeId(const butil::StringPiece& scope_name) { - if (scope_name.empty()) { - LOG(ERROR) << "Empty scope name"; - return false; - } - // Opcode 0xBC for Get Scope ID (see Collections.md) - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - policy::CB_GET_SCOPE_ID, - butil::HostToNet16(scope_name.size()), - 0, // no extras - policy::CB_BINARY_RAW_BYTES, - 0, // no vbucket - butil::HostToNet32(scope_name.size()), - 0, // opaque - 0 // no CAS - }; - if (_buf.append(&header, sizeof(header))) { - return false; - } - if (_buf.append(scope_name.data(), scope_name.size())) { - return false; - } - ++_pipelined_count; - return true; -} +// bool CouchbaseRequest::GetScopeId(const butil::StringPiece& scope_name) { +// if (scope_name.empty()) { +// LOG(ERROR) << "Empty scope name"; +// return false; +// } +// // Opcode 0xBC for Get Scope ID (see Collections.md) +// const policy::CouchbaseRequestHeader header = { +// policy::CB_MAGIC_REQUEST, +// policy::CB_GET_SCOPE_ID, +// butil::HostToNet16(scope_name.size()), +// 0, // no extras +// policy::CB_BINARY_RAW_BYTES, +// 0, // no vbucket +// butil::HostToNet32(scope_name.size()), +// 0, // opaque +// 0 // no CAS +// }; +// if (_buf.append(&header, sizeof(header))) { +// return false; +// } +// if (_buf.append(scope_name.data(), scope_name.size())) { +// return false; +// } +// ++_pipelined_count; +// return true; +// } bool CouchbaseRequest::SelectBucket(const butil::StringPiece& bucket_name) { if (bucket_name.empty()) { @@ -161,11 +260,11 @@ bool CouchbaseRequest::SelectBucket(const butil::StringPiece& bucket_name) { 0, 0}; if (_buf.append(&header, sizeof(header))) { - std::cout << "Failed to append header to buffer" << std::endl; + LOG(ERROR) << "Failed to append header to buffer"; return false; } if (_buf.append(bucket_name.data(), bucket_name.size())) { - std::cout << "Failed to append bucket name to buffer" << std::endl; + LOG(ERROR) << "Failed to append bucket name to buffer"; return false; } ++_pipelined_count; @@ -195,17 +294,17 @@ bool CouchbaseRequest::HelloRequest() { #endif agent += ";bssl/0x1010107f)"; - // Generate a random 33-byte connection id as hex string (66 hex chars) - unsigned char raw_id[33]; + // Generate a random connection ID as hex string + unsigned char raw_id[CONNECTION_ID_SIZE]; FILE* urandom = fopen("/dev/urandom", "rb"); - if (!urandom || fread(raw_id, 1, 33, urandom) != 33) { + if (!urandom || fread(raw_id, 1, CONNECTION_ID_SIZE, urandom) != CONNECTION_ID_SIZE) { if (urandom) fclose(urandom); - std::cout << "Failed to generate random connection id" << std::endl; + LOG(ERROR) << "Failed to generate random connection id"; return false; } fclose(urandom); - char hex_id[67] = {0}; - for (int i = 0; i < 33; ++i) { + char hex_id[RANDOM_ID_HEX_SIZE] = {0}; + for (int i = 0; i < CONNECTION_ID_SIZE; ++i) { sprintf(hex_id + i * 2, "%02x", raw_id[i]); } @@ -238,15 +337,15 @@ bool CouchbaseRequest::HelloRequest() { }; if (_buf.append(&header, sizeof(header))) { - std::cout << "Failed to append Hello header to buffer" << std::endl; + LOG(ERROR) << "Failed to append Hello header to buffer"; return false; } if (_buf.append(key.data(), key_len)) { - std::cout << "Failed to append Hello JSON key to buffer" << std::endl; + LOG(ERROR) << "Failed to append Hello JSON key to buffer"; return false; } if (_buf.append(reinterpret_cast(features), value_len)) { - std::cout << "Failed to append Hello features to buffer" << std::endl; + LOG(ERROR) << "Failed to append Hello features to buffer"; return false; } ++_pipelined_count; @@ -254,7 +353,9 @@ bool CouchbaseRequest::HelloRequest() { } bool CouchbaseRequest::Authenticate(const butil::StringPiece& username, - const butil::StringPiece& password) { + const butil::StringPiece& password, + brpc::Channel *channel, + const string server) { if (username.empty() || password.empty()) { LOG(ERROR) << "Empty username or password"; return false; @@ -279,21 +380,30 @@ bool CouchbaseRequest::Authenticate(const butil::StringPiece& username, password.length()), 0, 0}; - std::string* auth_str = new std::string(); - auth_str->clear(); - auth_str->append(reinterpret_cast(&header), sizeof(header)); - auth_str->append(kPlainAuthCommand, sizeof(kPlainAuthCommand) - 1); - auth_str->append(username.data()); - auth_str->append(kPadding, sizeof(kPadding)); - auth_str->append(username.data()); - auth_str->append(kPadding, sizeof(kPadding)); - auth_str->append(password.data()); - if (_buf.append(auth_str->data(), auth_str->size())) { - std::cout << "Failed to append auth string to buffer" << std::endl; - delete auth_str; + std::string auth_str; + auth_str.reserve(sizeof(header) + sizeof(kPlainAuthCommand) - 1 + + username.size() * 2 + password.size() + 2); + auth_str.append(reinterpret_cast(&header), sizeof(header)); + auth_str.append(kPlainAuthCommand, sizeof(kPlainAuthCommand) - 1); + auth_str.append(username.data(), username.size()); + auth_str.append(kPadding, sizeof(kPadding)); + auth_str.append(username.data(), username.size()); + auth_str.append(kPadding, sizeof(kPadding)); + auth_str.append(password.data(), password.size()); + if (_buf.append(auth_str.data(), auth_str.size())) { + LOG(ERROR) << "Failed to append auth string to buffer"; return false; } ++_pipelined_count; + + // Store the bRPC channel and the server address for future use related to current thread + uint64_t bthread_id = bthread_self(); + if(!metadata_tracking->set_channel_info_for_thread(bthread_id, channel, server)){ + LOG(ERROR) << "Channel for this thread already exists, only one channel per thread is allowed."; + _buf.pop_back(auth_str.size()); + --_pipelined_count; + return false; + } return true; } @@ -383,6 +493,7 @@ CouchbaseResponse::CouchbaseResponse() // redifinition CouchbaseResponse::CouchbaseResponse(const CouchbaseResponse& from) : NonreflectableMessage(from) { + metadata_tracking = &common_metadata_tracking; SharedCtor(); MergeFrom(from); } @@ -622,11 +733,72 @@ bool CouchbaseRequest::GetOrDelete(uint8_t command, return true; } -bool CouchbaseRequest::Get(const butil::StringPiece& key, uint8_t coll_id) { +bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, CouchbaseMetadataTracking *metadata_tracking){ + if(collection_name.empty()){ + LOG(ERROR) << "Empty collection name"; + return false; + } + CouchbaseMetadataTracking::ChannelInfo channel_info; + if(!metadata_tracking->get_channel_info_for_thread(bthread_self(), &channel_info)){ + LOG(ERROR) << "No channel found for this thread, make sure to call Authenticate() first"; + return false; + } + if(channel_info.server.empty()){ + LOG(ERROR) << "Server is empty for this thread, make sure to call Authenticate() first"; + return false; + } + if(channel_info.selected_bucket.empty()){ + LOG(ERROR) << "No bucket selected for this thread, make sure to call SelectBucket() first"; + return false; + } + bool result = metadata_tracking->get_bucket_to_collection_map(bthread_self(), channel_info.server, channel_info.selected_bucket, "_default", collection_name, coll_id); + if(!result){ + CouchbaseRequest temp_get_collectionID_request; + CouchbaseResponse temp_get_collectionID_response; + brpc::Controller temp_cntl; + brpc::Channel *channel = channel_info.channel; + temp_get_collectionID_request.GetCollectionId("_default", collection_name); + channel->CallMethod(NULL, &temp_cntl, &temp_get_collectionID_request, + &temp_get_collectionID_response, NULL); + if (temp_cntl.Failed()) { + LOG(ERROR) << "Failed to get collection ID for collection " << collection_name << " in bucket " << channel_info.selected_bucket << " on server " << channel_info.server + << ": " << temp_cntl.ErrorText(); + return false; + } + if (!temp_get_collectionID_response.PopCollectionId(coll_id)) { + LOG(ERROR) << "Failed to parse collection ID response for collection " << collection_name << " in bucket " << channel_info.selected_bucket << " on server " << channel_info.server + << ": " << temp_get_collectionID_response.LastError(); + return false; + } + else{ + // Cache the collection ID for future use + if(!metadata_tracking->set_bucket_to_collection_map(bthread_self(), collection_name, *coll_id)){ + LOG(ERROR) << "Failed to cache collection ID for collection " << collection_name << " in bucket " << channel_info.selected_bucket << " on server " << channel_info.server; + return false; + } + return true; + } + } + return true; +} + +bool CouchbaseRequest::Get(const butil::StringPiece& key, string collection_name) { + uint8_t coll_id = 0; // default collection ID + if(collection_name != "_default"){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + return false; + } + } return GetOrDelete(policy::CB_BINARY_GET, key, coll_id); } -bool CouchbaseRequest::Delete(const butil::StringPiece& key, uint8_t coll_id) { +bool CouchbaseRequest::Delete(const butil::StringPiece& key, string collection_name) { + uint8_t coll_id = 0; // default collection ID + if(collection_name != "_default"){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + return false; + } + } return GetOrDelete(policy::CB_BINARY_DELETE, key, coll_id); } @@ -968,7 +1140,13 @@ const char* CouchbaseResponse::couchbase_binary_command_to_string(uint8_t cmd) { bool CouchbaseRequest::Upsert(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id) { + string collection_name) { + uint8_t coll_id = 0; // default collection ID + if(collection_name != "_default"){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + return false; + } + } return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value, coll_id); } @@ -1005,7 +1183,13 @@ bool CouchbaseRequest::GetCollectionId( bool CouchbaseRequest::Add(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id) { + string collection_name) { + uint8_t coll_id = 0; // default collection ID + if(collection_name != "_default"){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + return false; + } + } return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, coll_id); } @@ -1013,7 +1197,13 @@ bool CouchbaseRequest::Add(const butil::StringPiece& key, bool CouchbaseRequest::Replace(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id) { + string collection_name) { + uint8_t coll_id = 0; // default collection ID + if(collection_name != "_default"){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + return false; + } + } return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, cas_value, coll_id); } @@ -1021,11 +1211,17 @@ bool CouchbaseRequest::Replace(const butil::StringPiece& key, bool CouchbaseRequest::Append(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id) { + string collection_name) { if (value.empty()) { LOG(ERROR) << "value to append must be non-empty"; return false; } + uint8_t coll_id = 0; // default collection ID + if(collection_name != "_default"){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + return false; + } + } return Store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value, coll_id); } @@ -1033,11 +1229,17 @@ bool CouchbaseRequest::Append(const butil::StringPiece& key, bool CouchbaseRequest::Prepend(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id) { + string collection_name) { if (value.empty()) { LOG(ERROR) << "value to prepend must be non-empty"; return false; } + uint8_t coll_id = 0; // default collection ID + if(collection_name != "_default"){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + return false; + } + } return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, cas_value, coll_id); } @@ -1057,8 +1259,17 @@ bool CouchbaseResponse::PopAppend(uint64_t* cas_value) { bool CouchbaseResponse::PopPrepend(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_PREPEND, cas_value); } -bool CouchbaseResponse::PopSelectBucket(uint64_t* cas_value) { - return PopStore(policy::CB_SELECT_BUCKET, cas_value); +bool CouchbaseResponse::PopSelectBucket(uint64_t* cas_value, std::string bucket_name) { + if(PopStore(policy::CB_SELECT_BUCKET, cas_value) == false){ + LOG(ERROR) << "Failed to select bucket: " << _err; + return false; + } + uint64_t bthread_id = bthread_self(); + if(metadata_tracking->set_current_bucket_for_thread(bthread_id, bucket_name) == false){ + LOG(FATAL) << "Failed to set current bucket for thread_id: " << bthread_id << ". This shouldn't happen normally."; + return false; + } + return true; } // Collection-related response method bool CouchbaseResponse::PopCollectionId(uint8_t* collection_id) { @@ -1194,14 +1405,14 @@ bool CouchbaseRequest::Counter(uint8_t command, const butil::StringPiece& key, bool CouchbaseRequest::Increment(const butil::StringPiece& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, - uint8_t coll_id) { + string collection_name) { return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, exptime); } bool CouchbaseRequest::Decrement(const butil::StringPiece& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, - uint8_t coll_id) { + string collection_name) { return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, exptime); } @@ -1303,7 +1514,7 @@ const size_t TOUCH_EXTRAS = // +---------------+---------------+---------------+---------------+ // Total 4 bytes bool CouchbaseRequest::Touch(const butil::StringPiece& key, uint32_t exptime, - uint8_t coll_id) { + string collection_name) { TouchHeaderWithExtras header_with_extras = { {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_TOUCH, butil::HostToNet16(key.size()), TOUCH_EXTRAS, diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 2a05d5f015..b6d71e280d 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -23,13 +23,44 @@ #include #include "brpc/nonreflectable_message.h" +#include #include "brpc/pb_compat.h" #include "butil/iobuf.h" #include "butil/strings/string_piece.h" +#include +#include +using namespace std; namespace brpc { +class CouchbaseMetadataTracking{ + public: + struct ChannelInfo { + brpc::Channel* channel; + string server; + string selected_bucket; + }; + private: + unordered_map thread_to_channel_info; + shared_mutex rw_thread_to_channel_info_mutex; + unordered_map> bucket_to_collection_map; + shared_mutex rw_bucket_to_collection_map_mutex; + + public: + CouchbaseMetadataTracking() {} + ~CouchbaseMetadataTracking() { + bucket_to_collection_map.clear(); + thread_to_channel_info.clear(); + } + bool set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string& server); + bool set_current_bucket_for_thread(uint64_t thread_id, const string& bucket); + bool set_bucket_to_collection_map(uint64_t thread_id, const string& collection, uint8_t collection_id); + bool get_channel_info_for_thread(uint64_t thread_id, ChannelInfo *channel_info); + bool get_bucket_to_collection_map(uint64_t thread_id, string server, string bucket, string scope, string collection, uint8_t *collection_id); +} static common_metadata_tracking; + class CouchbaseRequest : public NonreflectableMessage { private: + static CouchbaseMetadataTracking *metadata_tracking; int _pipelined_count; butil::IOBuf _buf; mutable int _cached_size_; @@ -57,7 +88,9 @@ class CouchbaseRequest : public NonreflectableMessage { bool SelectBucket(const butil::StringPiece& bucket_name); bool Authenticate(const butil::StringPiece& username, - const butil::StringPiece& password); + const butil::StringPiece& password, + brpc::Channel* channel, + const string server); bool HelloRequest(); // Collection Management Method @@ -67,38 +100,38 @@ class CouchbaseRequest : public NonreflectableMessage { bool GetScopeId(const butil::StringPiece& scope_name); // Collection-aware document operations - bool Get(const butil::StringPiece& key, uint8_t coll_id = 0); + bool Get(const butil::StringPiece& key, string collection_name = "_default"); bool Upsert(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id = 0); + string collection_name = "_default"); bool Add(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id = 0); + string collection_name = "_default"); bool Replace(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id = 0); + string collection_name = "_default"); bool Append(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id = 0); + string collection_name = "_default"); bool Prepend(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - uint8_t coll_id = 0); + string collection_name = "_default"); - bool Delete(const butil::StringPiece& key, uint8_t coll_id = 0); + bool Delete(const butil::StringPiece& key, string collection_name = "_default"); bool Flush(uint32_t timeout); bool Increment(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime, uint8_t coll_id = 0); + uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); bool Decrement(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime, uint8_t coll_id = 0); + uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); bool Touch(const butil::StringPiece& key, uint32_t exptime, - uint8_t coll_id = 0); + string collection_name = "_default"); bool Version(); @@ -122,7 +155,8 @@ class CouchbaseRequest : public NonreflectableMessage { class CouchbaseResponse : public NonreflectableMessage { private: - std::string _err; + string _err; + static CouchbaseMetadataTracking *metadata_tracking; butil::IOBuf _buf; mutable int _cached_size_; bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); @@ -217,21 +251,21 @@ class CouchbaseResponse : public NonreflectableMessage { static const char* status_str(Status); // Helper method to format error messages with status codes - static std::string format_error_message(uint16_t status_code, - const std::string& operation, - const std::string& error_msg = ""); + static string format_error_message(uint16_t status_code, + const string& operation, + const string& error_msg = ""); // Add methods to handle response parsing void Swap(CouchbaseResponse* other); bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool PopGet(std::string* value, uint32_t* flags, uint64_t* cas_value); - const std::string& LastError() const { return _err; } + bool PopGet(string* value, uint32_t* flags, uint64_t* cas_value); + const string& LastError() const { return _err; } bool PopUpsert(uint64_t* cas_value); bool PopAdd(uint64_t* cas_value); bool PopReplace(uint64_t* cas_value); bool PopAppend(uint64_t* cas_value); bool PopPrepend(uint64_t* cas_value); - bool PopSelectBucket(uint64_t* cas_value); + bool PopSelectBucket(uint64_t* cas_value, std::string bucket_name); // Collection-related response methods bool PopCollectionId(uint8_t* collection_id); @@ -241,6 +275,6 @@ class CouchbaseResponse : public NonreflectableMessage { bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); bool PopTouch(); - bool PopVersion(std::string* version); + bool PopVersion(string* version); }; } // namespace brpc \ No newline at end of file From 6a8200d334f3e921feb2a9a4a6d654c074879dbf Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 1 Oct 2025 08:52:18 +0530 Subject: [PATCH 14/49] added caching of collection manifests --- example/couchbase_c++/Makefile | 32 ++- example/couchbase_c++/couchbase_client.cpp | 15 +- src/brpc/couchbase.cpp | 315 +++++++++++++++++---- src/brpc/couchbase.h | 30 +- 4 files changed, 314 insertions(+), 78 deletions(-) diff --git a/example/couchbase_c++/Makefile b/example/couchbase_c++/Makefile index 84b31031be..75052b5763 100644 --- a/example/couchbase_c++/Makefile +++ b/example/couchbase_c++/Makefile @@ -25,8 +25,11 @@ LIBPATHS = $(addprefix -L, $(LIBS)) COMMA=, SOPATHS=$(addprefix -Wl$(COMMA)-rpath$(COMMA), $(LIBS)) -SOURCES = $(wildcard *.cpp) -OBJS = $(addsuffix .o, $(basename $(SOURCES))) +# Define targets and their sources +TARGETS = couchbase_client multithreaded_couchbase_client +COUCHBASE_CLIENT_OBJS = couchbase_client.o +MULTITHREADED_CLIENT_OBJS = multithreaded_couchbase_client.o +ALL_OBJS = $(COUCHBASE_CLIENT_OBJS) $(MULTITHREADED_CLIENT_OBJS) ifeq ($(SYSTEM),Darwin) ifneq ("$(LINK_SO)", "") @@ -43,15 +46,25 @@ else ifeq ($(SYSTEM),Linux) LINK_OPTIONS = -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) endif -.PHONY:all -all: couchbase_client +.PHONY: all clean couchbase_client multithreaded_couchbase_client help + +# Default target builds both clients +all: $(TARGETS) -.PHONY:clean clean: @echo "> Cleaning" - rm -rf couchbase_client $(OBJS) + rm -rf $(TARGETS) $(ALL_OBJS) + +# Build rules for individual targets +couchbase_client: $(COUCHBASE_CLIENT_OBJS) + @echo "> Linking $@" +ifneq ("$(LINK_SO)", "") + $(CXX) $(LIBPATHS) $(SOPATHS) $(LINK_OPTIONS_SO) -o $@ +else + $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ +endif -couchbase_client:$(OBJS) +multithreaded_couchbase_client: $(MULTITHREADED_CLIENT_OBJS) @echo "> Linking $@" ifneq ("$(LINK_SO)", "") $(CXX) $(LIBPATHS) $(SOPATHS) $(LINK_OPTIONS_SO) -o $@ @@ -59,10 +72,11 @@ else $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ endif -%.o:%.cpp +# Compilation rules +couchbase_client.o: couchbase_client.cpp @echo "> Compiling $@" $(CXX) -c $(HDRPATHS) $(CXXFLAGS) $< -o $@ -%.o:%.cc +multithreaded_couchbase_client.o: multithreaded_couchbase_client.cpp @echo "> Compiling $@" $(CXX) -c $(HDRPATHS) $(CXXFLAGS) $< -o $@ diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index d43ca313a3..7a5d7df446 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -98,7 +98,7 @@ int main() { } } - if (!auth_request.Authenticate(username.c_str(), password.c_str())) { + if (!auth_request.Authenticate(username.c_str(), password.c_str(), &channel, FLAGS_server)) { LOG(ERROR) << "Fail to create authentication request"; return -1; } @@ -144,7 +144,7 @@ int main() { } else { // Check ADD operation response status uint64_t cas_value; - if (select_bucket_response.PopSelectBucket(&cas_value)) { + if (select_bucket_response.PopSelectBucket(&cas_value, bucket_name.c_str())) { std::cout << GREEN << "Bucket Selection Successful, CAS: " << cas_value << RESET << std::endl; } else { @@ -640,7 +640,7 @@ int main() { const std::string coll_key = "user::collection_doc"; if (!coll_add_req.Add( coll_key.c_str(), R"({"type":"collection","op":"add","v":1})", - 0xabcddcba, FLAGS_exptime, 0, (uint8_t)testing_collection_id)) { + 0xabcddcba, FLAGS_exptime, 0, collection_name)) { LOG(ERROR) << "Fail to build collection ADD request"; } else { channel.CallMethod(NULL, &cntl, &coll_add_req, &coll_add_resp, NULL); @@ -663,7 +663,7 @@ int main() { // 2. GET from collection brpc::CouchbaseRequest coll_get_req; brpc::CouchbaseResponse coll_get_resp; - if (!coll_get_req.Get(coll_key.c_str(), (uint8_t)testing_collection_id)) { + if (!coll_get_req.Get(coll_key.c_str(), collection_name)) { LOG(ERROR) << "Fail to build collection GET request"; } else { channel.CallMethod(NULL, &cntl, &coll_get_req, &coll_get_resp, NULL); @@ -690,7 +690,7 @@ int main() { brpc::CouchbaseResponse coll_upsert_resp; if (!coll_upsert_req.Upsert( coll_key.c_str(), R"({"type":"collection","op":"upsert","v":2})", 0, - FLAGS_exptime, 0, (uint8_t)testing_collection_id)) { + FLAGS_exptime, 0, collection_name)) { LOG(ERROR) << "Fail to build collection UPSERT request"; } else { channel.CallMethod(NULL, &cntl, &coll_upsert_req, &coll_upsert_resp, @@ -713,7 +713,7 @@ int main() { // 4. GET again to verify upsert brpc::CouchbaseRequest coll_get2_req; brpc::CouchbaseResponse coll_get2_resp; - if (coll_get2_req.Get(coll_key.c_str(), (uint8_t)testing_collection_id)) { + if (coll_get2_req.Get(coll_key.c_str(), collection_name)) { channel.CallMethod(NULL, &cntl, &coll_get2_req, &coll_get2_resp, NULL); if (!cntl.Failed()) { std::string v; @@ -730,8 +730,7 @@ int main() { // 5. DELETE from collection brpc::CouchbaseRequest coll_del_req; brpc::CouchbaseResponse coll_del_resp; - if (!coll_del_req.Delete(coll_key.c_str(), - (uint8_t)testing_collection_id)) { + if (!coll_del_req.Delete(coll_key.c_str(), collection_name)) { LOG(ERROR) << "Fail to build collection DELETE request"; } else { channel.CallMethod(NULL, &cntl, &coll_del_req, &coll_del_resp, NULL); diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 541ea1ad37..d943831c0a 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -25,6 +25,8 @@ #include "butil/macros.h" #include "butil/string_printf.h" #include "butil/sys_byteorder.h" +#include "butil/third_party/rapidjson/document.h" +#include "butil/third_party/rapidjson/rapidjson.h" namespace brpc { @@ -32,9 +34,7 @@ namespace brpc { namespace { constexpr uint32_t APPLE_VBUCKET_COUNT = 64; constexpr uint32_t DEFAULT_VBUCKET_COUNT = 1024; - constexpr uint8_t COLLECTION_ID_SIZE = 1; - constexpr size_t MANIFEST_ID_SIZE = 8; - constexpr size_t CONNECTION_ID_SIZE = 33; + constexpr int CONNECTION_ID_SIZE = 33; constexpr size_t RANDOM_ID_HEX_SIZE = 67; // 33 bytes * 2 + null terminator } @@ -42,7 +42,7 @@ namespace { CouchbaseMetadataTracking* CouchbaseRequest::metadata_tracking = &common_metadata_tracking; CouchbaseMetadataTracking* CouchbaseResponse::metadata_tracking = &common_metadata_tracking; -bool CouchbaseMetadataTracking::set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string &server) { +bool brpc::CouchbaseMetadataTracking::set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string &server) { std::unique_lock write_lock(rw_thread_to_channel_info_mutex); auto it = thread_to_channel_info.find(thread_id); if (it != thread_to_channel_info.end()) { @@ -56,7 +56,7 @@ bool CouchbaseMetadataTracking::set_channel_info_for_thread(uint64_t thread_id, return true; } -bool CouchbaseMetadataTracking::set_current_bucket_for_thread(uint64_t thread_id, const string& bucket){ +bool brpc::CouchbaseMetadataTracking::set_current_bucket_for_thread(uint64_t thread_id, const string& bucket){ std::unique_lock write_lock(rw_thread_to_channel_info_mutex); auto it = thread_to_channel_info.find(thread_id); if(it == thread_to_channel_info.end()){ @@ -67,7 +67,7 @@ bool CouchbaseMetadataTracking::set_current_bucket_for_thread(uint64_t thread_id return true; } -bool CouchbaseMetadataTracking::set_bucket_to_collection_map(uint64_t thread_id, const string& collection, uint8_t collection_id){ +bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest(uint64_t thread_id, CouchbaseMetadataTracking::CollectionManifest manifest){ string server, selected_bucket; // First, get the server and bucket info with proper locking @@ -86,20 +86,16 @@ bool CouchbaseMetadataTracking::set_bucket_to_collection_map(uint64_t thread_id, } } - // key for inner map is bucket.scope.collection - // for now scope is always _default - string bucket_scope_collection = selected_bucket+"._default."+collection; - - // Then update the collection map with proper locking + // Then update the collection manifest with proper locking { - std::unique_lock write_lock(rw_bucket_to_collection_map_mutex); - bucket_to_collection_map[server][bucket_scope_collection] = collection_id; + std::unique_lock write_lock(rw_bucket_to_collection_manifest_mutex); + bucket_to_collection_manifest[server][selected_bucket] = manifest; } return true; } -bool CouchbaseMetadataTracking::get_channel_info_for_thread(uint64_t thread_id, ChannelInfo *channel_info){ +bool brpc::CouchbaseMetadataTracking::get_channel_info_for_thread(uint64_t thread_id, ChannelInfo *channel_info){ std::shared_lock read_lock(rw_thread_to_channel_info_mutex); auto it = thread_to_channel_info.find(thread_id); if(it == thread_to_channel_info.end()){ @@ -110,21 +106,155 @@ bool CouchbaseMetadataTracking::get_channel_info_for_thread(uint64_t thread_id, return true; } -bool CouchbaseMetadataTracking::get_bucket_to_collection_map(uint64_t thread_id, string server, string bucket, string scope, string collection, uint8_t *collection_id){ - string bucket_scope_collection = bucket+"."+scope+"."+collection; - std::shared_lock read_lock(rw_bucket_to_collection_map_mutex); - auto it1 = bucket_to_collection_map.find(server); - if(it1 == bucket_to_collection_map.end()){ +bool brpc::CouchbaseMetadataTracking::get_bucket_to_collection_manifest(uint64_t thread_id, string server, string bucket, CouchbaseMetadataTracking::CollectionManifest *manifest){ + std::shared_lock read_lock(rw_bucket_to_collection_manifest_mutex); + auto it1 = bucket_to_collection_manifest.find(server); + if(it1 == bucket_to_collection_manifest.end()){ return false; } - auto it2 = it1->second.find(bucket_scope_collection); + auto it2 = it1->second.find(bucket); if(it2 == it1->second.end()){ return false; } + *manifest = it2->second; + return true; +} + +bool brpc::CouchbaseMetadataTracking::get_manifest_to_collection_id(CouchbaseMetadataTracking::CollectionManifest *manifest, string scope, string collection, uint8_t *collection_id){ + if(manifest == nullptr || collection_id == nullptr){ + LOG(ERROR) << "Invalid input: manifest or collection_id is null"; + return false; + } + auto it1 = manifest->scope_to_collectionID_map.find(scope); + if(it1 == manifest->scope_to_collectionID_map.end()){ + LOG(ERROR) << "Scope: " << scope << " not found in manifest"; + return false; + } + auto it2 = it1->second.find(collection); + if(it2 == it1->second.end()){ + LOG(ERROR) << "Collection: " << collection << " not found in scope: " << scope; + return false; + } *collection_id = it2->second; return true; } +bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest(const string& json, CouchbaseMetadataTracking::CollectionManifest *manifest) { + if(manifest == nullptr){ + LOG(ERROR) << "Invalid input: manifest is null"; + return false; + } + + // Clear existing data + manifest->uid.clear(); + manifest->scope_to_collectionID_map.clear(); + + if (json.empty()) { + LOG(ERROR) << "JSON string is empty"; + return false; + } + + // Parse JSON using RapidJSON + BUTIL_RAPIDJSON_NAMESPACE::Document document; + document.Parse(json.c_str()); + + if (document.HasParseError()) { + LOG(ERROR) << "Failed to parse JSON: " << document.GetParseError(); + return false; + } + + if (!document.IsObject()) { + LOG(ERROR) << "JSON root is not an object"; + return false; + } + + // Extract uid + if (document.HasMember("uid") && document["uid"].IsString()) { + manifest->uid = document["uid"].GetString(); + } else { + LOG(ERROR) << "Missing or invalid 'uid' field in JSON"; + return false; + } + + // Extract scopes + if (!document.HasMember("scopes") || !document["scopes"].IsArray()) { + LOG(ERROR) << "Missing or invalid 'scopes' field in JSON"; + return false; + } + + const BUTIL_RAPIDJSON_NAMESPACE::Value& scopes = document["scopes"]; + for (BUTIL_RAPIDJSON_NAMESPACE::SizeType i = 0; i < scopes.Size(); ++i) { + const BUTIL_RAPIDJSON_NAMESPACE::Value& scope = scopes[i]; + + if (!scope.IsObject()) { + LOG(ERROR) << "Scope at index " << i << " is not an object"; + return false; + } + + // Extract scope name + if (!scope.HasMember("name") || !scope["name"].IsString()) { + LOG(ERROR) << "Missing or invalid 'name' field in scope at index " << i; + return false; + } + string scope_name = scope["name"].GetString(); + + // Extract collections + if (!scope.HasMember("collections") || !scope["collections"].IsArray()) { + LOG(ERROR) << "Missing or invalid 'collections' field in scope '" << scope_name << "'"; + return false; + } + + const BUTIL_RAPIDJSON_NAMESPACE::Value& collections = scope["collections"]; + unordered_map collection_map; + + for (BUTIL_RAPIDJSON_NAMESPACE::SizeType j = 0; j < collections.Size(); ++j) { + const BUTIL_RAPIDJSON_NAMESPACE::Value& collection = collections[j]; + + if (!collection.IsObject()) { + LOG(ERROR) << "Collection at index " << j << " in scope '" << scope_name << "' is not an object"; + return false; + } + + // Extract collection name + if (!collection.HasMember("name") || !collection["name"].IsString()) { + LOG(ERROR) << "Missing or invalid 'name' field in collection at index " << j << " in scope '" << scope_name << "'"; + return false; + } + string collection_name = collection["name"].GetString(); + + // Extract collection uid (hex string) + if (!collection.HasMember("uid") || !collection["uid"].IsString()) { + LOG(ERROR) << "Missing or invalid 'uid' field in collection '" << collection_name << "' in scope '" << scope_name << "'"; + return false; + } + string collection_uid_str = collection["uid"].GetString(); + + // Convert hex string to uint8_t + uint8_t collection_id = 0; + try { + // Convert hex string to integer + unsigned long uid_val = std::stoul(collection_uid_str, nullptr, 16); + if (uid_val > 255) { + LOG(ERROR) << "Collection uid '" << collection_uid_str << "' exceeds uint8_t range in collection '" << collection_name << "' in scope '" << scope_name << "'"; + return false; + } + collection_id = static_cast(uid_val); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse collection uid '" << collection_uid_str << "' as hex in collection '" << collection_name << "' in scope '" << scope_name << "': " << e.what(); + return false; + } + + // Add to collection map + collection_map[collection_name] = collection_id; + } + + // Add scope and its collections to manifest + manifest->scope_to_collectionID_map[scope_name] = std::move(collection_map); + } + + return true; +} + uint32_t CouchbaseRequest::hash_crc32(const char* key, size_t key_length) { static const uint32_t crc32tab[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, @@ -744,42 +874,55 @@ bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, return false; } if(channel_info.server.empty()){ - LOG(ERROR) << "Server is empty for this thread, make sure to call Authenticate() first"; - return false; - } + LOG(ERROR) << "Server is empty for this thread, make sure to call Authenticate() first"; + return false; + } if(channel_info.selected_bucket.empty()){ - LOG(ERROR) << "No bucket selected for this thread, make sure to call SelectBucket() first"; - return false; - } - bool result = metadata_tracking->get_bucket_to_collection_map(bthread_self(), channel_info.server, channel_info.selected_bucket, "_default", collection_name, coll_id); - if(!result){ - CouchbaseRequest temp_get_collectionID_request; - CouchbaseResponse temp_get_collectionID_response; + LOG(ERROR) << "No bucket selected for this thread, make sure to call SelectBucket() first"; + return false; + } + CouchbaseMetadataTracking::CollectionManifest manifest; + if(!metadata_tracking->get_bucket_to_collection_manifest(bthread_self(), channel_info.server, channel_info.selected_bucket, &manifest)){ + LOG(INFO) << "No cached collection manifest found for bucket " << channel_info.selected_bucket << " on server " << channel_info.server << ", fetching from server"; + // No cached manifest found, fetch from server + CouchbaseRequest temp_get_manifest_request; + CouchbaseResponse temp_get_manifest_response; brpc::Controller temp_cntl; brpc::Channel *channel = channel_info.channel; - temp_get_collectionID_request.GetCollectionId("_default", collection_name); - channel->CallMethod(NULL, &temp_cntl, &temp_get_collectionID_request, - &temp_get_collectionID_response, NULL); + temp_get_manifest_request.GetCollectionManifest(); + channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, + &temp_get_manifest_response, NULL); if (temp_cntl.Failed()) { - LOG(ERROR) << "Failed to get collection ID for collection " << collection_name << " in bucket " << channel_info.selected_bucket << " on server " << channel_info.server + LOG(ERROR) << "Failed to get collection manifest for bucket " << channel_info.selected_bucket << " on server " << channel_info.server << ": " << temp_cntl.ErrorText(); return false; } - if (!temp_get_collectionID_response.PopCollectionId(coll_id)) { - LOG(ERROR) << "Failed to parse collection ID response for collection " << collection_name << " in bucket " << channel_info.selected_bucket << " on server " << channel_info.server - << ": " << temp_get_collectionID_response.LastError(); + string manifest_json; + if (!temp_get_manifest_response.PopManifest(&manifest_json)) { + LOG(ERROR) << "Failed to parse response for collection Manifest in bucket " << channel_info.selected_bucket << " on server " << channel_info.server + << ": " << temp_get_manifest_response.LastError(); return false; } else{ - // Cache the collection ID for future use - if(!metadata_tracking->set_bucket_to_collection_map(bthread_self(), collection_name, *coll_id)){ + // convert JSON to manifest structure + if(!metadata_tracking->json_to_collection_manifest(manifest_json, &manifest)){ + LOG(ERROR) << "Failed to parse collection manifest JSON for bucket " << channel_info.selected_bucket << " on server " << channel_info.server; + return false; + } + // Cache the collection manifest + if(!metadata_tracking->set_bucket_to_collection_manifest(bthread_self(), manifest)){ LOG(ERROR) << "Failed to cache collection ID for collection " << collection_name << " in bucket " << channel_info.selected_bucket << " on server " << channel_info.server; return false; } return true; } } - return true; + else{ + if(!metadata_tracking->get_manifest_to_collection_id(&manifest, "_default", collection_name, coll_id)){ + return false; + } + return true; + } } bool CouchbaseRequest::Get(const butil::StringPiece& key, string collection_name) { @@ -1180,6 +1323,25 @@ bool CouchbaseRequest::GetCollectionId( return true; } +bool CouchbaseRequest::GetCollectionManifest() { + const policy::CouchbaseRequestHeader header = { + policy::CB_MAGIC_REQUEST, + policy::CB_GET_COLLECTIONS_MANIFEST, + 0, // no key + 0, // no extras + policy::CB_BINARY_RAW_BYTES, + 0, // no vbucket + 0, // no body (no key, no extras, no value) + 0, // opaque + 0 // no CAS + }; + if (_buf.append(&header, sizeof(header))) { + return false; + } + ++_pipelined_count; + return true; +} + bool CouchbaseRequest::Add(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, @@ -1321,19 +1483,6 @@ bool CouchbaseResponse::PopCollectionId(uint8_t* collection_id) { // Skip header _buf.pop_front(sizeof(header)); - // LOG(INFO) << "Total body length: " << header.total_body_length; - // LOG(INFO) << "Extras length: " << header.extras_length; - - // size_t remaining_size = _buf.size(); - // std::vector remaining_data(remaining_size); - // _buf.copy_to(remaining_data.data(), remaining_size); - - // // Print the remaining data - // for (size_t i = 0; i < remaining_size; ++i) { - // LOG(INFO) << "Byte " << i << ": " << - // static_cast(remaining_data[i]); - // } - // return true; uint64_t manifest_id_net = 0; _buf.copy_to(reinterpret_cast(&manifest_id_net), @@ -1355,6 +1504,66 @@ bool CouchbaseResponse::PopCollectionId(uint8_t* collection_id) { return true; } +bool CouchbaseResponse::PopManifest(std::string* manifest_json) { + const size_t n = _buf.size(); + policy::CouchbaseResponseHeader header; + if (n < sizeof(header)) { + butil::string_printf(&_err, "buffer is too small to contain a header"); + return false; + } + _buf.copy_to(&header, sizeof(header)); + + if (header.command != policy::CB_GET_COLLECTIONS_MANIFEST) { + butil::string_printf(&_err, "Not a get collections manifest response"); + return false; + } + + // Making sure buffer has the whole body (extras + key + value) + if (n < sizeof(header) + header.total_body_length) { + butil::string_printf(&_err, "Not enough data"); + return false; + } + + if (header.status != 0) { + // handle error case + if(header.extras_length != 0){ + LOG(ERROR) << "Get Collections Manifest response must not have extras"; + } + if(header.key_length != 0){ + LOG(ERROR) << "Get Collections Manifest response must not have key"; + } + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + // Possibly read error message from value if present + size_t value_size = + header.total_body_length - header.extras_length - header.key_length; + if (value_size > 0) { + std::string err_msg; + _buf.cutn(&err_msg, value_size); + _err = format_error_message(header.status, "Get Collections Manifest", err_msg); + } else { + _err = format_error_message(header.status, "Get Collections Manifest"); + } + return false; + } + + // Success case: the manifest should be in the value section + size_t value_size = header.total_body_length - header.extras_length - header.key_length; + if (value_size == 0) { + butil::string_printf(&_err, "No manifest data in response"); + _buf.pop_front(sizeof(header) + header.total_body_length); + return false; + } + + // Skip header and any extras/key + _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + + // Read the manifest JSON from the value section + _buf.cutn(manifest_json, value_size); + + _err.clear(); + return true; +} + struct IncrHeaderWithExtras { policy::CouchbaseRequestHeader header; uint64_t delta; diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index b6d71e280d..ece93a8bc2 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -39,23 +39,33 @@ class CouchbaseMetadataTracking{ string server; string selected_bucket; }; + + struct CollectionManifest{ + string uid; //uid of the manifest, it can be used to track if the manifest is updated + unordered_map> scope_to_collectionID_map; //scope -> (collection -> collection_id) + }; private: unordered_map thread_to_channel_info; shared_mutex rw_thread_to_channel_info_mutex; - unordered_map> bucket_to_collection_map; - shared_mutex rw_bucket_to_collection_map_mutex; + + unordered_map> bucket_to_collection_manifest; + shared_mutex rw_bucket_to_collection_manifest_mutex; public: CouchbaseMetadataTracking() {} ~CouchbaseMetadataTracking() { - bucket_to_collection_map.clear(); + bucket_to_collection_manifest.clear(); thread_to_channel_info.clear(); } - bool set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string& server); - bool set_current_bucket_for_thread(uint64_t thread_id, const string& bucket); - bool set_bucket_to_collection_map(uint64_t thread_id, const string& collection, uint8_t collection_id); - bool get_channel_info_for_thread(uint64_t thread_id, ChannelInfo *channel_info); - bool get_bucket_to_collection_map(uint64_t thread_id, string server, string bucket, string scope, string collection, uint8_t *collection_id); + bool set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string& server); + bool set_current_bucket_for_thread(uint64_t thread_id, const string& bucket); + bool set_bucket_to_collection_manifest(uint64_t thread_id, CollectionManifest manifest); + + bool get_channel_info_for_thread(uint64_t thread_id, ChannelInfo *channel_info); + bool get_bucket_to_collection_manifest(uint64_t thread_id, string server, string bucket, CollectionManifest *manifest); + bool get_manifest_to_collection_id(CollectionManifest *manifest, string scope, string collection, uint8_t *collection_id); + + bool json_to_collection_manifest(const string& json, CollectionManifest *manifest); } static common_metadata_tracking; class CouchbaseRequest : public NonreflectableMessage { @@ -99,6 +109,8 @@ class CouchbaseRequest : public NonreflectableMessage { bool GetScopeId(const butil::StringPiece& scope_name); + bool GetCollectionManifest(); + // Collection-aware document operations bool Get(const butil::StringPiece& key, string collection_name = "_default"); @@ -270,6 +282,8 @@ class CouchbaseResponse : public NonreflectableMessage { // Collection-related response methods bool PopCollectionId(uint8_t* collection_id); + bool PopManifest(std::string* manifest_json); + bool PopDelete(); bool PopFlush(); bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); From db1afc1ada30fe2b42aa44a101ddfd4acce0c036 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 1 Oct 2025 08:54:42 +0530 Subject: [PATCH 15/49] Added example code for multithreaded demonstration --- example/couchbase_c++/multithreaded_couchbase_client.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/example/couchbase_c++/multithreaded_couchbase_client.cpp b/example/couchbase_c++/multithreaded_couchbase_client.cpp index db919adb11..9fca9eb59f 100644 --- a/example/couchbase_c++/multithreaded_couchbase_client.cpp +++ b/example/couchbase_c++/multithreaded_couchbase_client.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include From 753e66203d823c2447f0550c9ddc62bbc2c63c6d Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 1 Oct 2025 09:10:04 +0530 Subject: [PATCH 16/49] updated CMake --- CMakeLists.txt | 4 +++- example/couchbase_c++/CMakeLists.txt | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 45fcf61126..b41a6caa5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -219,7 +219,9 @@ if(Protobuf_VERSION GREATER 4.21) absl::variant ) else() - use_cxx11() + # Use C++17 for shared_mutex support and modern features + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) endif() find_package(Threads REQUIRED) diff --git a/example/couchbase_c++/CMakeLists.txt b/example/couchbase_c++/CMakeLists.txt index 50a67397ce..b0dbaee803 100644 --- a/example/couchbase_c++/CMakeLists.txt +++ b/example/couchbase_c++/CMakeLists.txt @@ -70,13 +70,13 @@ set(CMAKE_CXX_FLAGS "${DEFINE_CLOCK_GETTIME} -DNDEBUG -O2 -D__const__=__unused__ if(CMAKE_VERSION VERSION_LESS "3.1.3") if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") endif() else() - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) endif() From 4bdb6a43f0503d3cca9115cc023e4269b84cb52c Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 2 Oct 2025 06:32:59 +0530 Subject: [PATCH 17/49] Abstracted CRUD operations --- example/couchbase_c++/couchbase_client.cpp | 728 +++--------------- .../multithreaded_couchbase_client.cpp | 91 +-- src/brpc/couchbase.cpp | 513 ++++++++---- src/brpc/couchbase.h | 468 ++++++----- src/brpc/policy/couchbase_protocol.cpp | 8 +- 5 files changed, 781 insertions(+), 1027 deletions(-) diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index 7a5d7df446..76fe059544 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -15,70 +15,26 @@ // specific language governing permissions and limitations // under the License. -#include #include -#include -#include -#include #include -#include #include -#include -#include +#include +#include // ANSI color codes for console output #define GREEN "\033[32m" #define RED "\033[31m" #define RESET "\033[0m" -DEFINE_string(server, "127.0.0.1:11210", "IP Address of server"); -DEFINE_string(connection_type, "single", - "Connection type. Available values: single, pooled, short"); -// DEFINE_string(username, "Administrator", "Couchbase username"); -// DEFINE_string(password, "password", "Couchbase password"); -// DEFINE_string(bucket_name, "testing", "Couchbase bucket name"); -DEFINE_bool(use_bthread, true, "Use bthread to send requests"); -DEFINE_string(load_balancer, "", "The algorithm for load balancing"); -DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); -DEFINE_bool(dont_fail, false, "Print fatal when some call failed"); -DEFINE_int32(exptime, 0, - "The to-be-got data will be expired after so many seconds"); - -uint32_t batch_size = 10; - -bvar::LatencyRecorder g_latency_recorder("client"); -bvar::Adder g_error_count("client_error_count"); -butil::static_atomic g_sender_count = BUTIL_STATIC_ATOMIC_INIT(0); - -// Global variables for timing statistics -std::vector thread_response_times; -std::mutex timing_mutex; +DEFINE_string(server, "cb.WXYZ.cloud.couchbase.com:11207", "IP Address of server"); int main() { - brpc::Channel channel; + // Create CouchbaseOperations instance for high-level operations + brpc::CouchbaseOperations couchbase_ops; - // Initialize the channel, NULL means using default options. - brpc::ChannelOptions options; - options.protocol = brpc::PROTOCOL_COUCHBASE; - options.connection_type = FLAGS_connection_type; - options.timeout_ms = 2000 /*milliseconds*/; - options.max_retry = 2; - - if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), - &options) != 0) { - LOG(ERROR) << "Fail to initialize channel"; - return -1; - } - std::cout << GREEN << "Channel initialized successfully" << RESET + std::cout << GREEN << "Using high-level CouchbaseOperations interface" << RESET << std::endl; - // Couchbase Authentication packet(SASL Auth) is now present in the channel - // now a request can be sent with which auth packet will also be sent. - - // create authrequest and authresponse - brpc::Controller cntl; - brpc::CouchbaseRequest auth_request; - brpc::CouchbaseResponse auth_response; // Ask username and password for authentication std::string username; @@ -98,20 +54,13 @@ int main() { } } - if (!auth_request.Authenticate(username.c_str(), password.c_str(), &channel, FLAGS_server)) { - LOG(ERROR) << "Fail to create authentication request"; + // Use high-level authentication method + brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate(username, password, FLAGS_server, true, "couchbase-cloud-cert.txt"); + if (!auth_result.success) { + LOG(ERROR) << "Authentication failed: " << auth_result.error_message; return -1; } - channel.CallMethod(NULL, &cntl, &auth_request, &auth_response, NULL); - - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - cntl.Reset(); - std::cout << GREEN << "Authentication successful, proceeding with Couchbase operations..." @@ -128,467 +77,133 @@ int main() { continue; } } - brpc::CouchbaseRequest select_bucket_request; - brpc::CouchbaseResponse select_bucket_response; - if (!select_bucket_request.SelectBucket(bucket_name.c_str())) { - LOG(ERROR) << "Fail to SELECT bucket request"; - return -1; - } - - channel.CallMethod(NULL, &cntl, &select_bucket_request, - &select_bucket_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); + // Use high-level bucket selection method + brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.SelectBucket(bucket_name); + if (!bucket_result.success) { + LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; return -1; } else { - // Check ADD operation response status - uint64_t cas_value; - if (select_bucket_response.PopSelectBucket(&cas_value, bucket_name.c_str())) { - std::cout << GREEN << "Bucket Selection Successful, CAS: " << cas_value - << RESET << std::endl; - } else { - std::cout << RED << select_bucket_response.LastError() << RESET - << std::endl; - } + std::cout << GREEN << "Bucket Selection Successful" << RESET << std::endl; } - cntl.Reset(); - - // Add operation - brpc::CouchbaseRequest add_request; - brpc::CouchbaseResponse add_response; - if (!add_request.Add( - butil::string_printf("user::test_brpc_binprot"), - R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", - 0xdeadbeef, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - - channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); - - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; + // Add operation using high-level method + std::string add_key = "user::test_brpc_binprot"; + std::string add_value = R"({"name": "John Doe", "age": 30, "email": "john@example.com"})"; + + brpc::CouchbaseOperations::Result add_result = couchbase_ops.Add(add_key, add_value); + if (add_result.success) { + std::cout << GREEN << "ADD operation successful" << RESET << std::endl; } else { - // Check ADD operation response status - uint64_t cas_value; - if (add_response.PopAdd(&cas_value)) { - std::cout << GREEN << "ADD operation successful, CAS: " << cas_value - << RESET << std::endl; - } else { - std::cout << RED << add_response.LastError() << RESET << std::endl; - } - } - - cntl.Reset(); - - if (!add_request.Add( - butil::string_printf("user::test_brpc_binprot"), - R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", - 0xdeadbeef, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; + std::cout << RED << "ADD operation failed: " << add_result.error_message << RESET << std::endl; } - channel.CallMethod(NULL, &cntl, &add_request, &add_response, NULL); - // Check second ADD operation response status (should fail with key exists) - if (cntl.Failed()) { - LOG(ERROR) << "RPC call failed: " << cntl.ErrorText(); + // Try to ADD the same key again (should fail with key exists) + brpc::CouchbaseOperations::Result add_result2 = couchbase_ops.Add(add_key, add_value); + if (add_result2.success) { + std::cout << GREEN << "Second ADD operation unexpectedly successful" << RESET << std::endl; } else { - uint64_t cas_value; - if (add_response.PopAdd(&cas_value)) { - std::cout << GREEN - << "Second ADD operation unexpectedly successful, CAS: " - << cas_value << RESET << std::endl; - } else { - std::cout << RED << "Second ADD operation failed as expected: " - << add_response.LastError() << RESET << std::endl; - } - } - - cntl.Reset(); - // Get operation - brpc::CouchbaseRequest get_request; - brpc::CouchbaseResponse get_response; - if (!get_request.Get(butil::string_printf("%s", "user::test_brpc_binprot"))) { - LOG(ERROR) << "Fail to GET request"; - return -1; - } - channel.CallMethod(NULL, &cntl, &get_request, &get_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; + std::cout << RED << "Second ADD operation failed as expected: " << add_result2.error_message << RESET << std::endl; } - - // Check GET operation response status - std::string value; - uint32_t flags = 0; - uint64_t cas = 0; - if (get_response.PopGet(&value, &flags, &cas)) { + // Get operation using high-level method + brpc::CouchbaseOperations::Result get_result = couchbase_ops.Get(add_key); + if (get_result.success) { std::cout << GREEN << "GET operation successful" << RESET << std::endl; - std::cout << "GET value: " << value << std::endl; - std::cout << "Flags: " << flags << std::endl; - std::cout << "CAS: " << cas << std::endl; + std::cout << "GET value: " << get_result.value << std::endl; } else { - std::cout << RED << "GET operation failed: " << get_response.LastError() - << RESET << std::endl; - std::cout << "Raw response (hex): "; - for (char c : get_response.raw_buffer().to_string()) { - printf("%02x ", static_cast(c)); - } - std::cout << std::endl; - } - cntl.Reset(); - - // Create a new request for binprot item1 - brpc::CouchbaseRequest add_request1; - brpc::CouchbaseResponse add_response1; - if (!add_request1.Add( - butil::string_printf("binprot_item1"), - R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", - 0xdeadbeef, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - std::cout << "Sending ADD request for binprot_item1, pipelined count: " - << add_request1.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &add_request1, &add_response1, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; + std::cout << RED << "GET operation failed: " << get_result.error_message << RESET << std::endl; } - // Check ADD binprot item1 response status - uint64_t cas_value; - if (add_response1.PopAdd(&cas_value)) { - std::cout << GREEN << "ADD binprot item1 successful, CAS: " << cas_value - << RESET << std::endl; + // Add binprot item1 using high-level method + std::string item1_key = "binprot_item1"; + brpc::CouchbaseOperations::Result item1_result = couchbase_ops.Add(item1_key, add_value); + if (item1_result.success) { + std::cout << GREEN << "ADD binprot item1 successful" << RESET << std::endl; } else { - std::cout << RED - << "ADD binprot item1 failed: " << add_response1.LastError() - << RESET << std::endl; + std::cout << RED << "ADD binprot item1 failed: " << item1_result.error_message << RESET << std::endl; } - cntl.Reset(); - // Create a new request for binprot item2 - brpc::CouchbaseRequest add_request2; - brpc::CouchbaseResponse add_response2; - if (!add_request2.Add( - butil::string_printf("binprot_item2"), - R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", - 0xdeadbeef, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - std::cout << "Sending ADD request for binprot_item2, pipelined count: " - << add_request2.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &add_request2, &add_response2, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check ADD binprot item2 response status - if (add_response2.PopAdd(&cas_value)) { - std::cout << GREEN << "ADD binprot item2 successful, CAS: " << cas_value - << RESET << std::endl; + // Add binprot item2 using high-level method + std::string item2_key = "binprot_item2"; + brpc::CouchbaseOperations::Result item2_result = couchbase_ops.Add(item2_key, add_value); + if (item2_result.success) { + std::cout << GREEN << "ADD binprot item2 successful" << RESET << std::endl; } else { - std::cout << RED - << "ADD binprot item2 failed: " << add_response2.LastError() - << RESET << std::endl; + std::cout << RED << "ADD binprot item2 failed: " << item2_result.error_message << RESET << std::endl; } - cntl.Reset(); - // Create a new request for binprot item3 - brpc::CouchbaseRequest add_request3; - brpc::CouchbaseResponse add_response3; - if (!add_request3.Add( - butil::string_printf("binprot_item3"), - R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", - 0xdeadbeef, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - std::cout << "Sending ADD request for binprot_item3, pipelined count: " - << add_request3.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &add_request3, &add_response3, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check ADD binprot item3 response status - if (add_response3.PopAdd(&cas_value)) { - std::cout << GREEN << "ADD binprot item3 successful, CAS: " << cas_value - << RESET << std::endl; + // Add binprot item3 using high-level method + std::string item3_key = "binprot_item3"; + brpc::CouchbaseOperations::Result item3_result = couchbase_ops.Add(item3_key, add_value); + if (item3_result.success) { + std::cout << GREEN << "ADD binprot item3 successful" << RESET << std::endl; } else { - std::cout << RED - << "ADD binprot item3 failed: " << add_response3.LastError() - << RESET << std::endl; - } - cntl.Reset(); - - // pipeline ADD operation - brpc::CouchbaseRequest pipeline_request; - brpc::CouchbaseResponse pipeline_response; - for (int i = 0; i < 10; ++i) { - if (!pipeline_request.Add( - butil::string_printf("pipeline_item_%d", i), - R"({"name": "Pipeline User", "age": 25, "email": "pipeline@example.com"})", - 0xdeadbeef, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to ADD request"; - return -1; - } - } - std::cout << "Sending pipeline ADD request, pipelined count: " - << pipeline_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &pipeline_request, &pipeline_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check each pipelined operation response status - std::cout << "Processing pipeline responses..." << std::endl; - int successful_operations = 0; - int failed_operations = 0; - - for (int i = 0; i < 10; ++i) { - uint64_t cas_value; - if (pipeline_response.PopAdd(&cas_value)) { - std::cout << GREEN << "Pipeline ADD operation " << i - << " successful, CAS: " << cas_value << RESET << std::endl; - successful_operations++; - } else { - std::cout << RED << "Pipeline ADD operation " << i - << " failed: " << pipeline_response.LastError() << RESET - << std::endl; - failed_operations++; - } - } - - std::cout << "Pipeline summary: " << successful_operations << " successful, " - << failed_operations << " failed operations" << std::endl; - - cntl.Reset(); - - // Example of mixed pipeline operations (GET operations on existing and - // non-existing keys) - std::cout << "\n=== Testing Mixed Pipeline Operations ===" << std::endl; - brpc::CouchbaseRequest mixed_pipeline_request; - brpc::CouchbaseResponse mixed_pipeline_response; - - // Add some GET operations to the pipeline - some will succeed, some will fail - std::vector keys_to_get = { - "user::test_brpc_binprot", // Should exist - "binprot_item1", // Should exist - "nonexistent_key1", // Should fail - "binprot_item2", // Should exist - "nonexistent_key2", // Should fail - "binprot_item3" // Should exist - }; - - for (const auto& key : keys_to_get) { - if (!mixed_pipeline_request.Get(key)) { - LOG(ERROR) << "Fail to add GET request for key: " << key; - return -1; - } - } - - std::cout << "Sending mixed pipeline GET request, pipelined count: " - << mixed_pipeline_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &mixed_pipeline_request, - &mixed_pipeline_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Process each GET response - int successful_gets = 0; - int failed_gets = 0; - - for (size_t i = 0; i < keys_to_get.size(); ++i) { - std::string value; - uint32_t flags; - uint64_t cas; - - if (mixed_pipeline_response.PopGet(&value, &flags, &cas)) { - std::cout << GREEN << "GET '" << keys_to_get[i] - << "' successful - Value length: " << value.length() - << ", Flags: " << flags << ", CAS: " << cas << RESET - << std::endl; - successful_gets++; - } else { - std::cout << RED << "GET '" << keys_to_get[i] - << "' failed: " << mixed_pipeline_response.LastError() << RESET - << std::endl; - failed_gets++; - } + std::cout << RED << "ADD binprot item3 failed: " << item3_result.error_message << RESET << std::endl; } - std::cout << "Mixed pipeline summary: " << successful_gets << " successful, " - << failed_gets << " failed GET operations" << std::endl; - cntl.Reset(); - // perform an upsert on the existing key - brpc::CouchbaseRequest upsert_request; - brpc::CouchbaseResponse upsert_response; - if (!upsert_request.Upsert( - butil::string_printf("user::test_brpc_binprot"), - R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})", - 0, FLAGS_exptime, 0)) { - LOG(ERROR) << "Fail to add UPSERT request for key: user::test_brpc_binprot"; - return -1; - } - - channel.CallMethod(NULL, &cntl, &upsert_request, &upsert_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - // Check UPSERT operation response status - if (upsert_response.PopUpsert(&cas_value)) { + // Perform an UPSERT on the existing key using high-level method + std::string upsert_key = "user::test_brpc_binprot"; + std::string upsert_value = R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})"; + brpc::CouchbaseOperations::Result upsert_result = couchbase_ops.Upsert(upsert_key, upsert_value); + if (upsert_result.success) { std::cout << GREEN - << "UPSERT operation successful when the document exists in the " - "server, CAS: " - << cas_value << RESET << std::endl; + << "UPSERT operation successful when the document exists in the server" + << RESET << std::endl; } else { std::cout << RED - << "UPSERT operation failed, when the document does exists in " - "the server: " - << upsert_response.LastError() << RESET << std::endl; - } - - cntl.Reset(); - // do the upsert operation - brpc::CouchbaseRequest upsert_request_new_doc; - brpc::CouchbaseResponse upsert_response_new_doc; - if (!upsert_request_new_doc.Upsert( - butil::string_printf("user::test_brpc_new_upsert"), - R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})", - 0, FLAGS_exptime, 0)) { - LOG(ERROR) - << "Fail to add UPSERT request for key: user::test_brpc_new_upsert"; - return -1; - } - - channel.CallMethod(NULL, &cntl, &upsert_request_new_doc, - &upsert_response_new_doc, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - // Check UPSERT operation response status - if (upsert_response_new_doc.PopUpsert(&cas_value)) { + << "UPSERT operation failed when the document exists in the server: " + << upsert_result.error_message << RESET << std::endl; + } + // Do UPSERT operation on a new document using high-level method + std::string new_upsert_key = "user::test_brpc_new_upsert"; + std::string new_upsert_value = R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})"; + brpc::CouchbaseOperations::Result new_upsert_result = couchbase_ops.Upsert(new_upsert_key, new_upsert_value); + if (new_upsert_result.success) { std::cout << GREEN - << "UPSERT operation successful when the document doesn't exists " - "in the server, CAS: " - << cas_value << RESET << std::endl; + << "UPSERT operation successful when the document doesn't exist in the server" + << RESET << std::endl; } else { std::cout << RED - << "UPSERT operation failed when document does not exists in the " - "server: " - << upsert_response_new_doc.LastError() << RESET << std::endl; - } - - cntl.Reset(); - - // check the upserted data - brpc::CouchbaseRequest check_upsert_request; - brpc::CouchbaseResponse check_upsert_response; - if (!check_upsert_request.Get( - butil::string_printf("user::test_brpc_new_upsert"))) { - LOG(ERROR) << "Fail to GET request for key: user::test_brpc_new_upsert"; - return -1; + << "UPSERT operation failed when document does not exist in the server: " + << new_upsert_result.error_message << RESET << std::endl; } - std::cout - << "Sending GET request for user::test_brpc_new_upsert, pipelined count: " - << check_upsert_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &check_upsert_request, &check_upsert_response, - NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check GET operation response status - if (check_upsert_response.PopGet(&value, &flags, &cas)) { + // Check the upserted data using high-level method + std::string check_key = "user::test_brpc_new_upsert"; + brpc::CouchbaseOperations::Result check_result = couchbase_ops.Get(check_key); + if (check_result.success) { std::cout << GREEN - << "GET after UPSERT operation successful - Value: " << value - << ", Flags: " << flags << ", CAS: " << cas << RESET << std::endl; + << "GET after UPSERT operation successful - Value: " << check_result.value + << RESET << std::endl; } else { std::cout << RED << "GET after UPSERT operation failed: " - << check_upsert_response.LastError() << RESET << std::endl; + << check_result.error_message << RESET << std::endl; } - cntl.Reset(); - - // delete the Non-existent Key - brpc::CouchbaseRequest delete_request; - brpc::CouchbaseResponse delete_response; - if (!delete_request.Delete(butil::string_printf("Nonexistent_key"))) { - LOG(ERROR) << "Fail to DELETE request for key: Nonexistent_key"; - return -1; - } - - std::cout << "Sending DELETE request for Nonexistent_key, pipelined count: " - << delete_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &delete_request, &delete_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check DELETE operation response status - if (delete_response.PopDelete()) { + // Delete a non-existent key using high-level method + std::string delete_key = "Nonexistent_key"; + brpc::CouchbaseOperations::Result delete_result = couchbase_ops.Delete(delete_key); + if (delete_result.success) { std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; } else { - std::cout << RED << "DELETE operation failed: as expected" - << delete_response.LastError() << RESET << std::endl; + std::cout << RED << "DELETE operation failed: as expected " + << delete_result.error_message << RESET << std::endl; } - cntl.Reset(); - - // delete the existing key - brpc::CouchbaseRequest delete_existing_request; - brpc::CouchbaseResponse delete_existing_response; - if (!delete_existing_request.Delete( - butil::string_printf("user::test_brpc_binprot"))) { - LOG(ERROR) << "Fail to DELETE request for key: user::test_brpc_binprot"; - return -1; - } - - std::cout - << "Sending DELETE request for user::test_brpc_binprot, pipelined count: " - << delete_existing_request.pipelined_count() << std::endl; - channel.CallMethod(NULL, &cntl, &delete_existing_request, - &delete_existing_response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase, " << cntl.ErrorText(); - return -1; - } - - // Check DELETE operation response status - if (delete_existing_response.PopDelete()) { + // Delete the existing key using high-level method + std::string delete_existing_key = "user::test_brpc_binprot"; + brpc::CouchbaseOperations::Result delete_existing_result = couchbase_ops.Delete(delete_existing_key); + if (delete_existing_result.success) { std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; } else { std::cout << RED << "DELETE operation failed: " - << delete_existing_response.LastError() << RESET << std::endl; + << delete_existing_result.error_message << RESET << std::endl; } - cntl.Reset(); - // Retrieve Collection ID for scope `_default` and collection // `testing_collection` - - brpc::CouchbaseRequest get_collection_request; - brpc::CouchbaseResponse get_collection_response; - uint8_t testing_collection_id = 0; // will hold the numeric collection id const std::string scope_name = "_default"; // default scope std::string collection_name = "testing_collection"; // target collection // enter collection name as user input @@ -598,161 +213,48 @@ int main() { if (!user_input.empty()) { collection_name = user_input; } - - if (!get_collection_request.GetCollectionId(scope_name.c_str(), - collection_name.c_str())) { - LOG(ERROR) << "Fail to build GetCollectionId request for scope '" - << scope_name << "' collection '" << collection_name << "'"; - return -1; - } - - channel.CallMethod(NULL, &cntl, &get_collection_request, - &get_collection_response, NULL); - - if (cntl.Failed()) { - LOG(ERROR) << "Fail to access Couchbase for GetCollectionId, " - << cntl.ErrorText(); - return -1; - } - - if (get_collection_response.PopCollectionId(&testing_collection_id)) { - std::cout << GREEN << "Retrieved collection id for _default." - << collection_name << " = " - << static_cast(testing_collection_id) - << " dec=" << static_cast(testing_collection_id) - << ", hex=0x" << std::hex - << static_cast(testing_collection_id) << RESET - << std::endl; - } else { - std::cout << RED << "Failed to retrieve collection id for _default." - << collection_name << ": " << get_collection_response.LastError() - << RESET << std::endl; - // We continue, but subsequent collection operations will skip if id=0 - } - cntl.Reset(); // ------------------------------------------------------------------ // Collection-scoped CRUD operations (only if collection id was retrieved) // ------------------------------------------------------------------ - if (testing_collection_id != 0) { - // 1. ADD in collection - brpc::CouchbaseRequest coll_add_req; - brpc::CouchbaseResponse coll_add_resp; - const std::string coll_key = "user::collection_doc"; - if (!coll_add_req.Add( - coll_key.c_str(), R"({"type":"collection","op":"add","v":1})", - 0xabcddcba, FLAGS_exptime, 0, collection_name)) { - LOG(ERROR) << "Fail to build collection ADD request"; + // 1. ADD in collection using high-level method + std::string coll_key = "user::collection_doc"; + std::string coll_value = R"({"type":"collection","op":"add","v":1})"; + brpc::CouchbaseOperations::Result coll_add_result = couchbase_ops.Add(coll_key, coll_value, collection_name); + if (coll_add_result.success) { + std::cout << GREEN << "Collection ADD success" << RESET << std::endl; } else { - channel.CallMethod(NULL, &cntl, &coll_add_req, &coll_add_resp, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Collection ADD RPC failed: " << cntl.ErrorText(); - } else { - uint64_t ccas; - if (coll_add_resp.PopAdd(&ccas)) { - std::cout << GREEN << "Collection ADD success, CAS=" << ccas << RESET - << std::endl; - } else { - std::cout << RED - << "Collection ADD failed: " << coll_add_resp.LastError() - << RESET << std::endl; - } - } - cntl.Reset(); + std::cout << RED << "Collection ADD failed: " << coll_add_result.error_message << RESET << std::endl; } - // 2. GET from collection - brpc::CouchbaseRequest coll_get_req; - brpc::CouchbaseResponse coll_get_resp; - if (!coll_get_req.Get(coll_key.c_str(), collection_name)) { - LOG(ERROR) << "Fail to build collection GET request"; + // 2. GET from collection using high-level method + brpc::CouchbaseOperations::Result coll_get_result = couchbase_ops.Get(coll_key, collection_name); + if (coll_get_result.success) { + std::cout << GREEN << "Collection GET success value=" << coll_get_result.value << RESET << std::endl; } else { - channel.CallMethod(NULL, &cntl, &coll_get_req, &coll_get_resp, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Collection GET RPC failed: " << cntl.ErrorText(); - } else { - std::string v; - uint32_t f = 0; - uint64_t ccas = 0; - if (coll_get_resp.PopGet(&v, &f, &ccas)) { - std::cout << GREEN << "Collection GET success value=" << v - << ", CAS=" << ccas << RESET << std::endl; - } else { - std::cout << RED - << "Collection GET failed: " << coll_get_resp.LastError() - << RESET << std::endl; - } - } - cntl.Reset(); + std::cout << RED << "Collection GET failed: " << coll_get_result.error_message << RESET << std::endl; } - // 3. UPSERT in collection - brpc::CouchbaseRequest coll_upsert_req; - brpc::CouchbaseResponse coll_upsert_resp; - if (!coll_upsert_req.Upsert( - coll_key.c_str(), R"({"type":"collection","op":"upsert","v":2})", 0, - FLAGS_exptime, 0, collection_name)) { - LOG(ERROR) << "Fail to build collection UPSERT request"; + // 3. UPSERT in collection using high-level method + std::string coll_upsert_value = R"({"type":"collection","op":"upsert","v":2})"; + brpc::CouchbaseOperations::Result coll_upsert_result = couchbase_ops.Upsert(coll_key, coll_upsert_value, collection_name); + if (coll_upsert_result.success) { + std::cout << GREEN << "Collection UPSERT success" << RESET << std::endl; } else { - channel.CallMethod(NULL, &cntl, &coll_upsert_req, &coll_upsert_resp, - NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Collection UPSERT RPC failed: " << cntl.ErrorText(); - } else { - uint64_t ccas; - if (coll_upsert_resp.PopUpsert(&ccas)) { - std::cout << GREEN << "Collection UPSERT success, CAS=" << ccas - << RESET << std::endl; - } else { - std::cout << RED << "Collection UPSERT failed: " - << coll_upsert_resp.LastError() << RESET << std::endl; - } - } - cntl.Reset(); + std::cout << RED << "Collection UPSERT failed: " << coll_upsert_result.error_message << RESET << std::endl; } - // 4. GET again to verify upsert - brpc::CouchbaseRequest coll_get2_req; - brpc::CouchbaseResponse coll_get2_resp; - if (coll_get2_req.Get(coll_key.c_str(), collection_name)) { - channel.CallMethod(NULL, &cntl, &coll_get2_req, &coll_get2_resp, NULL); - if (!cntl.Failed()) { - std::string v; - uint32_t f = 0; - uint64_t ccas = 0; - if (coll_get2_resp.PopGet(&v, &f, &ccas)) { - std::cout << GREEN << "Collection GET(after upsert) value=" << v - << RESET << std::endl; - } - } - cntl.Reset(); + // 4. GET again to verify upsert using high-level method + brpc::CouchbaseOperations::Result coll_get2_result = couchbase_ops.Get(coll_key, collection_name); + if (coll_get2_result.success) { + std::cout << GREEN << "Collection GET(after upsert) value=" << coll_get2_result.value << RESET << std::endl; } - // 5. DELETE from collection - brpc::CouchbaseRequest coll_del_req; - brpc::CouchbaseResponse coll_del_resp; - if (!coll_del_req.Delete(coll_key.c_str(), collection_name)) { - LOG(ERROR) << "Fail to build collection DELETE request"; + // 5. DELETE from collection using high-level method + brpc::CouchbaseOperations::Result coll_del_result = couchbase_ops.Delete(coll_key, collection_name); + if (coll_del_result.success) { + std::cout << GREEN << "Collection DELETE success" << RESET << std::endl; } else { - channel.CallMethod(NULL, &cntl, &coll_del_req, &coll_del_resp, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Collection DELETE RPC failed: " << cntl.ErrorText(); - } else { - if (coll_del_resp.PopDelete()) { - std::cout << GREEN << "Collection DELETE success" << RESET - << std::endl; - } else { - std::cout << RED - << "Collection DELETE failed: " << coll_del_resp.LastError() - << RESET << std::endl; - } - } - cntl.Reset(); + std::cout << RED << "Collection DELETE failed: " << coll_del_result.error_message << RESET << std::endl; } - } else { - std::cout << RED - << "Skipping collection-scoped CRUD operations (collection id " - "not available)" - << RESET << std::endl; - } return 0; } diff --git a/example/couchbase_c++/multithreaded_couchbase_client.cpp b/example/couchbase_c++/multithreaded_couchbase_client.cpp index 9fca9eb59f..3438317f9b 100644 --- a/example/couchbase_c++/multithreaded_couchbase_client.cpp +++ b/example/couchbase_c++/multithreaded_couchbase_client.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include #include #include +#include #include #include @@ -57,82 +57,37 @@ struct ThreadArgs { std::string bucket_name; }; -// Simple operation function -bool perform_operation(brpc::Channel& channel, const std::string& key, const std::string& collection = "") { - brpc::Controller cntl; - brpc::CouchbaseRequest request; - brpc::CouchbaseResponse response; - +// Simple operation function using high-level API +bool perform_operation(brpc::CouchbaseOperations& couchbase_ops, const std::string& key, const std::string& collection = "_default") { // Simple ADD operation - std::string value = butil::string_printf(R"({"thread_id": %d, "timestamp": %lld})", - bthread_self(), butil::gettimeofday_us()); - - bool success = collection.empty() ? - request.Add(key.c_str(), value.c_str(), 0xdeadbeef, 300, 0) : - request.Add(key.c_str(), value.c_str(), 0xdeadbeef, 300, 0, collection); - - if (!success) return false; + std::string value = butil::string_printf(R"({"thread_id": %llu, "timestamp": %lld})", + (unsigned long long)bthread_self(), butil::gettimeofday_us()); - channel.CallMethod(NULL, &cntl, &request, &response, NULL); - if (cntl.Failed()) return false; - - uint64_t cas_value; - return response.PopAdd(&cas_value); + brpc::CouchbaseOperations::Result result = couchbase_ops.Add(key, value, collection); + return result.success; } -// Thread worker function - very simple +// Thread worker function using high-level API void* thread_worker(void* arg) { ThreadArgs* args = static_cast(arg); LOG(INFO) << "Thread " << args->thread_id << " starting on bucket " << args->bucket_name; - // One channel per thread as requested - brpc::Channel channel; - brpc::ChannelOptions options; - options.protocol = brpc::PROTOCOL_COUCHBASE; - options.connection_type = FLAGS_connection_type; - options.timeout_ms = 5000; - - if (channel.Init(FLAGS_server.c_str(), "", &options) != 0) { - LOG(ERROR) << "Thread " << args->thread_id << ": Failed to init channel"; - return NULL; - } - - // Simple authentication - brpc::Controller auth_cntl; - brpc::CouchbaseRequest auth_request; - brpc::CouchbaseResponse auth_response; - - if (!auth_request.Authenticate(g_config.username.c_str(), g_config.password.c_str(), &channel, FLAGS_server)) { - LOG(ERROR) << "Thread " << args->thread_id << ": Auth request failed"; - return NULL; - } + // Create CouchbaseOperations instance for this thread + brpc::CouchbaseOperations couchbase_ops; - channel.CallMethod(NULL, &auth_cntl, &auth_request, &auth_response, NULL); - if (auth_cntl.Failed()) { - LOG(ERROR) << "Thread " << args->thread_id << ": Auth failed"; + // Authentication using high-level method + brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( + g_config.username, g_config.password, FLAGS_server, false, ""); + if (!auth_result.success) { + LOG(ERROR) << "Thread " << args->thread_id << ": Auth failed - " << auth_result.error_message; return NULL; } - // Select bucket - brpc::Controller bucket_cntl; - brpc::CouchbaseRequest bucket_request; - brpc::CouchbaseResponse bucket_response; - - if (!bucket_request.SelectBucket(args->bucket_name.c_str())) { - LOG(ERROR) << "Thread " << args->thread_id << ": Bucket request failed"; - return NULL; - } - - channel.CallMethod(NULL, &bucket_cntl, &bucket_request, &bucket_response, NULL); - if (bucket_cntl.Failed()) { - LOG(ERROR) << "Thread " << args->thread_id << ": Bucket selection failed"; - return NULL; - } - - if (!bucket_response.PopSelectBucket(NULL, args->bucket_name.c_str())) { - LOG(ERROR) << "Thread " << args->thread_id << ": Bucket selection failed"; - LOG(ERROR) << bucket_response.LastError(); + // Select bucket using high-level method + brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.SelectBucket(args->bucket_name); + if (!bucket_result.success) { + LOG(ERROR) << "Thread " << args->thread_id << ": Bucket selection failed - " << bucket_result.error_message; return NULL; } @@ -143,13 +98,13 @@ void* thread_worker(void* arg) { for (int i = 0; i < FLAGS_operations_per_thread; ++i) { std::string key = butil::string_printf("thread_%d_op_%d", args->thread_id, i); - // Choose collection randomly if available - std::string collection = ""; + // Choose collection if available, otherwise use default + std::string collection = "_default"; if (!g_config.collection_names.empty()) { collection = g_config.collection_names[i % g_config.collection_names.size()]; } - if (perform_operation(channel, key, collection)) { + if (perform_operation(couchbase_ops, key, collection)) { success_count++; } @@ -194,7 +149,7 @@ void get_config() { } int main(int argc, char* argv[]) { - google::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); std::cout << GREEN << "Starting 16 bthreads (4 per bucket)" << RESET << std::endl; diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index d943831c0a..bd5274540c 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -39,8 +39,8 @@ namespace { } // Static member definitions -CouchbaseMetadataTracking* CouchbaseRequest::metadata_tracking = &common_metadata_tracking; -CouchbaseMetadataTracking* CouchbaseResponse::metadata_tracking = &common_metadata_tracking; +CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseRequest::metadata_tracking = &common_metadata_tracking; +CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseResponse::metadata_tracking = &common_metadata_tracking; bool brpc::CouchbaseMetadataTracking::set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string &server) { std::unique_lock write_lock(rw_thread_to_channel_info_mutex); @@ -255,7 +255,7 @@ bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest(const string& return true; } -uint32_t CouchbaseRequest::hash_crc32(const char* key, size_t key_length) { +uint32_t CouchbaseOperations::CouchbaseRequest::hash_crc32(const char* key, size_t key_length) { static const uint32_t crc32tab[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, @@ -315,38 +315,23 @@ uint32_t CouchbaseRequest::hash_crc32(const char* key, size_t key_length) { #endif } -// redefinition -CouchbaseRequest::CouchbaseRequest() - : NonreflectableMessage() { - metadata_tracking = &common_metadata_tracking; - SharedCtor(); -} - -CouchbaseRequest::CouchbaseRequest(const CouchbaseRequest& from) - : NonreflectableMessage(from) { - SharedCtor(); - MergeFrom(from); -} - -void CouchbaseRequest::SharedCtor() { +void CouchbaseOperations::CouchbaseRequest::SharedCtor() { _pipelined_count = 0; _cached_size_ = 0; } -CouchbaseRequest::~CouchbaseRequest() { SharedDtor(); } +void CouchbaseOperations::CouchbaseRequest::SharedDtor() {} -void CouchbaseRequest::SharedDtor() {} +void CouchbaseOperations::CouchbaseRequest::SetCachedSize(int size) const { _cached_size_ = size; } -void CouchbaseRequest::SetCachedSize(int size) const { _cached_size_ = size; } - -void CouchbaseRequest::Clear() { +void CouchbaseOperations::CouchbaseRequest::Clear() { _buf.clear(); _pipelined_count = 0; } // Support for scope level collections will be added in future. // Get the Scope ID for a given scope name -// bool CouchbaseRequest::GetScopeId(const butil::StringPiece& scope_name) { +// bool CouchbaseOperations::CouchbaseRequest::GetScopeId(const butil::StringPiece& scope_name) { // if (scope_name.empty()) { // LOG(ERROR) << "Empty scope name"; // return false; @@ -373,7 +358,7 @@ void CouchbaseRequest::Clear() { // return true; // } -bool CouchbaseRequest::SelectBucket(const butil::StringPiece& bucket_name) { +bool CouchbaseOperations::CouchbaseRequest::SelectBucketRequest(const butil::StringPiece& bucket_name) { if (bucket_name.empty()) { LOG(ERROR) << "Empty bucket name"; return false; @@ -406,7 +391,7 @@ bool CouchbaseRequest::SelectBucket(const butil::StringPiece& bucket_name) { // This is typically the first request sent after connecting to the server. // It includes the agent name and a randomly generated connection ID in JSON // format. -bool CouchbaseRequest::HelloRequest() { +bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { std::string agent = "brpc/1.0.0 ("; #ifdef __APPLE__ agent += "Darwin/"; @@ -482,7 +467,7 @@ bool CouchbaseRequest::HelloRequest() { return true; } -bool CouchbaseRequest::Authenticate(const butil::StringPiece& username, +bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest(const butil::StringPiece& username, const butil::StringPiece& password, brpc::Channel *channel, const string server) { @@ -537,7 +522,7 @@ bool CouchbaseRequest::Authenticate(const butil::StringPiece& username, return true; } -bool CouchbaseRequest::MergePartialFromCodedStream( +bool CouchbaseOperations::CouchbaseRequest::MergePartialFromCodedStream( ::google::protobuf::io::CodedInputStream* input) { LOG(WARNING) << "You're not supposed to parse a CouchbaseRequest"; @@ -573,7 +558,7 @@ bool CouchbaseRequest::MergePartialFromCodedStream( return true; } -void CouchbaseRequest::SerializeWithCachedSizes( +void CouchbaseOperations::CouchbaseRequest::SerializeWithCachedSizes( ::google::protobuf::io::CodedOutputStream* output) const { LOG(WARNING) << "You're not supposed to serialize a CouchbaseRequest"; @@ -586,21 +571,21 @@ void CouchbaseRequest::SerializeWithCachedSizes( } } -size_t CouchbaseRequest::ByteSizeLong() const { +size_t CouchbaseOperations::CouchbaseRequest::ByteSizeLong() const { int total_size = static_cast(_buf.size()); _cached_size_ = total_size; return total_size; } -void CouchbaseRequest::MergeFrom(const CouchbaseRequest& from) { +void CouchbaseOperations::CouchbaseRequest::MergeFrom(const CouchbaseRequest& from) { CHECK_NE(&from, this); _buf.append(from._buf); _pipelined_count += from._pipelined_count; } -bool CouchbaseRequest::IsInitialized() const { return _pipelined_count != 0; } +bool CouchbaseOperations::CouchbaseRequest::IsInitialized() const { return _pipelined_count != 0; } -void CouchbaseRequest::Swap(CouchbaseRequest* other) { +void CouchbaseOperations::CouchbaseRequest::Swap(CouchbaseRequest* other) { if (other != this) { _buf.swap(other->_buf); std::swap(_pipelined_count, other->_pipelined_count); @@ -608,38 +593,23 @@ void CouchbaseRequest::Swap(CouchbaseRequest* other) { } } -::google::protobuf::Metadata CouchbaseRequest::GetMetadata() const { +::google::protobuf::Metadata CouchbaseOperations::CouchbaseRequest::GetMetadata() const { ::google::protobuf::Metadata metadata{}; metadata.descriptor = CouchbaseRequestBase::descriptor(); metadata.reflection = nullptr; return metadata; } -// redifinition -CouchbaseResponse::CouchbaseResponse() - : NonreflectableMessage() { - SharedCtor(); -} - -// redifinition -CouchbaseResponse::CouchbaseResponse(const CouchbaseResponse& from) - : NonreflectableMessage(from) { - metadata_tracking = &common_metadata_tracking; - SharedCtor(); - MergeFrom(from); -} -void CouchbaseResponse::SharedCtor() { _cached_size_ = 0; } +void CouchbaseOperations::CouchbaseResponse::SharedCtor() { _cached_size_ = 0; } -// redifinition -CouchbaseResponse::~CouchbaseResponse() { SharedDtor(); } -void CouchbaseResponse::SharedDtor() {} +void CouchbaseOperations::CouchbaseResponse::SharedDtor() {} -void CouchbaseResponse::SetCachedSize(int size) const { _cached_size_ = size; } +void CouchbaseOperations::CouchbaseResponse::SetCachedSize(int size) const { _cached_size_ = size; } -void CouchbaseResponse::Clear() {} +void CouchbaseOperations::CouchbaseResponse::Clear() {} -bool CouchbaseResponse::MergePartialFromCodedStream( +bool CouchbaseOperations::CouchbaseResponse::MergePartialFromCodedStream( ::google::protobuf::io::CodedInputStream* input) { LOG(WARNING) << "You're not supposed to parse a CouchbaseResponse"; @@ -653,7 +623,7 @@ bool CouchbaseResponse::MergePartialFromCodedStream( return true; } -void CouchbaseResponse::SerializeWithCachedSizes( +void CouchbaseOperations::CouchbaseResponse::SerializeWithCachedSizes( ::google::protobuf::io::CodedOutputStream* output) const { LOG(WARNING) << "You're not supposed to serialize a CouchbaseResponse"; @@ -666,28 +636,28 @@ void CouchbaseResponse::SerializeWithCachedSizes( } } -size_t CouchbaseResponse::ByteSizeLong() const { +size_t CouchbaseOperations::CouchbaseResponse::ByteSizeLong() const { int total_size = static_cast(_buf.size()); _cached_size_ = total_size; return total_size; } -void CouchbaseResponse::MergeFrom(const CouchbaseResponse& from) { +void CouchbaseOperations::CouchbaseResponse::MergeFrom(const CouchbaseResponse& from) { CHECK_NE(&from, this); _err = from._err; _buf.append(from._buf); } -bool CouchbaseResponse::IsInitialized() const { return !_buf.empty(); } +bool CouchbaseOperations::CouchbaseResponse::IsInitialized() const { return !_buf.empty(); } -void CouchbaseResponse::Swap(CouchbaseResponse* other) { +void CouchbaseOperations::CouchbaseResponse::Swap(CouchbaseResponse* other) { if (other != this) { _buf.swap(other->_buf); std::swap(_cached_size_, other->_cached_size_); } } -::google::protobuf::Metadata CouchbaseResponse::GetMetadata() const { +::google::protobuf::Metadata CouchbaseOperations::CouchbaseResponse::GetMetadata() const { ::google::protobuf::Metadata metadata{}; metadata.descriptor = CouchbaseResponseBase::descriptor(); metadata.reflection = nullptr; @@ -696,7 +666,7 @@ ::google::protobuf::Metadata CouchbaseResponse::GetMetadata() const { // =================================================================== -const char* CouchbaseResponse::status_str(Status st) { +const char* CouchbaseOperations::CouchbaseResponse::status_str(Status st) { switch (st) { case STATUS_SUCCESS: return "SUCCESS"; @@ -815,7 +785,7 @@ const char* CouchbaseResponse::status_str(Status st) { } // Helper method to format error messages with status codes -std::string CouchbaseResponse::format_error_message( +std::string CouchbaseOperations::CouchbaseResponse::format_error_message( uint16_t status_code, const std::string& operation, const std::string& error_msg) { if (error_msg.empty()) { @@ -832,7 +802,7 @@ std::string CouchbaseResponse::format_error_message( // MUST NOT have extras. // MUST have key. // MUST NOT have value. -bool CouchbaseRequest::GetOrDelete(uint8_t command, +bool CouchbaseOperations::CouchbaseRequest::GetOrDelete(uint8_t command, const butil::StringPiece& key, uint8_t coll_id) { // Collection ID @@ -863,7 +833,7 @@ bool CouchbaseRequest::GetOrDelete(uint8_t command, return true; } -bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, CouchbaseMetadataTracking *metadata_tracking){ +bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, brpc::CouchbaseMetadataTracking *metadata_tracking){ if(collection_name.empty()){ LOG(ERROR) << "Empty collection name"; return false; @@ -881,12 +851,12 @@ bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, LOG(ERROR) << "No bucket selected for this thread, make sure to call SelectBucket() first"; return false; } - CouchbaseMetadataTracking::CollectionManifest manifest; + brpc::CouchbaseMetadataTracking::CollectionManifest manifest; if(!metadata_tracking->get_bucket_to_collection_manifest(bthread_self(), channel_info.server, channel_info.selected_bucket, &manifest)){ LOG(INFO) << "No cached collection manifest found for bucket " << channel_info.selected_bucket << " on server " << channel_info.server << ", fetching from server"; // No cached manifest found, fetch from server - CouchbaseRequest temp_get_manifest_request; - CouchbaseResponse temp_get_manifest_response; + CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; + CouchbaseOperations::CouchbaseResponse temp_get_manifest_response; brpc::Controller temp_cntl; brpc::Channel *channel = channel_info.channel; temp_get_manifest_request.GetCollectionManifest(); @@ -925,7 +895,7 @@ bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, } } -bool CouchbaseRequest::Get(const butil::StringPiece& key, string collection_name) { +bool CouchbaseOperations::CouchbaseRequest::GetRequest(const butil::StringPiece& key, string collection_name) { uint8_t coll_id = 0; // default collection ID if(collection_name != "_default"){ if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ @@ -935,7 +905,7 @@ bool CouchbaseRequest::Get(const butil::StringPiece& key, string collection_name return GetOrDelete(policy::CB_BINARY_GET, key, coll_id); } -bool CouchbaseRequest::Delete(const butil::StringPiece& key, string collection_name) { +bool CouchbaseOperations::CouchbaseRequest::DeleteRequest(const butil::StringPiece& key, string collection_name) { uint8_t coll_id = 0; // default collection ID if(collection_name != "_default"){ if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ @@ -962,7 +932,7 @@ BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); // 0| Expiration | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseRequest::Flush(uint32_t timeout) { +bool CouchbaseOperations::CouchbaseRequest::FlushRequest(uint32_t timeout) { const uint8_t FLUSH_EXTRAS = (timeout == 0 ? 0 : 4); FlushHeaderWithExtras header_with_extras = { {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_FLUSH, 0, FLUSH_EXTRAS, @@ -994,7 +964,7 @@ bool CouchbaseRequest::Flush(uint32_t timeout) { // 0| Flags | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseResponse::PopGet(butil::IOBuf* value, uint32_t* flags, +bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; @@ -1066,7 +1036,7 @@ bool CouchbaseResponse::PopGet(butil::IOBuf* value, uint32_t* flags, return true; } -bool CouchbaseResponse::PopGet(std::string* value, uint32_t* flags, +bool CouchbaseOperations::CouchbaseResponse::PopGet(std::string* value, uint32_t* flags, uint64_t* cas_value) { butil::IOBuf tmp; if (PopGet(&tmp, flags, cas_value)) { @@ -1079,10 +1049,10 @@ bool CouchbaseResponse::PopGet(std::string* value, uint32_t* flags, // MUST NOT have extras // MUST NOT have key // MUST NOT have value -bool CouchbaseResponse::PopDelete() { +bool CouchbaseOperations::CouchbaseResponse::PopDelete() { return PopStore(policy::CB_BINARY_DELETE, NULL); } -bool CouchbaseResponse::PopFlush() { +bool CouchbaseOperations::CouchbaseResponse::PopFlush() { return PopStore(policy::CB_BINARY_FLUSH, NULL); } @@ -1107,7 +1077,7 @@ const size_t STORE_EXTRAS = // 4| Expiration | // +---------------+---------------+---------------+---------------+ // Total 8 bytes -bool CouchbaseRequest::Store(uint8_t command, const butil::StringPiece& key, +bool CouchbaseOperations::CouchbaseRequest::Store(uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, uint8_t coll_id) { @@ -1146,7 +1116,7 @@ bool CouchbaseRequest::Store(uint8_t command, const butil::StringPiece& key, // MUST NOT have extras // MUST NOT have key // MUST NOT have value -bool CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1191,7 +1161,7 @@ bool CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { return true; } -const char* CouchbaseResponse::couchbase_binary_command_to_string(uint8_t cmd) { +const char* CouchbaseOperations::CouchbaseResponse::couchbase_binary_command_to_string(uint8_t cmd) { switch (cmd) { case 0x1f: return "CB_HELLO_SELECT_FEATURES"; @@ -1280,7 +1250,7 @@ const char* CouchbaseResponse::couchbase_binary_command_to_string(uint8_t cmd) { } } -bool CouchbaseRequest::Upsert(const butil::StringPiece& key, +bool CouchbaseOperations::CouchbaseRequest::UpsertRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name) { @@ -1294,36 +1264,36 @@ bool CouchbaseRequest::Upsert(const butil::StringPiece& key, coll_id); } -// collection id and manifest are both returned when you call GetCollectionId -bool CouchbaseRequest::GetCollectionId( - const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name) { - // Format the collection path as "scope.collection" - std::string collection_path = - scope_name.as_string() + "." + collection_name.as_string(); +// Using GetCollectionManifest instead of fetching collection ID directly +// bool CouchbaseOperations::CouchbaseRequest::GetCollectionId( +// const butil::StringPiece& scope_name, +// const butil::StringPiece& collection_name) { +// // Format the collection path as "scope.collection" +// std::string collection_path = +// scope_name.as_string() + "." + collection_name.as_string(); - const policy::CouchbaseRequestHeader header = { - policy::CB_MAGIC_REQUEST, - policy::CB_COLLECTIONS_GET_CID, - butil::HostToNet16(collection_path.size()), - 0, // no extras - policy::CB_BINARY_RAW_BYTES, - 0, // no vbucket - butil::HostToNet32(collection_path.size()), - 0, // opaque - 0 // no CAS - }; - if (_buf.append(&header, sizeof(header))) { - return false; - } - if (_buf.append(collection_path.data(), collection_path.size())) { - return false; - } - ++_pipelined_count; - return true; -} +// const policy::CouchbaseRequestHeader header = { +// policy::CB_MAGIC_REQUEST, +// policy::CB_COLLECTIONS_GET_CID, +// butil::HostToNet16(collection_path.size()), +// 0, // no extras +// policy::CB_BINARY_RAW_BYTES, +// 0, // no vbucket +// butil::HostToNet32(collection_path.size()), +// 0, // opaque +// 0 // no CAS +// }; +// if (_buf.append(&header, sizeof(header))) { +// return false; +// } +// if (_buf.append(collection_path.data(), collection_path.size())) { +// return false; +// } +// ++_pipelined_count; +// return true; +// } -bool CouchbaseRequest::GetCollectionManifest() { +bool CouchbaseOperations::CouchbaseRequest::GetCollectionManifest() { const policy::CouchbaseRequestHeader header = { policy::CB_MAGIC_REQUEST, policy::CB_GET_COLLECTIONS_MANIFEST, @@ -1342,21 +1312,76 @@ bool CouchbaseRequest::GetCollectionManifest() { return true; } -bool CouchbaseRequest::Add(const butil::StringPiece& key, +// bool RefreshCollectionManifest(brpc::Channel* channel) { +// // first fetch the manifest +// // then compare the UID with the cached one +// CouchbaseRequest temp_get_manifest_request; +// CouchbaseResponse temp_get_manifest_response; +// brpc::Controller temp_cntl; +// temp_get_manifest_request.GetCollectionManifest(); +// channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, +// &temp_get_manifest_response, NULL); +// if (temp_cntl.Failed()) { +// LOG(ERROR) << "Failed to get collection manifest: " << temp_cntl.ErrorText(); +// return false; +// } +// string manifest_json; +// if (!temp_get_manifest_response.PopManifest(&manifest_json)) { +// LOG(ERROR) << "Failed to parse response for collection Manifest: " << temp_get_manifest_response.LastError(); +// return false; +// } +// // Compare the UID with the cached one +// // If they are different, refresh the cache +// brpc::CouchbaseMetadataTracking::CollectionManifest manifest; +// if(!common_metadata_tracking.json_to_collection_manifest(manifest_json, &manifest)){ +// LOG(ERROR) << "Failed to parse collection manifest JSON"; +// return false; +// } +// brpc::CouchbaseMetadataTracking::ChannelInfo temp_channel_info; +// common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &temp_channel_info); +// if(temp_channel_info.server.empty() || temp_channel_info.selected_bucket.empty()){ +// LOG(ERROR) << "No channel info found for this thread, make sure to call Authenticate() and SelectBucket() first"; +// return false; +// } +// brpc::CouchbaseMetadataTracking::CollectionManifest cached_manifest; +// if(!common_metadata_tracking.get_bucket_to_collection_manifest(bthread_self(), temp_channel_info.server, temp_channel_info.selected_bucket, &cached_manifest)){ +// // No cached manifest found, set the new one +// if(!common_metadata_tracking.set_bucket_to_collection_manifest(bthread_self(), manifest)){ +// LOG(ERROR) << "Failed to cache collection manifest for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; +// return false; +// } +// LOG(INFO) << "Cached collection manifest for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; +// return true; +// } +// if(manifest.uid != cached_manifest.uid) { +// LOG(INFO) << "Collection manifest has changed for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; +// if(!common_metadata_tracking.set_bucket_to_collection_manifest(bthread_self(), manifest)){ +// LOG(ERROR) << "Failed to update cached collection manifest for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; +// return false; +// } +// LOG(INFO) << "Updated cached collection manifest for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; +// } +// else{ +// LOG(INFO) << "Collection manifest is up-to-date for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; +// } +// return true; +// } + +bool CouchbaseOperations::CouchbaseRequest::AddRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name) { uint8_t coll_id = 0; // default collection ID - if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ - return false; - } - } + // if(collection_name != "_default"){ + // if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + // return false; + // } + // } return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, coll_id); } -bool CouchbaseRequest::Replace(const butil::StringPiece& key, +bool CouchbaseOperations::CouchbaseRequest::ReplaceRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name) { @@ -1370,7 +1395,7 @@ bool CouchbaseRequest::Replace(const butil::StringPiece& key, coll_id); } -bool CouchbaseRequest::Append(const butil::StringPiece& key, +bool CouchbaseOperations::CouchbaseRequest::AppendRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name) { @@ -1388,7 +1413,7 @@ bool CouchbaseRequest::Append(const butil::StringPiece& key, coll_id); } -bool CouchbaseRequest::Prepend(const butil::StringPiece& key, +bool CouchbaseOperations::CouchbaseRequest::PrependRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name) { @@ -1406,22 +1431,22 @@ bool CouchbaseRequest::Prepend(const butil::StringPiece& key, coll_id); } -bool CouchbaseResponse::PopUpsert(uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopUpsert(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_SET, cas_value); } -bool CouchbaseResponse::PopAdd(uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopAdd(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_ADD, cas_value); } -bool CouchbaseResponse::PopReplace(uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopReplace(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_REPLACE, cas_value); } -bool CouchbaseResponse::PopAppend(uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopAppend(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_APPEND, cas_value); } -bool CouchbaseResponse::PopPrepend(uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopPrepend(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_PREPEND, cas_value); } -bool CouchbaseResponse::PopSelectBucket(uint64_t* cas_value, std::string bucket_name) { +bool CouchbaseOperations::CouchbaseResponse::PopSelectBucket(uint64_t* cas_value, std::string bucket_name) { if(PopStore(policy::CB_SELECT_BUCKET, cas_value) == false){ LOG(ERROR) << "Failed to select bucket: " << _err; return false; @@ -1434,7 +1459,7 @@ bool CouchbaseResponse::PopSelectBucket(uint64_t* cas_value, std::string bucket_ return true; } // Collection-related response method -bool CouchbaseResponse::PopCollectionId(uint8_t* collection_id) { +bool CouchbaseOperations::CouchbaseResponse::PopCollectionId(uint8_t* collection_id) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1504,7 +1529,7 @@ bool CouchbaseResponse::PopCollectionId(uint8_t* collection_id) { return true; } -bool CouchbaseResponse::PopManifest(std::string* manifest_json) { +bool CouchbaseOperations::CouchbaseResponse::PopManifest(std::string* manifest_json) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1592,7 +1617,7 @@ const size_t INCR_EXTRAS = // 16| Expiration | // +---------------+---------------+---------------+---------------+ // Total 20 bytes -bool CouchbaseRequest::Counter(uint8_t command, const butil::StringPiece& key, +bool CouchbaseOperations::CouchbaseRequest::Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, uint64_t initial_value, uint32_t exptime) { IncrHeaderWithExtras header_with_extras = { @@ -1612,14 +1637,14 @@ bool CouchbaseRequest::Counter(uint8_t command, const butil::StringPiece& key, return true; } -bool CouchbaseRequest::Increment(const butil::StringPiece& key, uint64_t delta, +bool CouchbaseOperations::CouchbaseRequest::IncrementRequest(const butil::StringPiece& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name) { return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, exptime); } -bool CouchbaseRequest::Decrement(const butil::StringPiece& key, uint64_t delta, +bool CouchbaseOperations::CouchbaseRequest::DecrementRequest(const butil::StringPiece& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name) { return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, @@ -1637,7 +1662,7 @@ bool CouchbaseRequest::Decrement(const butil::StringPiece& key, uint64_t delta, // | | // +---------------+---------------+---------------+---------------+ // Total 8 bytes -bool CouchbaseResponse::PopCounter(uint8_t command, uint64_t* new_value, +bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; @@ -1693,10 +1718,10 @@ bool CouchbaseResponse::PopCounter(uint8_t command, uint64_t* new_value, return true; } -bool CouchbaseResponse::PopIncrement(uint64_t* new_value, uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopIncrement(uint64_t* new_value, uint64_t* cas_value) { return PopCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); } -bool CouchbaseResponse::PopDecrement(uint64_t* new_value, uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopDecrement(uint64_t* new_value, uint64_t* cas_value) { return PopCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); } @@ -1722,7 +1747,7 @@ const size_t TOUCH_EXTRAS = // 0| Expiration | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseRequest::Touch(const butil::StringPiece& key, uint32_t exptime, +bool CouchbaseOperations::CouchbaseRequest::TouchRequest(const butil::StringPiece& key, uint32_t exptime, string collection_name) { TouchHeaderWithExtras header_with_extras = { {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_TOUCH, @@ -1743,7 +1768,7 @@ bool CouchbaseRequest::Touch(const butil::StringPiece& key, uint32_t exptime, // MUST NOT have extras. // MUST NOT have key. // MUST NOT have value. -bool CouchbaseRequest::Version() { +bool CouchbaseOperations::CouchbaseRequest::VersionRequest() { const policy::CouchbaseRequestHeader header = {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_VERSION, 0, @@ -1763,11 +1788,11 @@ bool CouchbaseRequest::Version() { // MUST NOT have extras. // MUST NOT have key. // MUST have value. -bool CouchbaseResponse::PopTouch() { +bool CouchbaseOperations::CouchbaseResponse::PopTouch() { return PopStore(policy::CB_BINARY_TOUCH, NULL); } -bool CouchbaseResponse::PopVersion(std::string* version) { +bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1813,4 +1838,226 @@ bool CouchbaseResponse::PopVersion(std::string* version) { return true; } -} // namespace brpc +CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, string collection_name) { + //create CouchbaseRequest and CouchbaseResponse objects and then using the channel which is created for this thread in authenticate() use it to call() + CouchbaseRequest request; + CouchbaseResponse response; + brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; + common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); + brpc::Channel *channel = ch_info.channel; + brpc::Controller cntl; + CouchbaseOperations::Result result; + if(request.GetRequest(key, collection_name) == false){ + LOG(ERROR) << "Failed to create Get request for key: " << key; + result.success = false; + result.value = ""; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to get key: " << key << " from Couchbase: " << cntl.ErrorText(); + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + string value; + uint32_t flags = 0; + uint64_t cas = 0; + if(response.PopGet(&value, &flags, &cas) == false){ + result.success = false; + result.value = ""; + result.error_message = response.LastError(); + return result; + } + // Successfully got the value + result.success = true; + result.value = value; + return result; +} + +CouchbaseOperations::Result CouchbaseOperations::Upsert(const string& key, const string& value, string collection_name) { + CouchbaseRequest request; + CouchbaseResponse response; + brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; + common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); + brpc::Channel *channel = ch_info.channel; + brpc::Controller cntl; + CouchbaseOperations::Result result; + if(request.UpsertRequest(key, value, 0, 0, 0, collection_name) == false){ + LOG(ERROR) << "Failed to create Upsert request for key: " << key; + result.success = false; + result.value = ""; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to upsert key: " << key << " to Couchbase: " << cntl.ErrorText(); + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + if(response.PopUpsert(NULL) == false){ + result.success = false; + result.value = ""; + result.error_message = response.LastError(); + return result; + } + // Successfully upserted the value + result.success = true; + result.value = ""; + return result; +} + +CouchbaseOperations::Result CouchbaseOperations::Delete(const string& key, string collection_name) { + CouchbaseRequest request; + CouchbaseResponse response; + brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; + common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); + brpc::Channel *channel = ch_info.channel; + brpc::Controller cntl; + CouchbaseOperations::Result result; + if(request.DeleteRequest(key, collection_name) == false){ + LOG(ERROR) << "Failed to create Delete request for key: " << key; + result.success = false; + result.value = ""; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to delete key: " << key << " from Couchbase: " << cntl.ErrorText(); + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + if(response.PopDelete() == false){ + result.success = false; + result.value = ""; + result.error_message = response.LastError(); + return result; + } + // Successfully deleted the value + result.success = true; + result.value = ""; + return result; +} + +CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, const string& value, string collection_name) { + CouchbaseRequest request; + CouchbaseResponse response; + brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; + common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); + brpc::Channel *channel = ch_info.channel; + brpc::Controller cntl; + CouchbaseOperations::Result result; + if(request.AddRequest(key, value, 0, 0, 0, collection_name) == false){ + LOG(ERROR) << "Failed to create Add request for key: " << key; + result.success = false; + result.value = ""; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to add key: " << key << " to Couchbase: " << cntl.ErrorText(); + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + if(response.PopAdd(NULL) == false){ + result.success = false; + result.value = ""; + result.error_message = response.LastError(); + return result; + } + // Successfully added the value + result.success = true; + result.value = ""; + return result; +} + +CouchbaseOperations::Result CouchbaseOperations::Authenticate(const string& username, const string& password, const string& server_address, bool enable_ssl, string path_to_cert) { + // Create a channel to the Couchbase server + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_COUCHBASE; + options.connection_type = "single"; + options.timeout_ms = 1000; // 1 second + options.max_retry = 3; + + //enable_ssl + if(enable_ssl){ + brpc::ChannelSSLOptions* ssl_options = options.mutable_ssl_options(); + ssl_options->sni_name = server_address; + ssl_options->verify.verify_depth = 1; // Enable certificate verification, to disable SSL set it to 0 + ssl_options->verify.ca_file_path = path_to_cert; // Path to your downloaded TLS certificate + } + CouchbaseOperations::Result result; + brpc::Channel* channel = new brpc::Channel(); + if (channel->Init(server_address.c_str(), &options) != 0) { + LOG(ERROR) << "Failed to initialize Couchbase channel to " << server_address; + delete channel; + result.success = false; + result.value = ""; + result.error_message = "Failed to initialize Couchbase channel"; + return result; + } + // Create a CouchbaseRequest and CouchbaseResponse for authentication + CouchbaseRequest request; + CouchbaseResponse response; + brpc::Controller cntl; + if(request.AuthenticateRequest(username.c_str(), password.c_str(), channel, server_address) == false){ + LOG(ERROR) << "Failed to create Authenticate request for user: " << username; + delete channel; + result.success = false; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to access Couchbase: " << cntl.ErrorText(); + delete channel; + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + // Successfully authenticated + result.success = true; + return result; +} + +CouchbaseOperations::Result CouchbaseOperations::SelectBucket(const string& bucket_name) { + CouchbaseRequest request; + CouchbaseResponse response; + brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; + common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); + brpc::Channel *channel = ch_info.channel; + brpc::Controller cntl; + CouchbaseOperations::Result result; + if(request.SelectBucketRequest(bucket_name.c_str()) == false){ + LOG(ERROR) << "Failed to create Select Bucket request for bucket: " << bucket_name; + result.success = false; + result.value = ""; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to select bucket: " << bucket_name << " from Couchbase: " << cntl.ErrorText(); + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + if(response.PopSelectBucket(NULL, bucket_name) == false){ + result.success = false; + result.value = ""; + result.error_message = response.LastError(); + return result; + } + // Successfully selected the bucket + result.success = true; + result.value = ""; + return result; +} +}// namespace brpc \ No newline at end of file diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index ece93a8bc2..1e99fd5ea2 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -31,7 +31,18 @@ #include using namespace std; +namespace google { namespace protobuf { class Message; } } + namespace brpc { + +// Forward declarations for friend functions +class InputMessageBase; +class Controller; +namespace policy { + void ProcessCouchbaseResponse(InputMessageBase* msg); + void SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); +} + class CouchbaseMetadataTracking{ public: struct ChannelInfo { @@ -67,228 +78,267 @@ class CouchbaseMetadataTracking{ bool json_to_collection_manifest(const string& json, CollectionManifest *manifest); } static common_metadata_tracking; +class CouchbaseOperations { +private: + // Friend functions to allow access to private nested classes + friend bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, brpc::CouchbaseMetadataTracking *metadata_tracking); + friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); + friend void policy::SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); + + class CouchbaseRequest : public NonreflectableMessage { + private: + static brpc::CouchbaseMetadataTracking *metadata_tracking; + int _pipelined_count; + butil::IOBuf _buf; + mutable int _cached_size_; + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + bool GetOrDelete(uint8_t command, const butil::StringPiece& key, + uint8_t coll_id = 0); + bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + + bool Store(uint8_t command, const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, uint32_t exptime, + uint64_t cas_value, uint8_t coll_id = 0); + uint32_t hash_crc32(const char* key, size_t key_length); -class CouchbaseRequest : public NonreflectableMessage { - private: - static CouchbaseMetadataTracking *metadata_tracking; - int _pipelined_count; - butil::IOBuf _buf; - mutable int _cached_size_; - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - bool GetOrDelete(uint8_t command, const butil::StringPiece& key, - uint8_t coll_id = 0); - bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime); - - bool Store(uint8_t command, const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, uint32_t exptime, - uint64_t cas_value, uint8_t coll_id = 0); - uint32_t hash_crc32(const char* key, size_t key_length); - - public: - CouchbaseRequest(); - ~CouchbaseRequest() override; - CouchbaseRequest(const CouchbaseRequest& from); - inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { - CopyFrom(from); - return *this; - } - - bool SelectBucket(const butil::StringPiece& bucket_name); - bool Authenticate(const butil::StringPiece& username, - const butil::StringPiece& password, - brpc::Channel* channel, - const string server); - bool HelloRequest(); - - // Collection Management Method - bool GetCollectionId(const butil::StringPiece& scope_name, - const butil::StringPiece& collection_name); - - bool GetScopeId(const butil::StringPiece& scope_name); - - bool GetCollectionManifest(); - - // Collection-aware document operations - bool Get(const butil::StringPiece& key, string collection_name = "_default"); - - bool Upsert(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); + public: + CouchbaseRequest() : NonreflectableMessage() { + metadata_tracking = &common_metadata_tracking; + SharedCtor(); + } + ~CouchbaseRequest() { SharedDtor(); } + CouchbaseRequest(const CouchbaseRequest& from) : NonreflectableMessage(from) { + SharedCtor(); + MergeFrom(from); + } - bool Add(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); + inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { + CopyFrom(from); + return *this; + } - bool Replace(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); + bool SelectBucketRequest(const butil::StringPiece& bucket_name); + bool AuthenticateRequest(const butil::StringPiece& username, + const butil::StringPiece& password, + brpc::Channel* channel, + const string server); + bool HelloRequest(); - bool Append(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); + // Using GetCollectionManifest instead of fetching collection ID directly + // bool GetCollectionId(const butil::StringPiece& scope_name, + // const butil::StringPiece& collection_name); - bool Prepend(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); + bool GetScopeId(const butil::StringPiece& scope_name); - bool Delete(const butil::StringPiece& key, string collection_name = "_default"); - bool Flush(uint32_t timeout); + bool GetCollectionManifest(); - bool Increment(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); - bool Decrement(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); + bool RefreshCollectionManifest(); - bool Touch(const butil::StringPiece& key, uint32_t exptime, - string collection_name = "_default"); + // Collection-aware document operations + bool GetRequest(const butil::StringPiece& key, string collection_name = "_default"); - bool Version(); + bool UpsertRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default"); - int pipelined_count() const { return _pipelined_count; } + bool AddRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default"); - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - void Swap(CouchbaseRequest* other); - void MergeFrom(const CouchbaseRequest& from) override; - void Clear() override; - bool IsInitialized() const PB_527_OVERRIDE; - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + bool ReplaceRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default"); - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; -}; + bool AppendRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default"); + + bool PrependRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default"); + + bool DeleteRequest(const butil::StringPiece& key, string collection_name = "_default"); + + bool FlushRequest(uint32_t timeout); + + bool IncrementRequest(const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); + bool DecrementRequest(const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); + + bool TouchRequest(const butil::StringPiece& key, uint32_t exptime, + string collection_name = "_default"); -class CouchbaseResponse : public NonreflectableMessage { - private: - string _err; - static CouchbaseMetadataTracking *metadata_tracking; - butil::IOBuf _buf; - mutable int _cached_size_; - bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); - bool PopStore(uint8_t command, uint64_t* cas_value); - - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - - public: - CouchbaseResponse(); - ~CouchbaseResponse() override; - CouchbaseResponse(const CouchbaseResponse& from); - inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { - CopyFrom(from); - return *this; - } - enum Status { - STATUS_SUCCESS = 0x00, - STATUS_KEY_ENOENT = 0x01, - STATUS_KEY_EEXISTS = 0x02, - STATUS_E2BIG = 0x03, - STATUS_EINVAL = 0x04, - STATUS_NOT_STORED = 0x05, - STATUS_DELTA_BADVAL = 0x06, - STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, - STATUS_AUTH_ERROR = 0x20, - STATUS_AUTH_CONTINUE = 0x21, - STATUS_ERANGE = 0x22, - STATUS_ROLLBACK = 0x23, - STATUS_EACCESS = 0x24, - STATUS_NOT_INITIALIZED = 0x25, - STATUS_UNKNOWN_COMMAND = 0x81, - STATUS_ENOMEM = 0x82, - STATUS_NOT_SUPPORTED = 0x83, - STATUS_EINTERNAL = 0x84, - STATUS_EBUSY = 0x85, - STATUS_ETMPFAIL = 0x86, - STATUS_UNKNOWN_COLLECTION = 0x88, - STATUS_NO_COLLECTIONS_MANIFEST = 0x89, - STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, - STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, - STATUS_UNKNOWN_SCOPE = 0x8c, - STATUS_DCP_STREAM_ID_INVALID = 0x8d, - STATUS_DURABILITY_INVALID_LEVEL = 0xa0, - STATUS_DURABILITY_IMPOSSIBLE = 0xa1, - STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, - STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, - STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, - STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, - STATUS_SUBDOC_PATH_MISMATCH = 0xc1, - STATUS_SUBDOC_PATH_EINVAL = 0xc2, - STATUS_SUBDOC_PATH_E2BIG = 0xc3, - STATUS_SUBDOC_DOC_E2DEEP = 0xc4, - STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, - STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, - STATUS_SUBDOC_NUM_E2BIG = 0xc7, - STATUS_SUBDOC_DELTA_E2BIG = 0xc8, - STATUS_SUBDOC_PATH_EEXISTS = 0xc9, - STATUS_SUBDOC_VALUE_E2DEEP = 0xca, - STATUS_SUBDOC_INVALID_COMBO = 0xcb, - STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, - STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, - STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, - STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, - STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, - STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, - STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, - STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, - STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, - STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, - STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, - STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, - STATUS_XATTR_EINVAL = 0xe0 + bool VersionRequest(); + + int pipelined_count() const { return _pipelined_count; } + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + void Swap(CouchbaseRequest* other); + void MergeFrom(const CouchbaseRequest& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; }; - const char* couchbase_binary_command_to_string(uint8_t cmd); - void MergeFrom(const CouchbaseResponse& from) override; - void Clear() override; - bool IsInitialized() const PB_527_OVERRIDE; - - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - static const char* status_str(Status); - - // Helper method to format error messages with status codes - static string format_error_message(uint16_t status_code, - const string& operation, - const string& error_msg = ""); - - // Add methods to handle response parsing - void Swap(CouchbaseResponse* other); - bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool PopGet(string* value, uint32_t* flags, uint64_t* cas_value); - const string& LastError() const { return _err; } - bool PopUpsert(uint64_t* cas_value); - bool PopAdd(uint64_t* cas_value); - bool PopReplace(uint64_t* cas_value); - bool PopAppend(uint64_t* cas_value); - bool PopPrepend(uint64_t* cas_value); - bool PopSelectBucket(uint64_t* cas_value, std::string bucket_name); - - // Collection-related response methods - bool PopCollectionId(uint8_t* collection_id); - - bool PopManifest(std::string* manifest_json); - - bool PopDelete(); - bool PopFlush(); - bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); - bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); - bool PopTouch(); - bool PopVersion(string* version); + + class CouchbaseResponse : public NonreflectableMessage { + private: + string _err; + static brpc::CouchbaseMetadataTracking *metadata_tracking; + butil::IOBuf _buf; + mutable int _cached_size_; + bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); + bool PopStore(uint8_t command, uint64_t* cas_value); + + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + + public: + CouchbaseResponse() : NonreflectableMessage() { + SharedCtor(); + } + ~CouchbaseResponse() { SharedDtor(); } + CouchbaseResponse(const CouchbaseResponse& from) : NonreflectableMessage(from) { + metadata_tracking = &common_metadata_tracking; + SharedCtor(); + MergeFrom(from); + } + inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { + CopyFrom(from); + return *this; + } + enum Status { + STATUS_SUCCESS = 0x00, + STATUS_KEY_ENOENT = 0x01, + STATUS_KEY_EEXISTS = 0x02, + STATUS_E2BIG = 0x03, + STATUS_EINVAL = 0x04, + STATUS_NOT_STORED = 0x05, + STATUS_DELTA_BADVAL = 0x06, + STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, + STATUS_AUTH_ERROR = 0x20, + STATUS_AUTH_CONTINUE = 0x21, + STATUS_ERANGE = 0x22, + STATUS_ROLLBACK = 0x23, + STATUS_EACCESS = 0x24, + STATUS_NOT_INITIALIZED = 0x25, + STATUS_UNKNOWN_COMMAND = 0x81, + STATUS_ENOMEM = 0x82, + STATUS_NOT_SUPPORTED = 0x83, + STATUS_EINTERNAL = 0x84, + STATUS_EBUSY = 0x85, + STATUS_ETMPFAIL = 0x86, + STATUS_UNKNOWN_COLLECTION = 0x88, + STATUS_NO_COLLECTIONS_MANIFEST = 0x89, + STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, + STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, + STATUS_UNKNOWN_SCOPE = 0x8c, + STATUS_DCP_STREAM_ID_INVALID = 0x8d, + STATUS_DURABILITY_INVALID_LEVEL = 0xa0, + STATUS_DURABILITY_IMPOSSIBLE = 0xa1, + STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, + STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, + STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, + STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, + STATUS_SUBDOC_PATH_MISMATCH = 0xc1, + STATUS_SUBDOC_PATH_EINVAL = 0xc2, + STATUS_SUBDOC_PATH_E2BIG = 0xc3, + STATUS_SUBDOC_DOC_E2DEEP = 0xc4, + STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, + STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, + STATUS_SUBDOC_NUM_E2BIG = 0xc7, + STATUS_SUBDOC_DELTA_E2BIG = 0xc8, + STATUS_SUBDOC_PATH_EEXISTS = 0xc9, + STATUS_SUBDOC_VALUE_E2DEEP = 0xca, + STATUS_SUBDOC_INVALID_COMBO = 0xcb, + STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, + STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, + STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, + STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, + STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, + STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, + STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, + STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, + STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, + STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, + STATUS_XATTR_EINVAL = 0xe0 + }; + const char* couchbase_binary_command_to_string(uint8_t cmd); + void MergeFrom(const CouchbaseResponse& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + static const char* status_str(Status); + + // Helper method to format error messages with status codes + static string format_error_message(uint16_t status_code, + const string& operation, + const string& error_msg = ""); + + // Add methods to handle response parsing + void Swap(CouchbaseResponse* other); + bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); + bool PopGet(string* value, uint32_t* flags, uint64_t* cas_value); + const string& LastError() const { return _err; } + bool PopUpsert(uint64_t* cas_value); + bool PopAdd(uint64_t* cas_value); + bool PopReplace(uint64_t* cas_value); + bool PopAppend(uint64_t* cas_value); + bool PopPrepend(uint64_t* cas_value); + bool PopSelectBucket(uint64_t* cas_value, std::string bucket_name); + + // Collection-related response methods + bool PopCollectionId(uint8_t* collection_id); + + bool PopManifest(std::string* manifest_json); + + bool PopDelete(); + bool PopFlush(); + bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); + bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); + bool PopTouch(); + bool PopVersion(string* version); + }; + + public: + struct Result { + bool success; + string error_message; + string value; + }; + Result Get(const string& key, string collection_name = "_default"); + Result Upsert(const string& key, const string& value, string collection_name = "_default"); + Result Add(const string& key, const string& value, string collection_name = "_default"); + Result Delete(const string& key, string collection_name = "_default"); + Result Authenticate(const string& username, const string& password, const string& server_address, bool enable_ssl, string path_to_cert); + Result SelectBucket(const string& bucket_name); + CouchbaseOperations() {} + ~CouchbaseOperations() {} }; + } // namespace brpc \ No newline at end of file diff --git a/src/brpc/policy/couchbase_protocol.cpp b/src/brpc/policy/couchbase_protocol.cpp index f559df7cb5..20f7ff93bf 100644 --- a/src/brpc/policy/couchbase_protocol.cpp +++ b/src/brpc/policy/couchbase_protocol.cpp @@ -183,11 +183,11 @@ void ProcessCouchbaseResponse(InputMessageBase* msg_base) { if (cntl->response() == NULL) { cntl->SetFailed(ERESPONSE, "response is NULL!"); } else if (cntl->response()->GetDescriptor() != - CouchbaseResponse::descriptor()) { + CouchbaseOperations::CouchbaseResponse::descriptor()) { cntl->SetFailed(ERESPONSE, "Must be CouchbaseResponse"); } else { // We work around ParseFrom of pb which is just a placeholder. - ((CouchbaseResponse*)cntl->response())->raw_buffer() = msg->meta.movable(); + ((CouchbaseOperations::CouchbaseResponse*)cntl->response())->raw_buffer() = msg->meta.movable(); if (msg->pi.count != accessor.pipelined_count()) { cntl->SetFailed(ERESPONSE, "pipelined_count=%d of response does " @@ -206,10 +206,10 @@ void SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, if (request == NULL) { return cntl->SetFailed(EREQUEST, "request is NULL"); } - if (request->GetDescriptor() != CouchbaseRequest::descriptor()) { + if (request->GetDescriptor() != CouchbaseOperations::CouchbaseRequest::descriptor()) { return cntl->SetFailed(EREQUEST, "Must be CouchbaseRequest"); } - const CouchbaseRequest* mr = (const CouchbaseRequest*)request; + const CouchbaseOperations::CouchbaseRequest* mr = (const CouchbaseOperations::CouchbaseRequest*)request; // We work around SerializeTo of pb which is just a placeholder. *buf = mr->raw_buffer(); ControllerPrivateAccessor(cntl).set_pipelined_count(mr->pipelined_count()); From 80713beb0882ee46990fdc11916fcf80d655230e Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 2 Oct 2025 20:45:32 +0530 Subject: [PATCH 18/49] Added pipeline/batching support --- Makefile | 2 +- docs/en/couchbase_example.md | 615 +++++++++++++--- example/couchbase_c++/couchbase_client.cpp | 115 ++- src/brpc/couchbase.cpp | 815 +++++++++++++++------ src/brpc/couchbase.h | 565 +++++++------- 5 files changed, 1554 insertions(+), 558 deletions(-) diff --git a/Makefile b/Makefile index 16a10ae2ea..a2309c8d82 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ include config.mk # 2. Removed -Werror: Not block compilation for non-vital warnings, especially when the # code is tested on newer systems. If the code is used in production, config `config_brpc.sh -werror'. CPPFLAGS+=-DBTHREAD_USE_FAST_PTHREAD_MUTEX -D_GNU_SOURCE -DUSE_SYMBOLIZE -DNO_TCMALLOC -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DNDEBUG -DBRPC_REVISION=\"$(shell ./tools/get_brpc_revision.sh .)\" -CXXFLAGS+=$(CPPFLAGS) -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -Wno-deprecated-declarations -Wno-unused-but-set-variable +CXXFLAGS+=$(CPPFLAGS) -std=c++17 -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -Wno-deprecated-declarations -Wno-unused-but-set-variable CFLAGS=$(CPPFLAGS) -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-unused-parameter -fno-omit-frame-pointer -Wno-deprecated-declarations -Wno-unused-but-set-variable DEBUG_CXXFLAGS = $(filter-out -DNDEBUG,$(CXXFLAGS)) -DUNIT_TEST -DBVAR_NOT_LINK_DEFAULT_VARIABLES DEBUG_CFLAGS = $(filter-out -DNDEBUG,$(CFLAGS)) -DUNIT_TEST diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 5f4cd83a86..4f2b2992a0 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -1,41 +1,52 @@ ## Couchbase bRPC Binary Protocol Integration -This document explains the implementation of Couchbase Binary Protocol support added to this branch of bRPC, the available request/response operations, collection support, and how to run the provided example client against either a local Couchbase Server cluster or a Couchbase Capella (cloud) deployment. +This document explains the implementation of Couchbase Binary Protocol support added to this branch of bRPC, the available high-level operations, collection support, SSL authentication, and how to run the provided example client against either a local Couchbase Server cluster or a Couchbase Capella (cloud) deployment. However, the couchbase binary protocol implementation in bRPC requires us to do fine-grained optimizations which has been already done in the couchbase-cxx-client SDK, so we also added the support of couchbase using couchbase-cxx-SDK in bRPC and is available at [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc). --- ### 1. Overview -The integration adds a new protocol handler (`PROTOCOL_COUCHBASE`) that allows using bRPC's asynchronous / pipelined request machinery to talk directly to Couchbase Server using its Couchbase Binary Protocol. +The integration provides high-level APIs for communicating with Couchbase Server using its Binary Protocol, using the high-level `CouchbaseOperations` class which provides a simplified interface. + +> Each thread must create and use its own `CouchbaseOperations` instance. Sharing instances will cause race conditions and unpredictable behavior. The core pieces are: -* `src/brpc/policy/couchbase_protocol.[h|cpp]` – framing + parse loop for binary responses, and request serialization pass‑through. -* `src/brpc/couchbase.[h|cpp]` – high level request/response builders (`CouchbaseRequest`), parsers (`CouchbaseResponse`) and error-handlers. -* `example/couchbase_c++/couchbase_client.cpp` – an end‑to‑end example performing authentication, bucket selection, CRUD operations, pipelining, and collection‑scoped operations. +* `src/brpc/policy/couchbase_protocol.[h|cpp]` – framing + parse loop for binary responses, and request serialization. +* `src/brpc/couchbase.[h|cpp]` – high-level `CouchbaseOperations` class with simple methods, plus low-level request/response builders (`CouchbaseRequest`), parsers (`CouchbaseResponse`) and error-handlers. +* `example/couchbase_c++/couchbase_client.cpp` – an end‑to‑end example using the high-level API for authentication, bucket selection, CRUD operations, and collection‑scoped operations. +* `example/couchbase_c++/multithreaded_couchbase_client.cpp` – a multithreaded example showing how each thread creates its own `CouchbaseOperations` instance. Design goals: +* **SSL Support**: Built-in SSL/TLS support for secure connections to Couchbase Capella. +* **Per-instance Authentication**: Each `CouchbaseOperations` object maintains its own authenticated session. +* **Collection Support**: Native support for collection-scoped operations. * Keep wire structs identical to the binary protocol (24‑byte header, network order numeric fields). -* Allow batching multiple operations in one TCP write using bRPC's pipelining. -* Provide ergonomic helpers (Add, Get, Upsert, Delete, Increment/Decrement, SelectBucket, GetCollectionId, etc.). -* Future extensions. +* Future extensions for advanced features. --- ### 2. Features | Category | Supported Operations | Notes | |----------|----------------------|-------| -| Authentication | SASL `PLAIN` (`CB_BINARY_SASL_AUTH`) | Sent automatically when you enqueue an `Authenticate` request before others. | -| Bucket selection | `SelectBucket` (`CB_SELECT_BUCKET`) | Required before document operations (unless default bucket context). | -| Basic KV | Add / Set(Upsert) / Delete / Get | Flags + Exptime handled. CAS values returned. | -| Pipelining | Yes | Multiple independent binary requests in one buffer, responses drained in order. | -| Collections | `GetCollectionId`, collection‑scoped CRUD (key + collection id) | Parsing currently truncates to 8 bits in public API (upgrade path below). | -| Error Handling | Status → string mapping + formatted error message | Unsupported codes produce generic fallback. | +| **High-Level API** | `CouchbaseOperations` class | **Recommended**: Simple methods returning `Result` structs | +| **SSL/TLS Support** | Built-in SSL encryption | **Required** for Couchbase Capella, optional for local clusters | +| Authentication | SASL `PLAIN` with SSL | Each `CouchbaseOperations` instance requires authentication | +| Bucket selection | `SelectBucket()` method | Required before document operations | +| Basic KV | `Add()`, `Upsert()`, `Delete()`, `Get()` | Clean API with `Result` struct error handling | +| **Pipeline Operations** | `BeginPipeline()`, `PipelineRequest()`, `ExecutePipeline()` | **NEW**: Batch multiple operations in single network call for improved performance | +| Collections | Collection-scoped CRUD operations | Pass collection name as optional parameter (defaults to "_default") | +| Error Handling | `Result.success` + `Result.error_message` | Human-readable error messages with status codes | + +**Key Differences from Low-Level API:** +- **Simplified**: No need to manage channels, controllers, or response parsing +- **Thread-Safe Per Instance**: Each `CouchbaseOperations` instance can be used independently ⚠️ **BUT NEVER SHARE BETWEEN THREADS** +- **Error Handling**: Simple boolean success with descriptive error messages +- **SSL Built-in**: Automatic SSL handling for secure connections -Missing / Future candidates: Sub‑Document operations, Durability requirements, DCP, Collections Manifest retrieval, Extended Attributes (XATTR), Hello feature negotiation, TLS bootstrap. --- ### 3. Binary Protocol Mapping -Couchbase binary protcol header +Couchbase binary protcol header, for original documentation [click here](https://github.com/couchbase/kv_engine/blob/master/docs/BinaryProtocol.md). The following header format has been used to connect with the couchbase servers. ``` Byte/ 0 | 1 | 2 | 3 | / | | | | @@ -80,134 +91,560 @@ Overall packet structure:- ``` --- -### 4. Request Building (`CouchbaseRequest`) +### 4. High-Level API (`CouchbaseOperations`) -High‑level helpers append one or more wire messages onto an internal `IOBuf`. After you schedule all operations, bRPC sends the accumulated buffer over the channel. +**Recommended Approach**: Use the `CouchbaseOperations` class for simple, thread-safe operations. -Examples: +#### Basic Usage: ```cpp -CouchbaseRequest req; -req.Authenticate(user, pass); // SASL PLAIN -req.SelectBucket("travel-sample"); -req.Add("doc::1", json_body, flags, exptime, /*cas*/0); -req.Get("doc::1"); // Pipeline GET after ADD +#include -channel.CallMethod(nullptr, &cntl, &req, &resp, nullptr); +brpc::CouchbaseOperations couchbase_ops; + +// 1. Authenticate (REQUIRED for each instance) +brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( + username, password, server_address, enable_ssl, cert_path); +if (!auth_result.success) { + LOG(ERROR) << "Auth failed: " << auth_result.error_message; + return -1; +} + +// 2. Select bucket (REQUIRED) +brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.SelectBucket("my_bucket"); +if (!bucket_result.success) { + LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; + return -1; +} + +// 3. Perform operations +brpc::CouchbaseOperations::Result add_result = couchbase_ops.Add("user::123", json_value); +if (add_result.success) { + std::cout << "Document added successfully!" << std::endl; +} else { + std::cout << "Add failed: " << add_result.error_message << std::endl; +} ``` -Pipelining count (`pipelined_count()`) is tracked so the parser knows when the full response group is collected. +#### SSL Authentication (Essential for Couchbase Capella): +```cpp +// For Couchbase Capella (cloud) - SSL is REQUIRED +brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( + username, + password, + "cluster.cloud.couchbase.com:11207", // SSL port + true, // enable_ssl = true + "path/to/certificate.pem" // certificate path +); +``` -Collection ID retrieval: +#### Collection Operations: ```cpp -CouchbaseRequest coll_req; -coll_req.GetCollectionId("_default", "my_collection"); +// Default collection +auto result = couchbase_ops.Get("doc::1"); + +// Specific collection +auto result = couchbase_ops.Get("doc::1", "my_collection"); +auto add_result = couchbase_ops.Add("doc::2", value, "my_collection"); ``` -This builds a key of the form `scope.collection` with opcode `0xbb`. -Collection‑scoped CRUD reuse existing helpers with the final `coll_id` byte parameter. +#### Pipeline Operations (Performance Optimization): +The pipeline API allows batching multiple operations into a single network call, significantly improving performance for bulk operations: ---- -### 5. Response Parsing (`CouchbaseResponse`) +```cpp +// Begin a new pipeline +if (!couchbase_ops.BeginPipeline()) { + LOG(ERROR) << "Failed to begin pipeline"; + return -1; +} -Each `Pop*` method consumes the front of the internal response buffer, validating: -1. Header present. -2. Opcode matches expected operation. -3. Status == success (otherwise `_err` filled with formatted message). -4. Body length sufficient. +// Add multiple operations to the pipeline (not executed yet) +bool success = true; +success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "key1", "value1"); +success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, "key2", "value2"); +success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "key1"); +success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, "key3"); -Common patterns: +if (!success) { + couchbase_ops.ClearPipeline(); // Clean up on error + return -1; +} + +// Execute all operations in a single network call +std::vector results = couchbase_ops.ExecutePipeline(); + +// Process results in the same order as requests +for (size_t i = 0; i < results.size(); ++i) { + if (results[i].success) { + std::cout << "Operation " << i << " succeeded" << std::endl; + if (!results[i].value.empty()) { + std::cout << "Value: " << results[i].value << std::endl; + } + } else { + std::cout << "Operation " << i << " failed: " << results[i].error_message << std::endl; + } +} +``` + +**Pipeline with Collections**: ```cpp -uint64_t cas; -if (resp.PopAdd(&cas)) { /* success */ } else { LOG(ERROR) << resp.LastError(); } +// Pipeline operations can also use collections +couchbase_ops.BeginPipeline(); +couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "doc1", "value1", "my_collection"); +couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "doc1", "", "my_collection"); +auto results = couchbase_ops.ExecutePipeline(); +``` -std::string val; uint32_t flags; uint64_t get_cas; -if (resp.PopGet(&val, &flags, &get_cas)) { /* use val */ } +**Pipeline Management Methods**: +- `BeginPipeline()` - Start a new pipeline session +- `PipelineRequest(op_type, key, value, collection)` - Add operation to pipeline +- `ExecutePipeline()` - Execute all operations and return results +- `ClearPipeline()` - Clear pipeline without executing (cleanup) +- `IsPipelineActive()` - Check if pipeline is active +- `GetPipelineSize()` - Get number of queued operations + +**Performance Benefits**: +- **Reduced Network Overhead**: Multiple operations in single network round-trip +- **Better Throughput**: Especially beneficial for bulk operations +- **Preserved Order**: Results returned in same order as requests +- **Error Isolation**: Individual operation failures don't affect others + +#### Error Handling Pattern: +```cpp +brpc::CouchbaseOperations::Result result = couchbase_ops.SomeOperation(...); +if (!result.success) { + // Handle error + LOG(ERROR) << "Operation failed: " << result.error_message; +} else { + // Use result.value if applicable (for Get operations) + std::cout << "Retrieved value: " << result.value << std::endl; +} ``` -Collection ID parsing (current state): -* Reads 8‑byte Manifest UID + 4‑byte Collection ID extras. -* Truncates Collection ID to `uint8_t` for API compatibility. +--- +### 5. Request/Response Class (`CouchbaseRequest`/`CouchbaseResponse`) + +These classses are private to the `CouchbaseOpeartions` and is not exposed to the user. These classes are responsible for building the request that needs to be sent and received over the channel. A basic overview of how the request/response classes works internally has been shown below. -The raw response buffer can be inspected via `raw_buffer()` for debug / future operations. +#### Request Building: +```cpp +CouchbaseRequest req; +req.Authenticate(user, pass); // SASL PLAIN +req.SelectBucket("travel-sample"); +req.Add("doc::1", json_body, flags, exptime, /*cas*/0); +req.Get("doc::1"); // Pipeline GET after ADD + +channel.CallMethod(nullptr, &cntl, &req, &resp, nullptr); +``` + +#### Response Parsing: +Each `Pop*` method consumes the front of the internal response buffer, validating: +1. Header present. +2. Opcode matches expected operation. +3. Status == success (otherwise `_err` filled with formatted message). +4. Body length sufficient. --- ### 6. Example Client Walkthrough -`example/couchbase_c++/couchbase_client.cpp` flow: -1. Build channel with `PROTOCOL_COUCHBASE`. -2. Prompt for username/password (SASL PLAIN auth). -3. Select bucket. -4. Perform Add, duplicate Add (expected failure), Get. -5. Add several documents (pipelined) and read responses sequentially. -6. Mixed pipelined GET operations (existing + missing keys) to showcase error handling. -7. Upsert existing and new documents; verify with subsequent Get. -8. Delete existing and missing keys. -9. Retrieve Collection ID for a target collection, then perform collection‑scoped Add/Get/Upsert/Get/Delete if ID retrieved. - -Removed instrumentation: The example originally timed operations; those statements were stripped per request to keep output concise. +#### Single-Threaded Example (`couchbase_client.cpp`) +Uses the **high-level `CouchbaseOperations` API**: + +1. **Create `CouchbaseOperations` instance** - can create more than one per thread. +2. **Prompt for credentials** - username/password for authentication. +3. **SSL Authentication** - with support for Couchbase Capella certificate-based SSL. +4. **Select bucket** - required before any document operations. +5. **Basic CRUD operations**: + - Add document (should succeed) + - Try adding same key again (should fail with "key exists") + - Get document (retrieve the added document) +6. **Multiple document operations** - Add several documents with different keys. +7. **Upsert operations**: + - Upsert existing document (should update) + - Upsert new document (should create) + - Verify with Get operations +8. **Delete operations**: + - Delete non-existent key (should fail gracefully) + - Delete existing key (should succeed) +9. **Collection-scoped operations** - Add/Get/Upsert/Delete in specific collections. +10. **Pipeline operations demo**: + - Begin pipeline and add multiple operations + - Execute batch operations in single network call + - Process results in order + - Collection-scoped pipeline operations + - Error handling and cleanup + +#### Multithreaded Example (`multithreaded_couchbase_client.cpp`) +Demonstrates: +- **16 bthreads** (4 threads per bucket across 4 buckets) +- **Each thread creates its own `CouchbaseOperations` instance** +- **Independent authentication** per thread +- **Concurrent operations** across multiple buckets and collections + +Key difference from single-threaded: Each thread must authenticate independently. --- -### 7. Building and Running the Example +### 7. Building and Running the Examples -Build (Make): +#### Build both examples: ```bash cd example/couchbase_c++/ make +``` + +#### Run Single-Threaded Example: +```bash ./couchbase_client ``` -You will be prompted for: +#### Run Multithreaded Example: +```bash +./multithreaded_couchbase_client --operations_per_thread=20 --sleep_ms=100 ``` -Enter Couchbase username: Administrator + +#### Interactive Prompts: +Both examples will prompt for: +``` +Enter Couchbase username: your_username Enter Couchbase password: ******** -Enter Couchbase bucket name: travel-sample +Enter Couchbase bucket name: your_bucket ``` -Ensure buckets/collections you test exist (or create them via UI/CLI) before collection‑scoped CRUD. +For multithreaded example, additional prompts: +``` +Enter 4 bucket names: +Bucket 1: bucket1 +Bucket 2: bucket2 +Bucket 3: bucket3 +Bucket 4: bucket4 +Number of collections (0 for none): 2 +Collection 1: collection1 +Collection 2: collection2 +``` + +#### SSL Configuration: +- **Local Couchbase**: SSL is optional, set `enable_ssl = false` +- **Couchbase Capella**: SSL is **required**, download the certificate from Capella console +- Update the server address in the code or use command line flags: + ```bash + ./couchbase_client --server="your-cluster.cloud.couchbase.com:11207" + ``` + +Ensure buckets/collections exist before testing collection‑scoped operations. --- ### 8. Setting Up Couchbase #### A. Local Install (Non‑Docker) -Download from: https://www.couchbase.com/downloads/ (Community or Enterprise). Install and repeat the same initialization steps through the Web Console at `http://localhost:8091`. -- Open http://localhost:8091 in a browser and follow setup wizard: +Download from: https://www.couchbase.com/downloads/ (Community or Enterprise) and Install. + +Setup steps: +- Open http://localhost:8091 in a browser and follow setup wizard - Set admin credentials (Administrator / password) - Accept terms, choose services (Data, Query, Index at minimum) - Initialize cluster - Create a bucket (e.g. travel-sample or custom) -Create a collection (7.0+): +Create collections (7.0+): +- Navigate: Buckets → Your Bucket → Scopes & Collections +- Add a Scope (optional) or use `_default` +- Add a Collection (e.g. `testing_collection`) -- In the Web Console navigate: Buckets → Your Bucket → Scopes & Collections. -- Add a Scope (optional) or use `_default`. -- Add a Collection (e.g. `testing_collection`). +**SSL Configuration (Optional for Local)**: +```cpp +// Local without SSL +auto result = couchbase_ops.Authenticate(username, password, "localhost:11210", false, ""); +``` -#### B. Couchbase Capella (Cloud) +#### B. Couchbase Capella (Cloud) - **SSL Required** 1. Sign up / log in: https://cloud.couchbase.com/ -2. Create a Free Trial or a Hosted Cluster. -3. Create a bucket (or load a sample dataset). -4. Create a database access credential (API key or user with appropriate RBAC roles – Data Reader/Writer, Bucket Admin as needed). -5. Get the connection string (choose the internal or public endpoint). -6. Update `--server` flag (or code) to point to the KV endpoint host:port. +2. Create a Free Trial or Hosted Cluster +3. Create a bucket (or load sample dataset) +4. **Create database access credentials** with appropriate RBAC roles: + - Data Reader/Writer (minimum) + - Bucket Admin (for bucket operations) +5. **Download SSL Certificate**: + - Go to Cluster → Connect → Download Certificate + - Save as `couchbase-cloud-cert.pem` in your project directory +6. **Get connection endpoint**: + - Use the **KV endpoint** (port 11207 for SSL) + - Format: `your-cluster-id.cloud.couchbase.com:11207` + +**Capella SSL Authentication Example**: +```cpp +// Couchbase Capella - SSL is MANDATORY +auto result = couchbase_ops.Authenticate( + "your_username", + "your_password", + "your-cluster.cloud.couchbase.com:11207", // SSL port + true, // enable_ssl = true + "couchbase-cloud-cert.pem" // certificate file +); +``` + +**Important Notes for Capella**: +- **SSL is mandatory** - connections without SSL will fail +- Use port **11207** (SSL) instead of 11210 (non-SSL) +- Certificate verification is required for security +- Ensure firewall allows outbound connections on port 11207 + +--- +### 9. Pipeline Operations (Performance Optimization) + +Pipeline operations allow you to batch multiple Couchbase operations into a single network call, significantly improving performance for bulk operations. + +#### How Pipeline Operations Work + +1. **Begin Pipeline**: Start a new pipeline session +2. **Add Operations**: Queue multiple operations without executing them +3. **Execute Pipeline**: Send all operations in a single network call +4. **Process Results**: Handle results in the same order as requests + +#### Pipeline API Methods + +| Method | Description | Usage | +|--------|-------------|-------| +| `BeginPipeline()` | Start a new pipeline session | Must call before adding operations | +| `PipelineRequest(op_type, key, value, collection)` | Add operation to pipeline | Supports all CRUD operations | +| `ExecutePipeline()` | Execute all queued operations | Returns `vector` in request order | +| `ClearPipeline()` | Clear pipeline without executing | Use for cleanup on errors | +| `IsPipelineActive()` | Check if pipeline is active | Returns `bool` | +| `GetPipelineSize()` | Get number of queued operations | Returns `size_t` | + +#### Basic Pipeline Example + +```cpp +#include + +brpc::CouchbaseOperations couchbase_ops; +// ... authenticate and select bucket ... + +// 1. Begin pipeline +if (!couchbase_ops.BeginPipeline()) { + LOG(ERROR) << "Failed to begin pipeline"; + return -1; +} + +// 2. Add operations to pipeline (not executed yet) +bool success = true; +success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "user1", "{\"name\":\"John\"}"); +success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "user2", "{\"name\":\"Jane\"}"); +success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "user1"); +success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, "user3", "{\"name\":\"Bob\"}"); + +if (!success) { + couchbase_ops.ClearPipeline(); + return -1; +} + +// 3. Execute all operations in single network call +std::vector results = couchbase_ops.ExecutePipeline(); + +// 4. Process results (same order as requests) +for (size_t i = 0; i < results.size(); ++i) { + const auto& result = results[i]; + if (result.success) { + std::cout << "Operation " << i << " succeeded"; + if (!result.value.empty()) { + std::cout << " - Value: " << result.value; + } + std::cout << std::endl; + } else { + std::cout << "Operation " << i << " failed: " << result.error_message << std::endl; + } +} +``` + +#### Collection-Scoped Pipeline Operations + +```cpp +// Pipeline with collection operations +couchbase_ops.BeginPipeline(); + +// Add operations to specific collection +couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "doc1", "value1", "my_collection"); +couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "doc1", "", "my_collection"); +couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, "doc2", "", "my_collection"); + +auto results = couchbase_ops.ExecutePipeline(); +// Process results... +``` + +#### Pipeline Error Handling + +Pipeline operations provide granular error handling - each operation can succeed or fail independently: + +```cpp +couchbase_ops.BeginPipeline(); + +// Some operations may succeed, others may fail +couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "existing_key", "value"); // May fail if key exists +couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "nonexistent_key"); // May fail if key doesn't exist +couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, "new_key", "value"); // Should succeed + +auto results = couchbase_ops.ExecutePipeline(); + +// Check each result individually +for (size_t i = 0; i < results.size(); ++i) { + if (!results[i].success) { + LOG(WARNING) << "Operation " << i << " failed: " << results[i].error_message; + // Handle individual failures as needed + } +} +``` + +#### Performance Benefits + +- **Reduced Network Latency**: Multiple operations in single round-trip +- **Better Throughput**: Especially beneficial for bulk operations + +#### Pipeline Best Practices + +1. **Batch Related Operations**: Group logically related operations together +2. **Handle Partial Failures**: Individual operations can fail while others succeed +3. **Clear on Errors**: Use `ClearPipeline()` if pipeline setup fails +5. **Mixed Operations**: Combine different operation types (GET, ADD, UPSERT, DELETE) as needed + +#### When to Use Pipelines + +**Ideal Use Cases**: +- Bulk data loading/migration +- Batch processing workflows +- Multi-document transactions (where order matters) +- Performance-critical applications with multiple operations + +**Not Recommended For**: +- Single operations (use regular methods) +- Operations requiring immediate results for decision making +- Very large batches that might timeout --- -### 9. Error Handling Patterns +### 10. Error Handling Patterns + +#### High-Level API (Recommended) +The `CouchbaseOperations` class uses a simple `Result` struct: -Each `Pop*` method returns `false` on: -* Mismatched opcode -* Incomplete buffer -* Non‑zero status (error) – `_err` stores a human readable string like: `STATUS_KEY_EEXISTS: Add operation failed: Key already exists` +```cpp +struct Result { + bool success; // true if operation succeeded + string error_message; // human-readable error description + string value; // returned value (for Get operations) +}; +``` -Recommended usage: +**Recommended Pattern**: ```cpp -uint64_t cas; -if (!resp.PopAdd(&cas)) { - LOG(ERROR) << resp.LastError(); +auto result = couchbase_ops.Add("key", "value"); +if (!result.success) { + LOG(ERROR) << "Add failed: " << result.error_message; + // Handle error appropriately +} else { + std::cout << "Add succeeded!" << std::endl; +} + +// For Get operations, check both success and value +auto get_result = couchbase_ops.Get("key"); +if (get_result.success) { + std::cout << "Retrieved: " << get_result.value << std::endl; +} else { + LOG(ERROR) << "Get failed: " << get_result.error_message; } ``` -For pipelined batches, call the matching `Pop*` in the same sequence you enqueued the operations. +--- +### 11. Best Practices + +#### Thread Safety +> ⚠️ **: THREAD SAFETY REQUIREMENTS** +> - **Each thread MUST create its own `CouchbaseOperations` instance** +> - **Each instance MUST authenticate independently** +> - **NEVER share `CouchbaseOperations` objects between threads** +> - **Sharing instances will cause race conditions, data corruption, and crashes** + +#### SSL Security +- **Always use SSL for Couchbase Capella** (cloud deployments) +- **Verify certificates** - don't disable certificate validation in production +- **Use port 11207** for SSL connections +- **Store certificates securely** and update them when they expire + +#### Performance +- **Reuse `CouchbaseOperations` instances** - they maintain persistent connections +- **Use pipeline operations for bulk operations** +- **Pipeline operations preserve order** - results correspond to request order + +#### Code Example Template +```cpp +#include + +int main() { + brpc::CouchbaseOperations couchbase_ops; + + // Authenticate (adjust SSL settings as needed) + auto auth_result = couchbase_ops.Authenticate( + username, password, server_address, enable_ssl, cert_path); + if (!auth_result.success) { + LOG(ERROR) << "Authentication failed: " << auth_result.error_message; + return -1; + } + + // Select bucket + auto bucket_result = couchbase_ops.SelectBucket(bucket_name); + if (!bucket_result.success) { + LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; + return -1; + } + + // Perform operations with error handling + auto result = couchbase_ops.Add("key", "value", "collection_name"); + if (result.success) { + std::cout << "Success!" << std::endl; + } else { + LOG(ERROR) << "Operation failed: " << result.error_message; + } + + return 0; +} +``` --- -### 10. Summary -This implementation provides a foundation for high‑performance Couchbase KV and collection operations through bRPC's pipelined framework. Extending to sub‑doc, durability, TLS, and richer collection metadata are natural next steps. Contributions and issue reports are welcome. \ No newline at end of file +### 12. Summary and References +This implementation provides both high-level and low-level APIs for Couchbase KV and collection operations. Couchbase (the company) contributed to this implementation, but it is not officially supported; it is "[Community Supported](https://docs.couchbase.com/server/current/third-party/integrations.html#support-model)". + +- **High-level API**: Recommended for most applications - simple, with built-in SSL support +- **SSL Support**: Essential for Couchbase Capella and secure local deployments +- **Thread Safety**: Each thread should create its own authenticated `CouchbaseOperations` instance +- **Collection Support**: Native support for collection-scoped operations + + +--- + +## ⚠️ **CRITICAL THREAD SAFETY WARNING** ⚠️ + +> **🚨 NEVER SHARE `CouchbaseOperations` INSTANCES BETWEEN THREADS! 🚨** +> +> **Each thread MUST create its own `CouchbaseOperations` instance.** +> +>**Each thread can have multiple `CouchbaseOperations` instances.** +> +> **For thread safe design please use couchbase-cxx-SDK version of bRPC, [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc).** +> +> **✅ CORRECT:** +> ```cpp +> // Each thread creates its own instance +> void worker_thread() { +> brpc::CouchbaseOperations ops; // ✅ Thread-local instance +> ops.Authenticate(...); +> ops.Get("key"); // Safe +> } +> ``` +> +> **❌ WRONG - WILL CAUSE CRASHES:** +> ```cpp +> brpc::CouchbaseOperations global_ops; // ❌ Shared instance +> void worker_thread() { +> global_ops.Get("key"); // ❌ RACE CONDITION - WILL CRASH! +> } +> ``` +> +> **Why?** `CouchbaseOperations` contains mutable state (pipeline queues, buffers, connection state) that is NOT thread-safe. Sharing instances will cause data corruption, pipeline interference, and application crashes. + +Contributions and issue reports are welcome! \ No newline at end of file diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index 76fe059544..ef5e7d8b54 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -27,7 +27,7 @@ #define RED "\033[31m" #define RESET "\033[0m" -DEFINE_string(server, "cb.WXYZ.cloud.couchbase.com:11207", "IP Address of server"); +DEFINE_string(server, "cb.dqklxewecglohzwb.cloud.couchbase.com:11207", "IP Address of server"); int main() { // Create CouchbaseOperations instance for high-level operations @@ -256,5 +256,118 @@ int main() { } else { std::cout << RED << "Collection DELETE failed: " << coll_del_result.error_message << RESET << std::endl; } + + // ------------------------------------------------------------------ + // Pipeline Operations Demo + // ------------------------------------------------------------------ + std::cout << GREEN << "\n=== Pipeline Operations Demo ===" << RESET << std::endl; + + // Begin a new pipeline + if (!couchbase_ops.BeginPipeline()) { + std::cout << RED << "Failed to begin pipeline" << RESET << std::endl; + return -1; + } + + std::cout << "Pipeline started. Adding multiple operations..." << std::endl; + + // Add multiple operations to the pipeline + std::string pipeline_key1 = "pipeline::doc1"; + std::string pipeline_key2 = "pipeline::doc2"; + std::string pipeline_key3 = "pipeline::doc3"; + std::string pipeline_value1 = R"({"operation": "pipeline_add", "id": 1})"; + std::string pipeline_value2 = R"({"operation": "pipeline_upsert", "id": 2})"; + std::string pipeline_value3 = R"({"operation": "pipeline_add", "id": 3})"; + + // Pipeline operations - all prepared but not yet executed + bool pipeline_success = true; + pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, pipeline_key1, pipeline_value1); + pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, pipeline_key2, pipeline_value2); + pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, pipeline_key3, pipeline_value3); + pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, pipeline_key1); + pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, pipeline_key2); + + if (!pipeline_success) { + std::cout << RED << "Failed to add operations to pipeline" << RESET << std::endl; + couchbase_ops.ClearPipeline(); + return -1; + } + + std::cout << "Added " << couchbase_ops.GetPipelineSize() << " operations to pipeline" << std::endl; + + // Execute all operations in a single network call + std::cout << "Executing pipeline operations..." << std::endl; + std::vector pipeline_results = couchbase_ops.ExecutePipeline(); + + // Process results in order + std::cout << GREEN << "Pipeline execution completed. Results:" << RESET << std::endl; + for (size_t i = 0; i < pipeline_results.size(); ++i) { + const auto& result = pipeline_results[i]; + if (result.success) { + if (!result.value.empty()) { + std::cout << GREEN << " Operation " << (i+1) << " SUCCESS - Value: " << result.value << RESET << std::endl; + } else { + std::cout << GREEN << " Operation " << (i+1) << " SUCCESS" << RESET << std::endl; + } + } else { + std::cout << RED << " Operation " << (i+1) << " FAILED: " << result.error_message << RESET << std::endl; + } + } + + // Demonstrate pipeline with collection operations + std::cout << GREEN << "\n=== Pipeline with Collection Operations ===" << RESET << std::endl; + + if (!couchbase_ops.BeginPipeline()) { + std::cout << RED << "Failed to begin collection pipeline" << RESET << std::endl; + return -1; + } + + std::string coll_pipeline_key1 = "coll_pipeline::doc1"; + std::string coll_pipeline_key2 = "coll_pipeline::doc2"; + std::string coll_pipeline_value1 = R"({"collection_operation": "pipeline_add", "id": 1})"; + std::string coll_pipeline_value2 = R"({"collection_operation": "pipeline_upsert", "id": 2})"; + + // Add collection-scoped operations to pipeline + bool coll_pipeline_success = true; + coll_pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, coll_pipeline_key1, coll_pipeline_value1, collection_name); + coll_pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, coll_pipeline_key2, coll_pipeline_value2, collection_name); + coll_pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, coll_pipeline_key1, "", collection_name); + coll_pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, coll_pipeline_key1, "", collection_name); + + if (!coll_pipeline_success) { + std::cout << RED << "Failed to add collection operations to pipeline" << RESET << std::endl; + couchbase_ops.ClearPipeline(); + return -1; + } + + // Execute collection pipeline + std::vector coll_pipeline_results = couchbase_ops.ExecutePipeline(); + + std::cout << GREEN << "Collection pipeline execution completed. Results:" << RESET << std::endl; + for (size_t i = 0; i < coll_pipeline_results.size(); ++i) { + const auto& result = coll_pipeline_results[i]; + if (result.success) { + if (!result.value.empty()) { + std::cout << GREEN << " Collection Operation " << (i+1) << " SUCCESS - Value: " << result.value << RESET << std::endl; + } else { + std::cout << GREEN << " Collection Operation " << (i+1) << " SUCCESS" << RESET << std::endl; + } + } else { + std::cout << RED << " Collection Operation " << (i+1) << " FAILED: " << result.error_message << RESET << std::endl; + } + } + + // Clean up remaining pipeline documents + std::cout << GREEN << "\n=== Cleanup Pipeline Demo ===" << RESET << std::endl; + if (couchbase_ops.BeginPipeline()) { + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key1); + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key2); + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key3); + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, coll_pipeline_key2, "", collection_name); + + std::vector cleanup_results = couchbase_ops.ExecutePipeline(); + std::cout << "Cleanup completed (" << cleanup_results.size() << " operations)" << std::endl; + } + + std::cout << GREEN << "\n=== All operations completed successfully! ===" << RESET << std::endl; return 0; } diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index bd5274540c..4501d39da9 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -42,71 +42,18 @@ namespace { CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseRequest::metadata_tracking = &common_metadata_tracking; CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseResponse::metadata_tracking = &common_metadata_tracking; -bool brpc::CouchbaseMetadataTracking::set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string &server) { - std::unique_lock write_lock(rw_thread_to_channel_info_mutex); - auto it = thread_to_channel_info.find(thread_id); - if (it != thread_to_channel_info.end()) { - LOG(WARNING) << "Channel already exists for thread_id: " << thread_id; - return false; - } - ChannelInfo ch_info; - ch_info.channel = channel; - ch_info.server = server; - thread_to_channel_info[thread_id] = ch_info; - return true; -} - -bool brpc::CouchbaseMetadataTracking::set_current_bucket_for_thread(uint64_t thread_id, const string& bucket){ - std::unique_lock write_lock(rw_thread_to_channel_info_mutex); - auto it = thread_to_channel_info.find(thread_id); - if(it == thread_to_channel_info.end()){ - LOG(ERROR) << "No channel exists for thread_id: " << thread_id << ", establish a connection(with authentication) first."; - return false; - } - it->second.selected_bucket = bucket; - return true; -} - -bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest(uint64_t thread_id, CouchbaseMetadataTracking::CollectionManifest manifest){ - string server, selected_bucket; - - // First, get the server and bucket info with proper locking - { - std::shared_lock read_lock(rw_thread_to_channel_info_mutex); - auto it = thread_to_channel_info.find(thread_id); - if(it == thread_to_channel_info.end()){ - LOG(ERROR) << "No channel exists for thread_id: " << thread_id << ", establish a connection(with authentication) first."; - return false; - } - server = it->second.server; - selected_bucket = it->second.selected_bucket; - if(selected_bucket.empty()){ - LOG(ERROR) << "No bucket selected for thread_id: " << thread_id << ", select a bucket first."; - return false; - } - } +bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest(string server, string bucket, CouchbaseMetadataTracking::CollectionManifest manifest){ // Then update the collection manifest with proper locking { std::unique_lock write_lock(rw_bucket_to_collection_manifest_mutex); - bucket_to_collection_manifest[server][selected_bucket] = manifest; + bucket_to_collection_manifest[server][bucket] = manifest; } return true; } -bool brpc::CouchbaseMetadataTracking::get_channel_info_for_thread(uint64_t thread_id, ChannelInfo *channel_info){ - std::shared_lock read_lock(rw_thread_to_channel_info_mutex); - auto it = thread_to_channel_info.find(thread_id); - if(it == thread_to_channel_info.end()){ - LOG(WARNING) << "No channel info exists for thread_id: " << thread_id; - return false; - } - *channel_info = it->second; - return true; -} - -bool brpc::CouchbaseMetadataTracking::get_bucket_to_collection_manifest(uint64_t thread_id, string server, string bucket, CouchbaseMetadataTracking::CollectionManifest *manifest){ +bool brpc::CouchbaseMetadataTracking::get_bucket_to_collection_manifest(string server, string bucket, CouchbaseMetadataTracking::CollectionManifest *manifest){ std::shared_lock read_lock(rw_bucket_to_collection_manifest_mutex); auto it1 = bucket_to_collection_manifest.find(server); if(it1 == bucket_to_collection_manifest.end()){ @@ -468,9 +415,7 @@ bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { } bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest(const butil::StringPiece& username, - const butil::StringPiece& password, - brpc::Channel *channel, - const string server) { + const butil::StringPiece& password) { if (username.empty() || password.empty()) { LOG(ERROR) << "Empty username or password"; return false; @@ -510,15 +455,6 @@ bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest(const butil::Str return false; } ++_pipelined_count; - - // Store the bRPC channel and the server address for future use related to current thread - uint64_t bthread_id = bthread_self(); - if(!metadata_tracking->set_channel_info_for_thread(bthread_id, channel, server)){ - LOG(ERROR) << "Channel for this thread already exists, only one channel per thread is allowed."; - _buf.pop_back(auth_str.size()); - --_pipelined_count; - return false; - } return true; } @@ -833,55 +769,56 @@ bool CouchbaseOperations::CouchbaseRequest::GetOrDelete(uint8_t command, return true; } -bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, brpc::CouchbaseMetadataTracking *metadata_tracking){ +bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, brpc::CouchbaseMetadataTracking *metadata_tracking, + brpc::Channel* channel, const string& server, const string& selected_bucket){ if(collection_name.empty()){ LOG(ERROR) << "Empty collection name"; return false; } - CouchbaseMetadataTracking::ChannelInfo channel_info; - if(!metadata_tracking->get_channel_info_for_thread(bthread_self(), &channel_info)){ - LOG(ERROR) << "No channel found for this thread, make sure to call Authenticate() first"; + + if(channel == nullptr){ + LOG(ERROR) << "No channel found, make sure to call Authenticate() first"; return false; } - if(channel_info.server.empty()){ - LOG(ERROR) << "Server is empty for this thread, make sure to call Authenticate() first"; + if(server.empty()){ + LOG(ERROR) << "Server is empty, make sure to call Authenticate() first"; return false; } - if(channel_info.selected_bucket.empty()){ - LOG(ERROR) << "No bucket selected for this thread, make sure to call SelectBucket() first"; + if(selected_bucket.empty()){ + LOG(ERROR) << "No bucket selected, make sure to call SelectBucket() first"; return false; } + brpc::CouchbaseMetadataTracking::CollectionManifest manifest; - if(!metadata_tracking->get_bucket_to_collection_manifest(bthread_self(), channel_info.server, channel_info.selected_bucket, &manifest)){ - LOG(INFO) << "No cached collection manifest found for bucket " << channel_info.selected_bucket << " on server " << channel_info.server << ", fetching from server"; + if(!metadata_tracking->get_bucket_to_collection_manifest(server, selected_bucket, &manifest)){ + LOG(INFO) << "No cached collection manifest found for bucket " << selected_bucket << " on server " << server << ", fetching from server"; // No cached manifest found, fetch from server CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; CouchbaseOperations::CouchbaseResponse temp_get_manifest_response; brpc::Controller temp_cntl; - brpc::Channel *channel = channel_info.channel; temp_get_manifest_request.GetCollectionManifest(); channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, &temp_get_manifest_response, NULL); if (temp_cntl.Failed()) { - LOG(ERROR) << "Failed to get collection manifest for bucket " << channel_info.selected_bucket << " on server " << channel_info.server + LOG(ERROR) << "Failed to get collection manifest for bucket " << selected_bucket << " on server " << server << ": " << temp_cntl.ErrorText(); return false; } string manifest_json; if (!temp_get_manifest_response.PopManifest(&manifest_json)) { - LOG(ERROR) << "Failed to parse response for collection Manifest in bucket " << channel_info.selected_bucket << " on server " << channel_info.server + LOG(ERROR) << "Failed to parse response for collection Manifest in bucket " << selected_bucket << " on server " << server << ": " << temp_get_manifest_response.LastError(); return false; } else{ // convert JSON to manifest structure if(!metadata_tracking->json_to_collection_manifest(manifest_json, &manifest)){ - LOG(ERROR) << "Failed to parse collection manifest JSON for bucket " << channel_info.selected_bucket << " on server " << channel_info.server; + LOG(ERROR) << "Failed to parse collection manifest JSON for bucket " << selected_bucket << " on server " << server; return false; } // Cache the collection manifest - if(!metadata_tracking->set_bucket_to_collection_manifest(bthread_self(), manifest)){ - LOG(ERROR) << "Failed to cache collection ID for collection " << collection_name << " in bucket " << channel_info.selected_bucket << " on server " << channel_info.server; + if(!metadata_tracking->set_bucket_to_collection_manifest(server, selected_bucket, manifest)){ + LOG(ERROR) << "Failed to cache collection ID for collection " << collection_name << " in bucket " << selected_bucket << " on server " << server; return false; } return true; @@ -895,20 +832,22 @@ bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, } } -bool CouchbaseOperations::CouchbaseRequest::GetRequest(const butil::StringPiece& key, string collection_name) { +bool CouchbaseOperations::CouchbaseRequest::GetRequest(const butil::StringPiece& key, string collection_name, + brpc::Channel* channel, const string& server, const string& bucket) { uint8_t coll_id = 0; // default collection ID if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ return false; } } return GetOrDelete(policy::CB_BINARY_GET, key, coll_id); } -bool CouchbaseOperations::CouchbaseRequest::DeleteRequest(const butil::StringPiece& key, string collection_name) { +bool CouchbaseOperations::CouchbaseRequest::DeleteRequest(const butil::StringPiece& key, string collection_name, + brpc::Channel* channel, const string& server, const string& bucket) { uint8_t coll_id = 0; // default collection ID if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ return false; } } @@ -932,25 +871,26 @@ BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); // 0| Expiration | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseOperations::CouchbaseRequest::FlushRequest(uint32_t timeout) { - const uint8_t FLUSH_EXTRAS = (timeout == 0 ? 0 : 4); - FlushHeaderWithExtras header_with_extras = { - {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_FLUSH, 0, FLUSH_EXTRAS, - policy::CB_BINARY_RAW_BYTES, 0, butil::HostToNet32(FLUSH_EXTRAS), 0, 0}, - butil::HostToNet32(timeout)}; - if (FLUSH_EXTRAS == 0) { - if (_buf.append(&header_with_extras.header, - sizeof(policy::CouchbaseRequestHeader))) { - return false; - } - } else { - if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { - return false; - } - } - ++_pipelined_count; - return true; -} +// Warning: Not tested +// bool CouchbaseOperations::CouchbaseRequest::FlushRequest(uint32_t timeout) { +// const uint8_t FLUSH_EXTRAS = (timeout == 0 ? 0 : 4); +// FlushHeaderWithExtras header_with_extras = { +// {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_FLUSH, 0, FLUSH_EXTRAS, +// policy::CB_BINARY_RAW_BYTES, 0, butil::HostToNet32(FLUSH_EXTRAS), 0, 0}, +// butil::HostToNet32(timeout)}; +// if (FLUSH_EXTRAS == 0) { +// if (_buf.append(&header_with_extras.header, +// sizeof(policy::CouchbaseRequestHeader))) { +// return false; +// } +// } else { +// if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { +// return false; +// } +// } +// ++_pipelined_count; +// return true; +// } // (if found): // MUST have extras. @@ -1052,9 +992,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopGet(std::string* value, uint32_t bool CouchbaseOperations::CouchbaseResponse::PopDelete() { return PopStore(policy::CB_BINARY_DELETE, NULL); } -bool CouchbaseOperations::CouchbaseResponse::PopFlush() { - return PopStore(policy::CB_BINARY_FLUSH, NULL); -} +// Warning: Not tested +// bool CouchbaseOperations::CouchbaseResponse::PopFlush() { +// return PopStore(policy::CB_BINARY_FLUSH, NULL); +// } struct StoreHeaderWithExtras { policy::CouchbaseRequestHeader header; @@ -1253,10 +1194,11 @@ const char* CouchbaseOperations::CouchbaseResponse::couchbase_binary_command_to_ bool CouchbaseOperations::CouchbaseRequest::UpsertRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name) { + string collection_name, + brpc::Channel* channel, const string& server, const string& bucket) { uint8_t coll_id = 0; // default collection ID if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ return false; } } @@ -1370,42 +1312,46 @@ bool CouchbaseOperations::CouchbaseRequest::GetCollectionManifest() { bool CouchbaseOperations::CouchbaseRequest::AddRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name) { - uint8_t coll_id = 0; // default collection ID - // if(collection_name != "_default"){ - // if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ - // return false; - // } - // } - return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, - coll_id); -} - -bool CouchbaseOperations::CouchbaseRequest::ReplaceRequest(const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value, - string collection_name) { + string collection_name, + brpc::Channel* channel, const string& server, const string& bucket) { uint8_t coll_id = 0; // default collection ID if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ return false; - } + } } - return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, cas_value, + return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, coll_id); } +// Warning: Not tested +// bool CouchbaseOperations::CouchbaseRequest::ReplaceRequest(const butil::StringPiece& key, +// const butil::StringPiece& value, uint32_t flags, +// uint32_t exptime, uint64_t cas_value, +// string collection_name, +// brpc::Channel* channel, const string& server, const string& bucket) { +// uint8_t coll_id = 0; // default collection ID +// if(collection_name != "_default"){ +// if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ +// return false; +// } +// } +// return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, cas_value, +// coll_id); +// } + bool CouchbaseOperations::CouchbaseRequest::AppendRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name) { + string collection_name, + brpc::Channel* channel, const string& server, const string& bucket) { if (value.empty()) { LOG(ERROR) << "value to append must be non-empty"; return false; } uint8_t coll_id = 0; // default collection ID if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ return false; } } @@ -1416,14 +1362,15 @@ bool CouchbaseOperations::CouchbaseRequest::AppendRequest(const butil::StringPie bool CouchbaseOperations::CouchbaseRequest::PrependRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name) { + string collection_name, + brpc::Channel* channel, const string& server, const string& bucket) { if (value.empty()) { LOG(ERROR) << "value to prepend must be non-empty"; return false; } uint8_t coll_id = 0; // default collection ID if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking)){ + if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ return false; } } @@ -1437,9 +1384,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopUpsert(uint64_t* cas_value) { bool CouchbaseOperations::CouchbaseResponse::PopAdd(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_ADD, cas_value); } -bool CouchbaseOperations::CouchbaseResponse::PopReplace(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_REPLACE, cas_value); -} +// Warning: Not tested +// bool CouchbaseOperations::CouchbaseResponse::PopReplace(uint64_t* cas_value) { +// return PopStore(policy::CB_BINARY_REPLACE, cas_value); +// } bool CouchbaseOperations::CouchbaseResponse::PopAppend(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_APPEND, cas_value); } @@ -1451,11 +1399,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopSelectBucket(uint64_t* cas_value LOG(ERROR) << "Failed to select bucket: " << _err; return false; } - uint64_t bthread_id = bthread_self(); - if(metadata_tracking->set_current_bucket_for_thread(bthread_id, bucket_name) == false){ - LOG(FATAL) << "Failed to set current bucket for thread_id: " << bthread_id << ". This shouldn't happen normally."; - return false; - } + // Note: Bucket tracking is now handled at CouchbaseOperations level, not per-thread return true; } // Collection-related response method @@ -1637,19 +1581,24 @@ bool CouchbaseOperations::CouchbaseRequest::Counter(uint8_t command, const butil return true; } -bool CouchbaseOperations::CouchbaseRequest::IncrementRequest(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime, - string collection_name) { - return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, - exptime); -} +// Warning: Not tested +// bool CouchbaseOperations::CouchbaseRequest::IncrementRequest(const butil::StringPiece& key, uint64_t delta, +// uint64_t initial_value, uint32_t exptime, +// string collection_name, +// brpc::Channel* channel, const string& server, const string& bucket) { +// // Note: Counter method doesn't seem to use collection_name, may need to be updated if collection support is needed +// return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, +// exptime); +// } -bool CouchbaseOperations::CouchbaseRequest::DecrementRequest(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime, - string collection_name) { - return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, - exptime); -} +// bool CouchbaseOperations::CouchbaseRequest::DecrementRequest(const butil::StringPiece& key, uint64_t delta, +// uint64_t initial_value, uint32_t exptime, +// string collection_name, +// brpc::Channel* channel, const string& server, const string& bucket) { +// // Note: Counter method doesn't seem to use collection_name, may need to be updated if collection support is needed +// return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, +// exptime); +// } // MUST NOT have extras. // MUST NOT have key. @@ -1718,12 +1667,13 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, uint64_ return true; } -bool CouchbaseOperations::CouchbaseResponse::PopIncrement(uint64_t* new_value, uint64_t* cas_value) { - return PopCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); -} -bool CouchbaseOperations::CouchbaseResponse::PopDecrement(uint64_t* new_value, uint64_t* cas_value) { - return PopCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); -} +// Warning: Not tested +// bool CouchbaseOperations::CouchbaseResponse::PopIncrement(uint64_t* new_value, uint64_t* cas_value) { +// return PopCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); +// } +// bool CouchbaseOperations::CouchbaseResponse::PopDecrement(uint64_t* new_value, uint64_t* cas_value) { +// return PopCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); +// } // MUST have extras. // MUST have key. @@ -1747,23 +1697,25 @@ const size_t TOUCH_EXTRAS = // 0| Expiration | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseOperations::CouchbaseRequest::TouchRequest(const butil::StringPiece& key, uint32_t exptime, - string collection_name) { - TouchHeaderWithExtras header_with_extras = { - {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_TOUCH, - butil::HostToNet16(key.size()), TOUCH_EXTRAS, - policy::CB_BINARY_RAW_BYTES, 0, - butil::HostToNet32(TOUCH_EXTRAS + key.size()), 0, 0}, - butil::HostToNet32(exptime)}; - if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { - return false; - } - if (_buf.append(key.data(), key.size())) { - return false; - } - ++_pipelined_count; - return true; -} +// Warning: Not tested +// bool CouchbaseOperations::CouchbaseRequest::TouchRequest(const butil::StringPiece& key, uint32_t exptime, +// string collection_name, +// brpc::Channel* channel, const string& server, const string& bucket) { +// TouchHeaderWithExtras header_with_extras = { +// {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_TOUCH, +// butil::HostToNet16(key.size()), TOUCH_EXTRAS, +// policy::CB_BINARY_RAW_BYTES, 0, +// butil::HostToNet32(TOUCH_EXTRAS + key.size()), 0, 0}, +// butil::HostToNet32(exptime)}; +// if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { +// return false; +// } +// if (_buf.append(key.data(), key.size())) { +// return false; +// } +// ++_pipelined_count; +// return true; +// } // MUST NOT have extras. // MUST NOT have key. @@ -1788,9 +1740,10 @@ bool CouchbaseOperations::CouchbaseRequest::VersionRequest() { // MUST NOT have extras. // MUST NOT have key. // MUST have value. -bool CouchbaseOperations::CouchbaseResponse::PopTouch() { - return PopStore(policy::CB_BINARY_TOUCH, NULL); -} +// Warning: Not tested +// bool CouchbaseOperations::CouchbaseResponse::PopTouch() { +// return PopStore(policy::CB_BINARY_TOUCH, NULL); +// } bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { const size_t n = _buf.size(); @@ -1842,12 +1795,9 @@ CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, string c //create CouchbaseRequest and CouchbaseResponse objects and then using the channel which is created for this thread in authenticate() use it to call() CouchbaseRequest request; CouchbaseResponse response; - brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; - common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); - brpc::Channel *channel = ch_info.channel; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.GetRequest(key, collection_name) == false){ + if(request.GetRequest(key, collection_name, channel, server_address, selected_bucket) == false){ LOG(ERROR) << "Failed to create Get request for key: " << key; result.success = false; result.value = ""; @@ -1879,12 +1829,9 @@ CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, string c CouchbaseOperations::Result CouchbaseOperations::Upsert(const string& key, const string& value, string collection_name) { CouchbaseRequest request; CouchbaseResponse response; - brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; - common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); - brpc::Channel *channel = ch_info.channel; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.UpsertRequest(key, value, 0, 0, 0, collection_name) == false){ + if(request.UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ LOG(ERROR) << "Failed to create Upsert request for key: " << key; result.success = false; result.value = ""; @@ -1913,12 +1860,9 @@ CouchbaseOperations::Result CouchbaseOperations::Upsert(const string& key, const CouchbaseOperations::Result CouchbaseOperations::Delete(const string& key, string collection_name) { CouchbaseRequest request; CouchbaseResponse response; - brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; - common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); - brpc::Channel *channel = ch_info.channel; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.DeleteRequest(key, collection_name) == false){ + if(request.DeleteRequest(key, collection_name, channel, server_address, selected_bucket) == false){ LOG(ERROR) << "Failed to create Delete request for key: " << key; result.success = false; result.value = ""; @@ -1947,12 +1891,9 @@ CouchbaseOperations::Result CouchbaseOperations::Delete(const string& key, strin CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, const string& value, string collection_name) { CouchbaseRequest request; CouchbaseResponse response; - brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; - common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); - brpc::Channel *channel = ch_info.channel; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.AddRequest(key, value, 0, 0, 0, collection_name) == false){ + if(request.AddRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ LOG(ERROR) << "Failed to create Add request for key: " << key; result.success = false; result.value = ""; @@ -1994,10 +1935,10 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate(const string& user ssl_options->verify.ca_file_path = path_to_cert; // Path to your downloaded TLS certificate } CouchbaseOperations::Result result; - brpc::Channel* channel = new brpc::Channel(); - if (channel->Init(server_address.c_str(), &options) != 0) { + brpc::Channel* new_channel = new brpc::Channel(); + if (new_channel->Init(server_address.c_str(), &options) != 0) { LOG(ERROR) << "Failed to initialize Couchbase channel to " << server_address; - delete channel; + delete new_channel; result.success = false; result.value = ""; result.error_message = "Failed to initialize Couchbase channel"; @@ -2007,22 +1948,24 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate(const string& user CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; - if(request.AuthenticateRequest(username.c_str(), password.c_str(), channel, server_address) == false){ + if(request.AuthenticateRequest(username.c_str(), password.c_str()) == false){ LOG(ERROR) << "Failed to create Authenticate request for user: " << username; - delete channel; + delete new_channel; result.success = false; return result; } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); + new_channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { LOG(ERROR) << "Failed to access Couchbase: " << cntl.ErrorText(); - delete channel; + delete new_channel; result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); return result; } // Successfully authenticated + channel = new_channel; + this->server_address = server_address; result.success = true; return result; } @@ -2030,9 +1973,6 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate(const string& user CouchbaseOperations::Result CouchbaseOperations::SelectBucket(const string& bucket_name) { CouchbaseRequest request; CouchbaseResponse response; - brpc::CouchbaseMetadataTracking::ChannelInfo ch_info; - common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &ch_info); - brpc::Channel *channel = ch_info.channel; brpc::Controller cntl; CouchbaseOperations::Result result; if(request.SelectBucketRequest(bucket_name.c_str()) == false){ @@ -2056,8 +1996,471 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket(const string& buck return result; } // Successfully selected the bucket + selected_bucket = bucket_name; result.success = true; result.value = ""; return result; } + +// Warning: Not tested +// CouchbaseOperations::Result CouchbaseOperations::Replace(const string& key, const string& value, string collection_name) { +// CouchbaseRequest request; +// CouchbaseResponse response; +// brpc::Controller cntl; +// CouchbaseOperations::Result result; +// if(request.ReplaceRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ +// LOG(ERROR) << "Failed to create Replace request for key: " << key; +// result.success = false; +// result.value = ""; +// return result; +// } +// channel->CallMethod(NULL, &cntl, &request, &response, NULL); +// if (cntl.Failed()) { +// LOG(ERROR) << "Failed to replace key: " << key << " to Couchbase: " << cntl.ErrorText(); +// result.success = false; +// result.value = ""; +// result.error_message = cntl.ErrorText(); +// return result; +// } +// uint64_t cas_value; +// if(response.PopReplace(&cas_value) == false){ +// result.success = false; +// result.value = ""; +// result.error_message = response.LastError(); +// return result; +// } +// // Successfully replaced the value +// result.success = true; +// result.value = ""; +// return result; +// } + +CouchbaseOperations::Result CouchbaseOperations::Append(const string& key, const string& value, string collection_name) { + CouchbaseRequest request; + CouchbaseResponse response; + brpc::Controller cntl; + CouchbaseOperations::Result result; + if(request.AppendRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ + LOG(ERROR) << "Failed to create Append request for key: " << key; + result.success = false; + result.value = ""; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to append to key: " << key << " to Couchbase: " << cntl.ErrorText(); + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + uint64_t cas_value; + if(response.PopAppend(&cas_value) == false){ + result.success = false; + result.value = ""; + result.error_message = response.LastError(); + return result; + } + // Successfully appended the value + result.success = true; + result.value = ""; + return result; +} + +CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, const string& value, string collection_name) { + CouchbaseRequest request; + CouchbaseResponse response; + brpc::Controller cntl; + CouchbaseOperations::Result result; + if(request.PrependRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ + LOG(ERROR) << "Failed to create Prepend request for key: " << key; + result.success = false; + result.value = ""; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to prepend to key: " << key << " to Couchbase: " << cntl.ErrorText(); + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + uint64_t cas_value; + if(response.PopPrepend(&cas_value) == false){ + result.success = false; + result.value = ""; + result.error_message = response.LastError(); + return result; + } + // Successfully prepended the value + result.success = true; + result.value = ""; + return result; +} + +// Warning: Not tested +// CouchbaseOperations::Result CouchbaseOperations::Increment(const string& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name) { +// CouchbaseRequest request; +// CouchbaseResponse response; +// brpc::Controller cntl; +// CouchbaseOperations::Result result; +// if(request.IncrementRequest(key, delta, initial_value, exptime, collection_name, channel, server_address, selected_bucket) == false){ +// LOG(ERROR) << "Failed to create Increment request for key: " << key; +// result.success = false; +// result.value = ""; +// return result; +// } +// channel->CallMethod(NULL, &cntl, &request, &response, NULL); +// if (cntl.Failed()) { +// LOG(ERROR) << "Failed to increment key: " << key << " in Couchbase: " << cntl.ErrorText(); +// result.success = false; +// result.value = ""; +// result.error_message = cntl.ErrorText(); +// return result; +// } +// uint64_t new_value, cas_value; +// if(response.PopIncrement(&new_value, &cas_value) == false){ +// result.success = false; +// result.value = ""; +// result.error_message = response.LastError(); +// return result; +// } +// // Successfully incremented the value +// result.success = true; +// result.value = std::to_string(new_value); +// return result; +// } + +// Warning: Not tested +// CouchbaseOperations::Result CouchbaseOperations::Decrement(const string& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name) { +// CouchbaseRequest request; +// CouchbaseResponse response; +// brpc::Controller cntl; +// CouchbaseOperations::Result result; +// if(request.DecrementRequest(key, delta, initial_value, exptime, collection_name, channel, server_address, selected_bucket) == false){ +// LOG(ERROR) << "Failed to create Decrement request for key: " << key; +// result.success = false; +// result.value = ""; +// return result; +// } +// channel->CallMethod(NULL, &cntl, &request, &response, NULL); +// if (cntl.Failed()) { +// LOG(ERROR) << "Failed to decrement key: " << key << " in Couchbase: " << cntl.ErrorText(); +// result.success = false; +// result.value = ""; +// result.error_message = cntl.ErrorText(); +// return result; +// } +// uint64_t new_value, cas_value; +// if(response.PopDecrement(&new_value, &cas_value) == false){ +// result.success = false; +// result.value = ""; +// result.error_message = response.LastError(); +// return result; +// } +// // Successfully decremented the value +// result.success = true; +// result.value = std::to_string(new_value); +// return result; +// } + +// Warning: Not tested +// CouchbaseOperations::Result CouchbaseOperations::Touch(const string& key, uint32_t exptime, string collection_name) { +// CouchbaseRequest request; +// CouchbaseResponse response; +// brpc::Controller cntl; +// CouchbaseOperations::Result result; +// if(request.TouchRequest(key, exptime, collection_name, channel, server_address, selected_bucket) == false){ +// LOG(ERROR) << "Failed to create Touch request for key: " << key; +// result.success = false; +// result.value = ""; +// return result; +// } +// channel->CallMethod(NULL, &cntl, &request, &response, NULL); +// if (cntl.Failed()) { +// LOG(ERROR) << "Failed to touch key: " << key << " in Couchbase: " << cntl.ErrorText(); +// result.success = false; +// result.value = ""; +// result.error_message = cntl.ErrorText(); +// return result; +// } +// if(response.PopTouch() == false){ +// result.success = false; +// result.value = ""; +// result.error_message = response.LastError(); +// return result; +// } +// // Successfully touched the key +// result.success = true; +// result.value = ""; +// return result; +// } + +// Warning: Not tested +// CouchbaseOperations::Result CouchbaseOperations::Flush(uint32_t timeout) { +// CouchbaseRequest request; +// CouchbaseResponse response; +// brpc::Controller cntl; +// CouchbaseOperations::Result result; +// if(request.FlushRequest(timeout) == false){ +// LOG(ERROR) << "Failed to create Flush request"; +// result.success = false; +// result.value = ""; +// return result; +// } +// channel->CallMethod(NULL, &cntl, &request, &response, NULL); +// if (cntl.Failed()) { +// LOG(ERROR) << "Failed to flush Couchbase: " << cntl.ErrorText(); +// result.success = false; +// result.value = ""; +// result.error_message = cntl.ErrorText(); +// return result; +// } +// if(response.PopFlush() == false){ +// result.success = false; +// result.value = ""; +// result.error_message = response.LastError(); +// return result; +// } +// // Successfully flushed +// result.success = true; +// result.value = ""; +// return result; +// } + +CouchbaseOperations::Result CouchbaseOperations::Version() { + CouchbaseRequest request; + CouchbaseResponse response; + brpc::Controller cntl; + CouchbaseOperations::Result result; + if(request.VersionRequest() == false){ + LOG(ERROR) << "Failed to create Version request"; + result.success = false; + result.value = ""; + return result; + } + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to get version from Couchbase: " << cntl.ErrorText(); + result.success = false; + result.value = ""; + result.error_message = cntl.ErrorText(); + return result; + } + string version; + if(response.PopVersion(&version) == false){ + result.success = false; + result.value = ""; + result.error_message = response.LastError(); + return result; + } + // Successfully got version + result.success = true; + result.value = version; + return result; +} + +bool CouchbaseOperations::BeginPipeline() { + if (pipeline_active) { + LOG(WARNING) << "Pipeline already active. Call ClearPipeline() first."; + return false; + } + + // Clear any previous state + while (!pipeline_operations_queue.empty()) { + pipeline_operations_queue.pop(); + } + pipeline_request.Clear(); + + pipeline_active = true; + return true; +} + +bool CouchbaseOperations::PipelineRequest(pipeline_operation_type op_type, const string& key, const string& value, string collection_name) { + if (!pipeline_active) { + LOG(ERROR) << "Pipeline not active. Call BeginPipeline() first."; + return false; + } + + switch(op_type){ + case GET: + if(pipeline_request.GetRequest(key, collection_name, channel, server_address, selected_bucket)== false){ + return false; + } + pipeline_operations_queue.push(GET); + break; + case UPSERT: + if(pipeline_request.UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket)== false){ + return false; + } + pipeline_operations_queue.push(UPSERT); + break; + case ADD: + if(pipeline_request.AddRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket)== false){ + return false; + } + pipeline_operations_queue.push(ADD); + break; + case APPEND: + if(pipeline_request.AppendRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket)== false){ + return false; + } + pipeline_operations_queue.push(APPEND); + break; + case PREPEND: + if(pipeline_request.PrependRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket)== false){ + return false; + } + pipeline_operations_queue.push(PREPEND); + break; + case DELETE: + if(pipeline_request.DeleteRequest(key, collection_name, channel, server_address, selected_bucket)== false){ + return false; + } + pipeline_operations_queue.push(DELETE); + break; + default: + LOG(ERROR) << "Invalid operation type for pipelining"; + return false; + } + return true; +} +vector CouchbaseOperations::ExecutePipeline() { + vector results; + + if (!pipeline_active || pipeline_operations_queue.empty()) { + LOG(ERROR) << "No pipeline active or no operations queued"; + return results; + } + + brpc::Controller cntl; + channel->CallMethod(NULL, &cntl, &pipeline_request, &pipeline_response, NULL); + + if (cntl.Failed()) { + LOG(ERROR) << "Pipeline execution failed: " << cntl.ErrorText(); + // Create failure results for all operations + size_t op_count = pipeline_operations_queue.size(); + results.reserve(op_count); + + CouchbaseOperations::Result failure_result; + failure_result.success = false; + failure_result.error_message = cntl.ErrorText(); + + for (size_t i = 0; i < op_count; ++i) { + results.push_back(failure_result); + } + + ClearPipeline(); + return results; + } + + // Process each operation in the order they were added + CouchbaseOperations::CouchbaseResponse *response = &pipeline_response; + while(!pipeline_operations_queue.empty()){ + CouchbaseOperations::Result result; + pipeline_operation_type op_type = pipeline_operations_queue.front(); + pipeline_operations_queue.pop(); + switch(op_type){ + case GET: { + string value; + uint32_t flags = 0; + uint64_t cas = 0; + if(response->PopGet(&value, &flags, &cas) == false){ + result.success = false; + result.value = ""; + result.error_message = response->LastError(); + } else { + result.success = true; + result.value = value; + } + results.push_back(result); + break; + } + case UPSERT: { + if(response->PopUpsert(NULL) == false){ + result.success = false; + result.value = ""; + result.error_message = response->LastError(); + } else { + result.success = true; + result.value = ""; + } + results.push_back(result); + break; + } + case ADD: { + if(response->PopAdd(NULL) == false){ + result.success = false; + result.value = ""; + result.error_message = response->LastError(); + } else { + result.success = true; + result.value = ""; + } + results.push_back(result); + break; + } + case APPEND: { + uint64_t cas_value; + if(response->PopAppend(&cas_value) == false){ + result.success = false; + result.value = ""; + result.error_message = response->LastError(); + } else { + result.success = true; + result.value = ""; + } + results.push_back(result); + break; + } + case PREPEND: { + uint64_t cas_value; + if(response->PopPrepend(&cas_value) == false){ + result.success = false; + result.value = ""; + result.error_message = response->LastError(); + } else { + result.success = true; + result.value = ""; + } + results.push_back(result); + break; + } + case DELETE: { + if(response->PopDelete() == false){ + result.success = false; + result.value = ""; + result.error_message = response->LastError(); + } else { + result.success = true; + result.value = ""; + } + results.push_back(result); + break; + } + default: + LOG(ERROR) << "Invalid operation type in pipeline response processing"; + result.success = false; + result.value = ""; + result.error_message = "Invalid operation type"; + results.push_back(result); + break; + } + } + + pipeline_active = false; + pipeline_request.Clear(); + + return results; +} + +bool CouchbaseOperations::ClearPipeline() { + while (!pipeline_operations_queue.empty()) { + pipeline_operations_queue.pop(); + } + pipeline_request.Clear(); + pipeline_active = false; + return true; +} }// namespace brpc \ No newline at end of file diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 1e99fd5ea2..09b82049a3 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -21,6 +21,7 @@ #endif #include +#include #include "brpc/nonreflectable_message.h" #include @@ -45,20 +46,11 @@ namespace policy { class CouchbaseMetadataTracking{ public: - struct ChannelInfo { - brpc::Channel* channel; - string server; - string selected_bucket; - }; - struct CollectionManifest{ string uid; //uid of the manifest, it can be used to track if the manifest is updated unordered_map> scope_to_collectionID_map; //scope -> (collection -> collection_id) }; private: - unordered_map thread_to_channel_info; - shared_mutex rw_thread_to_channel_info_mutex; - unordered_map> bucket_to_collection_manifest; shared_mutex rw_bucket_to_collection_manifest_mutex; @@ -66,266 +58,25 @@ class CouchbaseMetadataTracking{ CouchbaseMetadataTracking() {} ~CouchbaseMetadataTracking() { bucket_to_collection_manifest.clear(); - thread_to_channel_info.clear(); } - bool set_channel_info_for_thread(uint64_t thread_id, brpc::Channel* channel, const string& server); - bool set_current_bucket_for_thread(uint64_t thread_id, const string& bucket); - bool set_bucket_to_collection_manifest(uint64_t thread_id, CollectionManifest manifest); + bool set_bucket_to_collection_manifest(string server, string bucket, CollectionManifest manifest); - bool get_channel_info_for_thread(uint64_t thread_id, ChannelInfo *channel_info); - bool get_bucket_to_collection_manifest(uint64_t thread_id, string server, string bucket, CollectionManifest *manifest); + bool get_bucket_to_collection_manifest(string server, string bucket, CollectionManifest *manifest); bool get_manifest_to_collection_id(CollectionManifest *manifest, string scope, string collection, uint8_t *collection_id); bool json_to_collection_manifest(const string& json, CollectionManifest *manifest); } static common_metadata_tracking; class CouchbaseOperations { -private: - // Friend functions to allow access to private nested classes - friend bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, brpc::CouchbaseMetadataTracking *metadata_tracking); - friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); - friend void policy::SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); - - class CouchbaseRequest : public NonreflectableMessage { - private: - static brpc::CouchbaseMetadataTracking *metadata_tracking; - int _pipelined_count; - butil::IOBuf _buf; - mutable int _cached_size_; - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - bool GetOrDelete(uint8_t command, const butil::StringPiece& key, - uint8_t coll_id = 0); - bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime); - - bool Store(uint8_t command, const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, uint32_t exptime, - uint64_t cas_value, uint8_t coll_id = 0); - uint32_t hash_crc32(const char* key, size_t key_length); - - public: - CouchbaseRequest() : NonreflectableMessage() { - metadata_tracking = &common_metadata_tracking; - SharedCtor(); - } - ~CouchbaseRequest() { SharedDtor(); } - CouchbaseRequest(const CouchbaseRequest& from) : NonreflectableMessage(from) { - SharedCtor(); - MergeFrom(from); - } - - inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { - CopyFrom(from); - return *this; - } - - bool SelectBucketRequest(const butil::StringPiece& bucket_name); - bool AuthenticateRequest(const butil::StringPiece& username, - const butil::StringPiece& password, - brpc::Channel* channel, - const string server); - bool HelloRequest(); - - // Using GetCollectionManifest instead of fetching collection ID directly - // bool GetCollectionId(const butil::StringPiece& scope_name, - // const butil::StringPiece& collection_name); - - bool GetScopeId(const butil::StringPiece& scope_name); - - bool GetCollectionManifest(); - - bool RefreshCollectionManifest(); - - // Collection-aware document operations - bool GetRequest(const butil::StringPiece& key, string collection_name = "_default"); - - bool UpsertRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); - - bool AddRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); - - bool ReplaceRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); - - bool AppendRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); - - bool PrependRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default"); - - bool DeleteRequest(const butil::StringPiece& key, string collection_name = "_default"); - - bool FlushRequest(uint32_t timeout); - - bool IncrementRequest(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); - bool DecrementRequest(const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); - - bool TouchRequest(const butil::StringPiece& key, uint32_t exptime, - string collection_name = "_default"); - - bool VersionRequest(); - - int pipelined_count() const { return _pipelined_count; } - - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - void Swap(CouchbaseRequest* other); - void MergeFrom(const CouchbaseRequest& from) override; - void Clear() override; - bool IsInitialized() const PB_527_OVERRIDE; - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - }; - - class CouchbaseResponse : public NonreflectableMessage { - private: - string _err; - static brpc::CouchbaseMetadataTracking *metadata_tracking; - butil::IOBuf _buf; - mutable int _cached_size_; - bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); - bool PopStore(uint8_t command, uint64_t* cas_value); - - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - public: - CouchbaseResponse() : NonreflectableMessage() { - SharedCtor(); - } - ~CouchbaseResponse() { SharedDtor(); } - CouchbaseResponse(const CouchbaseResponse& from) : NonreflectableMessage(from) { - metadata_tracking = &common_metadata_tracking; - SharedCtor(); - MergeFrom(from); - } - inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { - CopyFrom(from); - return *this; - } - enum Status { - STATUS_SUCCESS = 0x00, - STATUS_KEY_ENOENT = 0x01, - STATUS_KEY_EEXISTS = 0x02, - STATUS_E2BIG = 0x03, - STATUS_EINVAL = 0x04, - STATUS_NOT_STORED = 0x05, - STATUS_DELTA_BADVAL = 0x06, - STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, - STATUS_AUTH_ERROR = 0x20, - STATUS_AUTH_CONTINUE = 0x21, - STATUS_ERANGE = 0x22, - STATUS_ROLLBACK = 0x23, - STATUS_EACCESS = 0x24, - STATUS_NOT_INITIALIZED = 0x25, - STATUS_UNKNOWN_COMMAND = 0x81, - STATUS_ENOMEM = 0x82, - STATUS_NOT_SUPPORTED = 0x83, - STATUS_EINTERNAL = 0x84, - STATUS_EBUSY = 0x85, - STATUS_ETMPFAIL = 0x86, - STATUS_UNKNOWN_COLLECTION = 0x88, - STATUS_NO_COLLECTIONS_MANIFEST = 0x89, - STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, - STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, - STATUS_UNKNOWN_SCOPE = 0x8c, - STATUS_DCP_STREAM_ID_INVALID = 0x8d, - STATUS_DURABILITY_INVALID_LEVEL = 0xa0, - STATUS_DURABILITY_IMPOSSIBLE = 0xa1, - STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, - STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, - STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, - STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, - STATUS_SUBDOC_PATH_MISMATCH = 0xc1, - STATUS_SUBDOC_PATH_EINVAL = 0xc2, - STATUS_SUBDOC_PATH_E2BIG = 0xc3, - STATUS_SUBDOC_DOC_E2DEEP = 0xc4, - STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, - STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, - STATUS_SUBDOC_NUM_E2BIG = 0xc7, - STATUS_SUBDOC_DELTA_E2BIG = 0xc8, - STATUS_SUBDOC_PATH_EEXISTS = 0xc9, - STATUS_SUBDOC_VALUE_E2DEEP = 0xca, - STATUS_SUBDOC_INVALID_COMBO = 0xcb, - STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, - STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, - STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, - STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, - STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, - STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, - STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, - STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, - STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, - STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, - STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, - STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, - STATUS_XATTR_EINVAL = 0xe0 + enum pipeline_operation_type { + GET = 1, + UPSERT = 2, + ADD = 3, + REPLACE = 4, + APPEND = 5, + PREPEND = 6, + DELETE = 7 }; - const char* couchbase_binary_command_to_string(uint8_t cmd); - void MergeFrom(const CouchbaseResponse& from) override; - void Clear() override; - bool IsInitialized() const PB_527_OVERRIDE; - - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - static const char* status_str(Status); - - // Helper method to format error messages with status codes - static string format_error_message(uint16_t status_code, - const string& operation, - const string& error_msg = ""); - - // Add methods to handle response parsing - void Swap(CouchbaseResponse* other); - bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool PopGet(string* value, uint32_t* flags, uint64_t* cas_value); - const string& LastError() const { return _err; } - bool PopUpsert(uint64_t* cas_value); - bool PopAdd(uint64_t* cas_value); - bool PopReplace(uint64_t* cas_value); - bool PopAppend(uint64_t* cas_value); - bool PopPrepend(uint64_t* cas_value); - bool PopSelectBucket(uint64_t* cas_value, std::string bucket_name); - - // Collection-related response methods - bool PopCollectionId(uint8_t* collection_id); - - bool PopManifest(std::string* manifest_json); - - bool PopDelete(); - bool PopFlush(); - bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); - bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); - bool PopTouch(); - bool PopVersion(string* version); - }; - - public: struct Result { bool success; string error_message; @@ -334,11 +85,303 @@ class CouchbaseOperations { Result Get(const string& key, string collection_name = "_default"); Result Upsert(const string& key, const string& value, string collection_name = "_default"); Result Add(const string& key, const string& value, string collection_name = "_default"); + // Warning: Not tested + // Result Replace(const string& key, const string& value, string collection_name = "_default"); + Result Append(const string& key, const string& value, string collection_name = "_default"); + Result Prepend(const string& key, const string& value, string collection_name = "_default"); Result Delete(const string& key, string collection_name = "_default"); + // Warning: Not tested + // Result Increment(const string& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); + // Result Decrement(const string& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); + // Result Touch(const string& key, uint32_t exptime, string collection_name = "_default"); + // Result Flush(uint32_t timeout = 0); + Result Version(); Result Authenticate(const string& username, const string& password, const string& server_address, bool enable_ssl, string path_to_cert); Result SelectBucket(const string& bucket_name); - CouchbaseOperations() {} + + // Pipeline management + bool BeginPipeline(); + bool PipelineRequest(pipeline_operation_type op_type, const string& key, const string& value = "", string collection_name = "_default"); + vector ExecutePipeline(); // Return by value instead of pointer + bool ClearPipeline(); + + // Pipeline status + bool IsPipelineActive() const { return pipeline_active; } + size_t GetPipelineSize() const { return pipeline_operations_queue.size(); } + + CouchbaseOperations() : pipeline_active(false) {} ~CouchbaseOperations() {} + private: + // Friend functions to allow access to private nested classes + friend bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, brpc::CouchbaseMetadataTracking *metadata_tracking, brpc::Channel* channel, const string& server, const string& selected_bucket); + friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); + friend void policy::SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); + + brpc::Channel* channel; + string server_address; + string selected_bucket; + + class CouchbaseRequest : public NonreflectableMessage { + private: + static brpc::CouchbaseMetadataTracking *metadata_tracking; + int _pipelined_count; + butil::IOBuf _buf; + mutable int _cached_size_; + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + bool GetOrDelete(uint8_t command, const butil::StringPiece& key, + uint8_t coll_id = 0); + bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + + bool Store(uint8_t command, const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, uint32_t exptime, + uint64_t cas_value, uint8_t coll_id = 0); + uint32_t hash_crc32(const char* key, size_t key_length); + + public: + CouchbaseRequest() : NonreflectableMessage() { + metadata_tracking = &common_metadata_tracking; + SharedCtor(); + } + ~CouchbaseRequest() { SharedDtor(); } + CouchbaseRequest(const CouchbaseRequest& from) : NonreflectableMessage(from) { + SharedCtor(); + MergeFrom(from); + } + + inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { + CopyFrom(from); + return *this; + } + + bool SelectBucketRequest(const butil::StringPiece& bucket_name); + bool AuthenticateRequest(const butil::StringPiece& username, + const butil::StringPiece& password); + bool HelloRequest(); + + // Using GetCollectionManifest instead of fetching collection ID directly + // bool GetCollectionId(const butil::StringPiece& scope_name, + // const butil::StringPiece& collection_name); + + bool GetScopeId(const butil::StringPiece& scope_name); + + bool GetCollectionManifest(); + + bool RefreshCollectionManifest(); + + // Collection-aware document operations + bool GetRequest(const butil::StringPiece& key, string collection_name = "_default", + brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + bool UpsertRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default", + brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + bool AddRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default", + brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + // Warning: Not tested + // bool ReplaceRequest(const butil::StringPiece& key, const butil::StringPiece& value, + // uint32_t flags, uint32_t exptime, uint64_t cas_value, + // string collection_name = "_default", + // brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + bool AppendRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default", + brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + bool PrependRequest(const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name = "_default", + brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + bool DeleteRequest(const butil::StringPiece& key, string collection_name = "_default", + brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + // Warning: Not tested + // bool FlushRequest(uint32_t timeout); + + // bool IncrementRequest(const butil::StringPiece& key, uint64_t delta, + // uint64_t initial_value, uint32_t exptime, string collection_name = "_default", + // brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + // bool DecrementRequest(const butil::StringPiece& key, uint64_t delta, + // uint64_t initial_value, uint32_t exptime, string collection_name = "_default", + // brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + // bool TouchRequest(const butil::StringPiece& key, uint32_t exptime, + // string collection_name = "_default", + // brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + + bool VersionRequest(); + + int pipelined_count() const { return _pipelined_count; } + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + void Swap(CouchbaseRequest* other); + void MergeFrom(const CouchbaseRequest& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; + }; + + class CouchbaseResponse : public NonreflectableMessage { + private: + string _err; + static brpc::CouchbaseMetadataTracking *metadata_tracking; + butil::IOBuf _buf; + mutable int _cached_size_; + bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); + bool PopStore(uint8_t command, uint64_t* cas_value); + + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + + public: + CouchbaseResponse() : NonreflectableMessage() { + SharedCtor(); + } + ~CouchbaseResponse() { SharedDtor(); } + CouchbaseResponse(const CouchbaseResponse& from) : NonreflectableMessage(from) { + metadata_tracking = &common_metadata_tracking; + SharedCtor(); + MergeFrom(from); + } + inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { + CopyFrom(from); + return *this; + } + + // the status codes are from Couchbase Binary Protocol documentation, + // for original reference of status codes visit + // https://github.com/couchbase/kv_engine/blob/master/include/mcbp/protocol/status.h + enum Status { + STATUS_SUCCESS = 0x00, + STATUS_KEY_ENOENT = 0x01, + STATUS_KEY_EEXISTS = 0x02, + STATUS_E2BIG = 0x03, + STATUS_EINVAL = 0x04, + STATUS_NOT_STORED = 0x05, + STATUS_DELTA_BADVAL = 0x06, + STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, + STATUS_AUTH_ERROR = 0x20, + STATUS_AUTH_CONTINUE = 0x21, + STATUS_ERANGE = 0x22, + STATUS_ROLLBACK = 0x23, + STATUS_EACCESS = 0x24, + STATUS_NOT_INITIALIZED = 0x25, + STATUS_UNKNOWN_COMMAND = 0x81, + STATUS_ENOMEM = 0x82, + STATUS_NOT_SUPPORTED = 0x83, + STATUS_EINTERNAL = 0x84, + STATUS_EBUSY = 0x85, + STATUS_ETMPFAIL = 0x86, + STATUS_UNKNOWN_COLLECTION = 0x88, + STATUS_NO_COLLECTIONS_MANIFEST = 0x89, + STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, + STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, + STATUS_UNKNOWN_SCOPE = 0x8c, + STATUS_DCP_STREAM_ID_INVALID = 0x8d, + STATUS_DURABILITY_INVALID_LEVEL = 0xa0, + STATUS_DURABILITY_IMPOSSIBLE = 0xa1, + STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, + STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, + STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, + STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, + STATUS_SUBDOC_PATH_MISMATCH = 0xc1, + STATUS_SUBDOC_PATH_EINVAL = 0xc2, + STATUS_SUBDOC_PATH_E2BIG = 0xc3, + STATUS_SUBDOC_DOC_E2DEEP = 0xc4, + STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, + STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, + STATUS_SUBDOC_NUM_E2BIG = 0xc7, + STATUS_SUBDOC_DELTA_E2BIG = 0xc8, + STATUS_SUBDOC_PATH_EEXISTS = 0xc9, + STATUS_SUBDOC_VALUE_E2DEEP = 0xca, + STATUS_SUBDOC_INVALID_COMBO = 0xcb, + STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, + STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, + STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, + STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, + STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, + STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, + STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, + STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, + STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, + STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, + STATUS_XATTR_EINVAL = 0xe0 + }; + const char* couchbase_binary_command_to_string(uint8_t cmd); + void MergeFrom(const CouchbaseResponse& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + static const char* status_str(Status); + + // Helper method to format error messages with status codes + static string format_error_message(uint16_t status_code, + const string& operation, + const string& error_msg = ""); + + // Add methods to handle response parsing + void Swap(CouchbaseResponse* other); + bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); + bool PopGet(string* value, uint32_t* flags, uint64_t* cas_value); + const string& LastError() const { return _err; } + bool PopUpsert(uint64_t* cas_value); + bool PopAdd(uint64_t* cas_value); + // Warning: Not tested + // bool PopReplace(uint64_t* cas_value); + bool PopAppend(uint64_t* cas_value); + bool PopPrepend(uint64_t* cas_value); + bool PopSelectBucket(uint64_t* cas_value, std::string bucket_name); + + // Collection-related response methods + bool PopCollectionId(uint8_t* collection_id); + + bool PopManifest(std::string* manifest_json); + + bool PopDelete(); + // Warning: Not tested + // bool PopFlush(); + // bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); + // bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); + // bool PopTouch(); + bool PopVersion(string* version); + }; + + // Pipeline management - per instance + queue pipeline_operations_queue; + CouchbaseRequest pipeline_request; + CouchbaseResponse pipeline_response; + bool pipeline_active; }; } // namespace brpc \ No newline at end of file From e77e3cd3c1e90522b6f22fbb83af327c6bb3fa7a Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 2 Oct 2025 20:56:38 +0530 Subject: [PATCH 19/49] commented unused variables --- docs/en/couchbase_example.md | 2 +- src/brpc/couchbase.cpp | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 4f2b2992a0..f4e6e1d68d 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -625,7 +625,7 @@ This implementation provides both high-level and low-level APIs for Couchbase KV > >**Each thread can have multiple `CouchbaseOperations` instances.** > -> **For thread safe design please use couchbase-cxx-SDK version of bRPC, [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc).** +> **For thread safe design please use couchbase-cxx-SDK version of bRPC, While it does not leverage many of the bRPC features around memory management and IO, it does provide a more complete set of Couchbase features and may be useful to those who have apps using bRPC with either memcached binprot or Couchbase and need some of the additional services and can be accessed at [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc).** > > **✅ CORRECT:** > ```cpp diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 4501d39da9..29ff39b6f0 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -1678,13 +1678,13 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, uint64_ // MUST have extras. // MUST have key. // MUST NOT have value. -struct TouchHeaderWithExtras { - policy::CouchbaseRequestHeader header; - uint32_t exptime; -} __attribute__((packed)); -BAIDU_CASSERT(sizeof(TouchHeaderWithExtras) == 28, must_match); -const size_t TOUCH_EXTRAS = - sizeof(TouchHeaderWithExtras) - sizeof(policy::CouchbaseRequestHeader); +// struct TouchHeaderWithExtras { +// policy::CouchbaseRequestHeader header; +// uint32_t exptime; +// } __attribute__((packed)); +// BAIDU_CASSERT(sizeof(TouchHeaderWithExtras) == 28, must_match); +// const size_t TOUCH_EXTRAS = +// sizeof(TouchHeaderWithExtras) - sizeof(policy::CouchbaseRequestHeader); // MAY have extras. // MUST NOT have key. From 1ce8f289eead45b0020c81d23bbc1a1264c65c2c Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 2 Oct 2025 22:38:58 +0530 Subject: [PATCH 20/49] Updated support for C++17 --- test/Makefile | 2 +- test/brpc_http_rpc_protocol_unittest.cpp | 3 +-- test/brpc_load_balancer_unittest.cpp | 10 ++++++++-- test/find_cstr_unittest.cpp | 5 ++++- test/flat_map_unittest.cpp | 18 ++++++++++++++---- test/object_pool_unittest.cpp | 10 ++++++++-- test/resource_pool_unittest.cpp | 10 ++++++++-- 7 files changed, 44 insertions(+), 14 deletions(-) diff --git a/test/Makefile b/test/Makefile index 30e196bbce..d48d3e3429 100644 --- a/test/Makefile +++ b/test/Makefile @@ -19,7 +19,7 @@ NEED_GPERFTOOLS=1 NEED_GTEST=1 include ../config.mk CPPFLAGS+=-DBTHREAD_USE_FAST_PTHREAD_MUTEX -D_GNU_SOURCE -DUSE_SYMBOLIZE -DNO_TCMALLOC -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DUNIT_TEST -Dprivate=public -Dprotected=public -DBVAR_NOT_LINK_DEFAULT_VARIABLES --include sstream_workaround.h -CXXFLAGS+=$(CPPFLAGS) -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++0x +CXXFLAGS+=$(CPPFLAGS) -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++17 # On Darwin, only gtest library is needed, other libraries have been linked in `brpc.dbg.dylib` ifeq ($(SYSTEM),Darwin) diff --git a/test/brpc_http_rpc_protocol_unittest.cpp b/test/brpc_http_rpc_protocol_unittest.cpp index 5a6839c86f..1f7b8420cc 100644 --- a/test/brpc_http_rpc_protocol_unittest.cpp +++ b/test/brpc_http_rpc_protocol_unittest.cpp @@ -20,7 +20,6 @@ // Date: Sun Jul 13 15:04:18 CST 2014 #include -#include #include #include #include @@ -1128,7 +1127,7 @@ class UploadServiceImpl : public ::test::UploadService { private: void check_header(brpc::Controller* cntl) { const std::string* test_header = cntl->http_request().GetHeader(TEST_PROGRESSIVE_HEADER); - GOOGLE_CHECK_NOTNULL(test_header); + CHECK_NE(test_header, nullptr); CHECK_EQ(*test_header, TEST_PROGRESSIVE_HEADER_VAL); } }; diff --git a/test/brpc_load_balancer_unittest.cpp b/test/brpc_load_balancer_unittest.cpp index cca44a0753..6b3f968ddb 100644 --- a/test/brpc_load_balancer_unittest.cpp +++ b/test/brpc_load_balancer_unittest.cpp @@ -21,6 +21,8 @@ #include #include +#include +#include #include #include "bthread/bthread.h" #include "gperftools_helper.h" @@ -405,7 +407,9 @@ TEST_F(LoadBalancerTest, la_sanity) { ValidateLALB(lalb, cur_count); const size_t before_removal = cur_count; - std::random_shuffle(ids.begin(), ids.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(ids.begin(), ids.end(), g); for (size_t i = 0; i < N / 2; ++i) { const brpc::ServerId id = ids.back(); ids.pop_back(); @@ -538,7 +542,9 @@ TEST_F(LoadBalancerTest, update_while_selection) { } else { removed.assign(ids.begin(), ids.begin() + 255); } - std::random_shuffle(removed.begin(), removed.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(removed.begin(), removed.end(), g); removed.pop_back(); ASSERT_EQ(removed.size(), lb->RemoveServersInBatch(removed)); ASSERT_EQ(removed.size(), lb->AddServersInBatch(removed)); diff --git a/test/find_cstr_unittest.cpp b/test/find_cstr_unittest.cpp index 340ae0b21f..6b424a5e8e 100644 --- a/test/find_cstr_unittest.cpp +++ b/test/find_cstr_unittest.cpp @@ -16,6 +16,7 @@ // under the License. #include +#include #include #include "butil/find_cstr.h" #include "butil/time.h" @@ -75,7 +76,9 @@ TEST_F(FindCstrTest, perf) { j = 0; } } - std::random_shuffle(all_keys.begin(), all_keys.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(all_keys.begin(), all_keys.end(), g); int sum = 0; butil::Timer tm; tm.start(); diff --git a/test/flat_map_unittest.cpp b/test/flat_map_unittest.cpp index a394218ad6..834c7e2cb1 100644 --- a/test/flat_map_unittest.cpp +++ b/test/flat_map_unittest.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include #include @@ -874,7 +876,9 @@ TEST_F(FlatMapTest, perf_cmp_with_map_storing_pointers) { ASSERT_EQ(m1.size(), m2.size()); ASSERT_EQ(m1.size(), m3.size()); - std::random_shuffle(r.begin(), r.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(r.begin(), r.end(), g); sum = 0; tm.start(); @@ -1253,7 +1257,9 @@ void perf_insert_erase(bool random, const T& value) { } if (random) { - random_shuffle(keys.begin(), keys.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(keys.begin(), keys.end(), g); } id_map.clear(); @@ -1318,7 +1324,9 @@ void perf_insert_erase(bool random, const T& value) { << "/" << hash_tm.n_elapsed() / keys.size(); if (random) { - random_shuffle(keys.begin(), keys.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(keys.begin(), keys.end(), g); } id_tm.start(); @@ -1438,7 +1446,9 @@ void perf_seek(const T& value) { hash_map[keys[i]] = value; } - random_shuffle(keys.begin(), keys.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(keys.begin(), keys.end(), g); long sum = 0; id_tm.start(); diff --git a/test/object_pool_unittest.cpp b/test/object_pool_unittest.cpp index cfc891aff7..abfdb84602 100644 --- a/test/object_pool_unittest.cpp +++ b/test/object_pool_unittest.cpp @@ -16,6 +16,8 @@ // under the License. #include +#include +#include #include "butil/time.h" #include "butil/macros.h" @@ -268,7 +270,9 @@ void* get_and_return_int(void*) { } tm1.stop(); - std::random_shuffle(v.begin(), v.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(v.begin(), v.end(), g); tm2.start(); for (size_t i = 0; i < v.size(); ++i) { @@ -308,7 +312,9 @@ void* new_and_delete_int(void*) { } tm1.stop(); - std::random_shuffle(v2.begin(), v2.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(v2.begin(), v2.end(), g); tm2.start(); for (size_t i = 0; i < v2.size(); ++i) { diff --git a/test/resource_pool_unittest.cpp b/test/resource_pool_unittest.cpp index 9a56ff3bb5..3a7e213d43 100644 --- a/test/resource_pool_unittest.cpp +++ b/test/resource_pool_unittest.cpp @@ -16,6 +16,8 @@ // under the License. #include +#include +#include #include "butil/time.h" #include "butil/macros.h" #include "butil/fast_rand.h" @@ -316,7 +318,9 @@ void* get_and_return_int(void*) { } tm1.stop(); - std::random_shuffle(v.begin(), v.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(v.begin(), v.end(), g); tm2.start(); for (size_t i = 0; i < v.size(); ++i) { @@ -355,7 +359,9 @@ void* new_and_delete_int(void*) { } tm1.stop(); - std::random_shuffle(v2.begin(), v2.end()); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(v2.begin(), v2.end(), g); tm2.start(); for (size_t i = 0; i < v2.size(); ++i) { From f03e0f6fa6227ddf2e61da7124a1745958a4b8af Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 2 Oct 2025 23:16:54 +0530 Subject: [PATCH 21/49] fixed some issue. --- src/brpc/couchbase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 29ff39b6f0..058c5e2e17 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -32,7 +32,7 @@ namespace brpc { // Couchbase protocol constants namespace { - constexpr uint32_t APPLE_VBUCKET_COUNT = 64; + [[maybe_unused]] constexpr uint32_t APPLE_VBUCKET_COUNT = 64; constexpr uint32_t DEFAULT_VBUCKET_COUNT = 1024; constexpr int CONNECTION_ID_SIZE = 33; constexpr size_t RANDOM_ID_HEX_SIZE = 67; // 33 bytes * 2 + null terminator From 68b4a516fb3b8376cb481e4963cd8834de3b0bf6 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Fri, 3 Oct 2025 04:06:46 +0530 Subject: [PATCH 22/49] Using Mutex instead of shared lock to support c++11 --- MODULE.bazel.lock | 992 ++++++++++++++++++ Makefile | 2 +- .../couchbase_c++/couchbase-cloud-cert.txt | 20 + src/brpc/couchbase.cpp | 4 +- src/brpc/couchbase.h | 74 +- test/Makefile | 2 +- test/brpc_http_rpc_protocol_unittest.cpp | 3 +- test/brpc_load_balancer_unittest.cpp | 10 +- test/find_cstr_unittest.cpp | 5 +- test/flat_map_unittest.cpp | 18 +- test/object_pool_unittest.cpp | 10 +- test/resource_pool_unittest.cpp | 10 +- 12 files changed, 1101 insertions(+), 49 deletions(-) create mode 100644 MODULE.bazel.lock create mode 100644 example/couchbase_c++/couchbase-cloud-cert.txt diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock new file mode 100644 index 0000000000..a57f5aba60 --- /dev/null +++ b/MODULE.bazel.lock @@ -0,0 +1,992 @@ +{ + "lockFileVersion": 11, + "registryFileHashes": { + "https://baidu.github.io/babylon/registry/bazel_registry.json": "not found", + "https://baidu.github.io/babylon/registry/modules/libevent/2.1.12/MODULE.bazel": "1ba8ed9390cd17472cd0495f0bdfdc4d2de62781fd6ef157b32682f07033bd4d", + "https://baidu.github.io/babylon/registry/modules/libevent/2.1.12/source.json": "97292d6719ae1313ace6f2f049aea985a9bd34570583bba96fe72e3fe722aaa1", + "https://baidu.github.io/babylon/registry/modules/thrift/0.21.0/MODULE.bazel": "8d28c0adba6b3010c8f4e8297039f890286fe3429584559d4332bfbb105a9bb0", + "https://baidu.github.io/babylon/registry/modules/thrift/0.21.0/source.json": "9b0a0c088bad57ec8bdef2e4d1c33cc541d7005dfb6387c03bc64b42a0d6f4cc", + "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", + "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", + "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", + "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/source.json": "14892cc698e02ffedf4967546e6bedb7245015906888d3465fcf27c90a26da10", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", + "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", + "https://bcr.bazel.build/modules/apple_support/1.17.1/source.json": "6b2b8c74d14e8d485528a938e44bdb72a5ba17632b9e14ef6e68a5ee96c8347f", + "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", + "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", + "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", + "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", + "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", + "https://bcr.bazel.build/modules/bazel_features/1.19.0/source.json": "d7bf14517c1b25b9d9c580b0f8795fceeae08a7590f507b76aace528e941375d", + "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", + "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", + "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", + "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", + "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", + "https://bcr.bazel.build/modules/boost.algorithm/1.83.0/MODULE.bazel": "f5afbca8fe26f73016c040ae2525c8cfdfb7c7f8a0b2aa7412e08447a8bf76b4", + "https://bcr.bazel.build/modules/boost.algorithm/1.83.0/source.json": "cbc02be90eedd45da42f55b7e352a3aeac56ae72f62237308b5e6e1733b2f2c5", + "https://bcr.bazel.build/modules/boost.array/1.83.0.bcr.1/MODULE.bazel": "ffe2956a8f5d90839e710cb298f0a342acc5a1d8fdfeee9578cae9657df87e38", + "https://bcr.bazel.build/modules/boost.array/1.83.0.bcr.1/source.json": "6d3925fef402c04bd7ea463203c1892c6ab96351c2264566e4bc9e6543fc777f", + "https://bcr.bazel.build/modules/boost.array/1.83.0/MODULE.bazel": "76bc954c31d8c840d89f347ec3d8f4542fc74229e9ab9d74b7e21342a8becd39", + "https://bcr.bazel.build/modules/boost.assert/1.83.0.bcr.1/MODULE.bazel": "abc07418c37c6e1876f56d959ff1b7cdcac70c8b7236bbacc69ab66686968910", + "https://bcr.bazel.build/modules/boost.assert/1.83.0.bcr.1/source.json": "05d5c7ef77352f32264b9ca625df8ca742b78a94d4b057347a54204d6cd9ded6", + "https://bcr.bazel.build/modules/boost.assert/1.83.0/MODULE.bazel": "7b132fe0760e15733253280bd4b9e0c8867b6a8e27322d3def361cab7a729ac7", + "https://bcr.bazel.build/modules/boost.bind/1.83.0.bcr.1/MODULE.bazel": "c226a7152112ec5329aa831fd7daab296db2e8bf1a427e05c33bb770c885e044", + "https://bcr.bazel.build/modules/boost.bind/1.83.0.bcr.1/source.json": "1ad06dbc6d538e539abaeb58d647d787b63c2b0eb54f61168c8501d420cdd691", + "https://bcr.bazel.build/modules/boost.bind/1.83.0/MODULE.bazel": "11e5db47f917eb4a0238e89bc0eda2b41a979682c679cc21209dbd347c3f4723", + "https://bcr.bazel.build/modules/boost.concept_check/1.83.0.bcr.1/MODULE.bazel": "987ddeeca7c552f7c15557ed9f1e1de26e976d1380002cd41e56ea0ef7e1d25e", + "https://bcr.bazel.build/modules/boost.concept_check/1.83.0.bcr.1/source.json": "0a5cb9c8a8b424c04a717b2cbf483f1ee576611b3370550369671ea4f5852e3c", + "https://bcr.bazel.build/modules/boost.concept_check/1.83.0/MODULE.bazel": "a783fc53e77b42e8baa39a4e8f031d0dc2c87f2da386569bb9d46bc12a652c66", + "https://bcr.bazel.build/modules/boost.config/1.83.0.bcr.1/MODULE.bazel": "a9049b46df8ba7d4a895122cfbda4cf71ad01f441fad62117931cfafecba35d1", + "https://bcr.bazel.build/modules/boost.config/1.83.0.bcr.1/source.json": "4810a18d8d9827bf2822ca8e3e1491cfb882657224bc2def70b334120415ccc2", + "https://bcr.bazel.build/modules/boost.config/1.83.0/MODULE.bazel": "beb2e8899be8dc7e3fc2d5b6d5302e665f42be85046e60a955bcbd7ba8174bf8", + "https://bcr.bazel.build/modules/boost.container_hash/1.83.0.bcr.1/MODULE.bazel": "baf420d50979d18d247681bb4f3e097c054f314c2a3c98c14f1b750871329ad7", + "https://bcr.bazel.build/modules/boost.container_hash/1.83.0.bcr.1/source.json": "f3248fbd4a2cd88333343bd78f317e3f7b085b6a2a31adab26f402083bdbdb26", + "https://bcr.bazel.build/modules/boost.container_hash/1.83.0/MODULE.bazel": "be8328ca4dcbe7832438eedf35607f277dd7b4372d75996f5287f40867d1cd2e", + "https://bcr.bazel.build/modules/boost.conversion/1.83.0.bcr.1/MODULE.bazel": "d05ea5d39c0399ddd0331922e64a6bb8bdfe40b83cb6d4ecee62d7cb68f6c36b", + "https://bcr.bazel.build/modules/boost.conversion/1.83.0.bcr.1/source.json": "7b91cda19dcf46d35b47fd30ea9a6226c880d27f6abd51f8107c5015fec9bd46", + "https://bcr.bazel.build/modules/boost.conversion/1.83.0/MODULE.bazel": "d7c3ae92bb0e806da3174eeba584a66aa02a6677a3768eec62f89c7159b48723", + "https://bcr.bazel.build/modules/boost.core/1.83.0.bcr.1/MODULE.bazel": "6aa5d86d84dbeefeda0f5e3ac596db7426ecb6f6ebdf69383c2963292ee568cf", + "https://bcr.bazel.build/modules/boost.core/1.83.0.bcr.1/source.json": "62f0884fe4d287b72146c3355df6d737b016f312bb3bf9c1bbf41fd84400b046", + "https://bcr.bazel.build/modules/boost.core/1.83.0/MODULE.bazel": "769d1faa7dc89832c1a435110a0b447b2e280b3da740ca1811711418fc2e476e", + "https://bcr.bazel.build/modules/boost.describe/1.83.0.bcr.1/MODULE.bazel": "04f902525e3f53a7eec9e10bb58623d5b249afaa4f245917cc827f63933351ba", + "https://bcr.bazel.build/modules/boost.describe/1.83.0.bcr.1/source.json": "b3201219ad62723a684c98d1f13448346774d5979adf99d88583765150a9095c", + "https://bcr.bazel.build/modules/boost.describe/1.83.0/MODULE.bazel": "63ca2aa1c8f62e8a187767d294529f1a762d0f6a4bc1b6081c3e1af63458bac4", + "https://bcr.bazel.build/modules/boost.detail/1.83.0.bcr.2/MODULE.bazel": "0016cbb9f265ebd4efcbb5f1aff14ccce5754bd98c3d9ff375d3a39f4173f8de", + "https://bcr.bazel.build/modules/boost.detail/1.83.0.bcr.2/source.json": "0044cf130a00ccde75322449d2a525e1ca95af2bd61b49873125943c4d3ff7ad", + "https://bcr.bazel.build/modules/boost.detail/1.83.0/MODULE.bazel": "9bbd10b5c30d0c17908fd0c94e9a26f4899540d1c1a97088911d1195feb98da9", + "https://bcr.bazel.build/modules/boost.dynamic_bitset/1.83.0.bcr.1/MODULE.bazel": "2093a9d09159b97ada67ed370492f61c775ae7df5018c8ba9c69053ed7813dab", + "https://bcr.bazel.build/modules/boost.dynamic_bitset/1.83.0.bcr.1/source.json": "c8470feff0d94ef0c950dfa6b214cc119bbaae95355e27838b9d295c7b222b21", + "https://bcr.bazel.build/modules/boost.exception/1.83.0/MODULE.bazel": "3cc2e06995cb92e329449793e389051035ec3516180aaec0bbfd36efc6c90d7b", + "https://bcr.bazel.build/modules/boost.exception/1.83.0/source.json": "09d071c13a55cd740d2354729eb02cc716a2673c6e6030725690af16f32ad6fa", + "https://bcr.bazel.build/modules/boost.function/1.83.0.bcr.1/MODULE.bazel": "b555cccb955cc4bd8d548f81ec29fdd33284ae04afd9ed92b254abe6357a5ed7", + "https://bcr.bazel.build/modules/boost.function/1.83.0.bcr.1/source.json": "a85e42738a1fd7eb3df45ca31fe213ca23832adb0032a7bcf9cd8dac42445f86", + "https://bcr.bazel.build/modules/boost.function/1.83.0/MODULE.bazel": "fa8cf838f76289180210f17e5f5effb51b91ce0be0a2cc72a2b134e53006bbd6", + "https://bcr.bazel.build/modules/boost.function_types/1.83.0.bcr.1/MODULE.bazel": "266d931a312bafb39053b6a0260e59882c1389e8126f9dc5b3f37a1c2b66a152", + "https://bcr.bazel.build/modules/boost.function_types/1.83.0.bcr.1/source.json": "c421295de00ee97f4d605d8a873ce002b64d9e4226f61db710ceaaf2ff1c581a", + "https://bcr.bazel.build/modules/boost.function_types/1.83.0/MODULE.bazel": "8974442d5a0929ef7df5f63fa5d49221b2274cd19f2be0ca91b99d2538b74d64", + "https://bcr.bazel.build/modules/boost.functional/1.83.0.bcr.1/MODULE.bazel": "8e0f52ea1e5f1d34442347c1ddcb202ce2cc610175226df58770c69d7267b4a8", + "https://bcr.bazel.build/modules/boost.functional/1.83.0.bcr.1/source.json": "6a3c58ca81f5226221ac1565aa3a7c93e5d04b27d2f9bd71e1b113ff19f5a658", + "https://bcr.bazel.build/modules/boost.functional/1.83.0/MODULE.bazel": "f6c5ec6684a05d5c235a9141a417c1c18eee1af1783a30f7f3f19bd3020f128b", + "https://bcr.bazel.build/modules/boost.fusion/1.83.0.bcr.1/MODULE.bazel": "1a651840d5710e5f2b882535f9fd7a18d6932c24559185362a3dfe0ca3702ee4", + "https://bcr.bazel.build/modules/boost.fusion/1.83.0.bcr.1/source.json": "8977203271c1f1db3f329e5cdd4960d26060ef13bdc7e47820b77f61d2369909", + "https://bcr.bazel.build/modules/boost.fusion/1.83.0/MODULE.bazel": "394a7dd5bb57baad66c30db1cf871c2b7d05129637945c04c3b26e11bd0cb4de", + "https://bcr.bazel.build/modules/boost.integer/1.83.0.bcr.1/MODULE.bazel": "29655d7e043ba79065ed8fb8b8b99133470ff240f4606bebb6085d2c856578df", + "https://bcr.bazel.build/modules/boost.integer/1.83.0.bcr.1/source.json": "92d1105b5e005c47b10c09b42dfbc86374a069b62055c7471eb8546545ee534b", + "https://bcr.bazel.build/modules/boost.integer/1.83.0/MODULE.bazel": "2bbbc2e0e01f901ce84ef66859635e3ad0b645eeb3a133e243af8e750f4db54d", + "https://bcr.bazel.build/modules/boost.io/1.83.0.bcr.1/MODULE.bazel": "ce3b42ee33880f7250263a158c51b6c33c38ccbff754b78703c8af3f842e1cb1", + "https://bcr.bazel.build/modules/boost.io/1.83.0.bcr.1/source.json": "94e74cc68d9ba9821aa35c207a943c8140856fbc494b3524d3729f0caa9e97db", + "https://bcr.bazel.build/modules/boost.io/1.83.0/MODULE.bazel": "47457a879a81af920b096b76affd242f898fdf406aec12c62c6772a84aa0e0a0", + "https://bcr.bazel.build/modules/boost.iterator/1.83.0.bcr.1/MODULE.bazel": "6ac7783146113154a83e841bfa4449105718006378edff7fbfc53aeddf00812b", + "https://bcr.bazel.build/modules/boost.iterator/1.83.0.bcr.1/source.json": "16e10bfe39dd3096147160668219e7a9bb8a42bbce5bd6a8a9bfcb9c428f7912", + "https://bcr.bazel.build/modules/boost.iterator/1.83.0/MODULE.bazel": "a526477248e085c73e8059708b61b9238536f923ede8c310e979aec4672f2e5d", + "https://bcr.bazel.build/modules/boost.move/1.83.0.bcr.1/MODULE.bazel": "e901ceb363762eb989a7a53418fe7ce2acf8f542a413109105184a4e22e38096", + "https://bcr.bazel.build/modules/boost.move/1.83.0.bcr.1/source.json": "978a8ff249833ab0764a7c2a83a9bda299960112613f22c1ee3bb3ec97fbddbf", + "https://bcr.bazel.build/modules/boost.move/1.83.0/MODULE.bazel": "9294f124718b42857e0eabde467fd78b2b9f8844529749fc1ffe27bc44d23940", + "https://bcr.bazel.build/modules/boost.mp11/1.83.0.bcr.1/MODULE.bazel": "cf0d62cfca8865a1e1672953bd5b08cf5cb95e3aae819427565606305dd29165", + "https://bcr.bazel.build/modules/boost.mp11/1.83.0.bcr.1/source.json": "554cee46c279651fc3ffca71d947e8b99cb26bd87661debe7946eb21b61031f7", + "https://bcr.bazel.build/modules/boost.mp11/1.83.0/MODULE.bazel": "5b47c08596fd8b0f1cf5e183b22012a8380bacbdf68535de0a469a26bcb32f47", + "https://bcr.bazel.build/modules/boost.mpl/1.83.0.bcr.1/MODULE.bazel": "9cd97b224f9c1d8ffefdf48a7163767ca3968fbdfbc76f057caefbb4054be33e", + "https://bcr.bazel.build/modules/boost.mpl/1.83.0.bcr.1/source.json": "2ad7f4a254613fe2c0722d6127d9b366dbd19dcd8e44ad23f3ff66bf48163019", + "https://bcr.bazel.build/modules/boost.mpl/1.83.0/MODULE.bazel": "3110db5c9c5000140076797d61592f8abffd50e91e04a153ee24bf47d30e38ec", + "https://bcr.bazel.build/modules/boost.numeric_conversion/1.83.0.bcr.1/MODULE.bazel": "1920d71f656a0433947579a86c22e0dd897816eadee3ae20d5c51dab9505deb3", + "https://bcr.bazel.build/modules/boost.numeric_conversion/1.83.0.bcr.1/source.json": "fc35a1dbb8810812f05fa54667f0edeac142f5fc144f9426236694a1d01e875c", + "https://bcr.bazel.build/modules/boost.numeric_conversion/1.83.0/MODULE.bazel": "34fbcfe4eab607de5285b543f63142042f98c0895d765476f251c7bb45acf33d", + "https://bcr.bazel.build/modules/boost.optional/1.83.0.bcr.1/MODULE.bazel": "971e786a19e31dfbd8d311dff3bbffc7c1e16f348e019dfca50620fd9c475092", + "https://bcr.bazel.build/modules/boost.optional/1.83.0.bcr.1/source.json": "486e1173f8b24120078f072f434a79fb3aa8c6b0dafbc19c56b7780ea2336595", + "https://bcr.bazel.build/modules/boost.optional/1.83.0/MODULE.bazel": "d2d9c9afa139aa075212e451570aee269d73d5ccda31ab0ab33a9a4b406c0cf7", + "https://bcr.bazel.build/modules/boost.predef/1.83.0.bcr.1/MODULE.bazel": "0dd34241f892423a9dcd20e1256b6a1c787d60849e9576c724a71607ad8a1955", + "https://bcr.bazel.build/modules/boost.predef/1.83.0.bcr.1/source.json": "deb64fec0911e7a5d38d8517ecdd470b0b3983506662e01f63f8925688c05c01", + "https://bcr.bazel.build/modules/boost.predef/1.83.0/MODULE.bazel": "7bfd5416496b14c32e0684e57e47b8452fc5fe0e4afad294b31c2140b1d73dc8", + "https://bcr.bazel.build/modules/boost.preprocessor/1.83.0.bcr.1/MODULE.bazel": "edb9fb7900ea7002cbefffd97302b071d7cbd8f948b51c7b1a75043bd2985eba", + "https://bcr.bazel.build/modules/boost.preprocessor/1.83.0.bcr.1/source.json": "69dc4f6fc76305c21c4a651c94ccfdc8a76d8fbae1151e7c1d1a4599dffc0f03", + "https://bcr.bazel.build/modules/boost.preprocessor/1.83.0/MODULE.bazel": "5d1096729ebd16d2679c798110c0896720be23959c59afa36547d04815e255c8", + "https://bcr.bazel.build/modules/boost.random/1.83.0.bcr.1/MODULE.bazel": "cabe3ba820c9588a9ca22548b88ad8c4b307be085691b286ff0dae8a46b25fa0", + "https://bcr.bazel.build/modules/boost.random/1.83.0.bcr.1/source.json": "5cc5c8c13e525c5b1c12ea5b31359a16bfaf25b750c0601c0624b9342872e05d", + "https://bcr.bazel.build/modules/boost.range/1.83.0.bcr.1/MODULE.bazel": "136d623462d1d5c7cf79df83b5ce17a8582a92abb116da9d88c5e5594e5a7d92", + "https://bcr.bazel.build/modules/boost.range/1.83.0.bcr.1/source.json": "f99062101034f19d9bf2bef3e07cc99bf192640b72560663227e5375efd1e144", + "https://bcr.bazel.build/modules/boost.range/1.83.0/MODULE.bazel": "ceba0feb376949eecb77944bc37ac434261dede457fc449958164fca5d1430db", + "https://bcr.bazel.build/modules/boost.regex/1.83.0.bcr.1/MODULE.bazel": "5e2c235ea0bdbd8fa942f429ed8aefa1ad3e1471993a17a7b82c1f127dbcb3a2", + "https://bcr.bazel.build/modules/boost.regex/1.83.0.bcr.1/source.json": "1d5fda14ae73d5f12ced3c8731006f28b54bdaf75da35f6cc06c2b57b724d011", + "https://bcr.bazel.build/modules/boost.regex/1.83.0/MODULE.bazel": "75a260515a22080dc48aa02fd6dc1aa6bc42695dd7c6a449f37343cd0ea9b018", + "https://bcr.bazel.build/modules/boost.smart_ptr/1.83.0.bcr.1/MODULE.bazel": "5652293a4b393e5a56ef4113e0f41b6121d22a0d8f7ccd3e27c4042b947683e3", + "https://bcr.bazel.build/modules/boost.smart_ptr/1.83.0.bcr.1/source.json": "1b1b6b549073ef6b1bdbd9898490f73ddb6324774d441cd24d6d1181d4bc75f4", + "https://bcr.bazel.build/modules/boost.smart_ptr/1.83.0/MODULE.bazel": "67537f1bcdf50ab51ad79d9ac4e089018a7ebad1a2d362efcc54f5fc5ba45bd4", + "https://bcr.bazel.build/modules/boost.static_assert/1.83.0.bcr.1/MODULE.bazel": "2b605adc483c6241865f1e862437331bc6f56c0d376769908b70ba18d3da1f07", + "https://bcr.bazel.build/modules/boost.static_assert/1.83.0.bcr.1/source.json": "a0eac8de976fff7efdf498933d7494df30eff471c51c1edfc822007069697ed7", + "https://bcr.bazel.build/modules/boost.static_assert/1.83.0/MODULE.bazel": "680325e3252ae8306555bcf0539d16dcf9ccf9656d8781dfa3449a554d8da016", + "https://bcr.bazel.build/modules/boost.system/1.83.0.bcr.1/MODULE.bazel": "5f905d0fbb1ce99231f3fa278b2e5999aa7395c6393ac42d479ae21824adf03f", + "https://bcr.bazel.build/modules/boost.system/1.83.0.bcr.1/source.json": "0676ab63c01c5ddf731a5cf54667ffc6560e9fb52401a2a9ac6a10c5a9909019", + "https://bcr.bazel.build/modules/boost.throw_exception/1.83.0.bcr.1/MODULE.bazel": "b757c832f5f5f818d87c9eaa993d3eb211554197321c3edf641e2c8821cf19c2", + "https://bcr.bazel.build/modules/boost.throw_exception/1.83.0.bcr.1/source.json": "c752d584840e9183141f9d53f07f2051016c16771a973cdd1487f9585980c2e5", + "https://bcr.bazel.build/modules/boost.throw_exception/1.83.0/MODULE.bazel": "5df92502378293277ca48837e41f33805ede9e6165acefbf83d96b861919e56e", + "https://bcr.bazel.build/modules/boost.tokenizer/1.83.0/MODULE.bazel": "6501c3df8303a842740307b7c4443bb5d01d1ce10abf88d497d9607790bf1d23", + "https://bcr.bazel.build/modules/boost.tokenizer/1.83.0/source.json": "57aa54da8b6ca096ed45a05c752cf68a343567b03e3c069a4f77f485e9389cb9", + "https://bcr.bazel.build/modules/boost.tti/1.83.0.bcr.1/MODULE.bazel": "86dd0d443379e67bb41e9b8c9097d652699ddfc0986bd2fb0462f6f5294ee84d", + "https://bcr.bazel.build/modules/boost.tti/1.83.0.bcr.1/source.json": "660900b6e3615af5b222cf699ff220787d0550e380e28408e475a6a0d354d794", + "https://bcr.bazel.build/modules/boost.tuple/1.83.0.bcr.1/MODULE.bazel": "1d540b5efd3b65eeabd3621e5187a799e21bfa9ffc6afd7d4ad307cc4a27a6d4", + "https://bcr.bazel.build/modules/boost.tuple/1.83.0.bcr.1/source.json": "7aa33ec2aaae45605049ea0ec1c1de9517a1e1278a22d0a521521d4023f9ad87", + "https://bcr.bazel.build/modules/boost.tuple/1.83.0/MODULE.bazel": "96640d5e7abec507a5dd63a9f8a6f90e45bc02d7fed6a0fb51e5e869bba9fecd", + "https://bcr.bazel.build/modules/boost.type_traits/1.83.0.bcr.1/MODULE.bazel": "89bbca2de42dc79ffcabb3625103771e4b04566eee6b8cfe7fa807d133d053f7", + "https://bcr.bazel.build/modules/boost.type_traits/1.83.0.bcr.1/source.json": "051e4969558e4a356c5059f4a4f41335b3968b91fa8ae1772898676ec95ded56", + "https://bcr.bazel.build/modules/boost.type_traits/1.83.0/MODULE.bazel": "4a094b5ecac0d41b0c071455ec0b7384e74b54118b4861bd054d5d9ff521d748", + "https://bcr.bazel.build/modules/boost.typeof/1.83.0.bcr.1/MODULE.bazel": "6b3dd6cfe993a8ccf98bce79506be41ec893c65013d7a5be9177c8a185e86d87", + "https://bcr.bazel.build/modules/boost.typeof/1.83.0.bcr.1/source.json": "0ec40beca58de1b17c1ddb18c357ef525977e4312540122b3b91cbe485556f17", + "https://bcr.bazel.build/modules/boost.typeof/1.83.0/MODULE.bazel": "da614fe02971fa4d77c33e94274d1d72ca7fae6ef3e94288f90064b1c8bb6278", + "https://bcr.bazel.build/modules/boost.unordered/1.83.0/MODULE.bazel": "2fffc6c8bbb119a59561d40dc8add727636b778adb4b1df2a523cd1b1b2393ac", + "https://bcr.bazel.build/modules/boost.unordered/1.83.0/source.json": "c1488a104633d662cd782a7afcd7278f55384eada175a1154c237fd70dbb175c", + "https://bcr.bazel.build/modules/boost.utility/1.83.0.bcr.1/MODULE.bazel": "1346dc27d6c8b7ced10896224ed3e406adac3fd79c8450d78c291228f1b9075d", + "https://bcr.bazel.build/modules/boost.utility/1.83.0.bcr.1/source.json": "15636369b5452784e7bd04f7ae52c751e591dd4ebb852688fccc66234d452929", + "https://bcr.bazel.build/modules/boost.utility/1.83.0/MODULE.bazel": "e122ee2a63d4e76dec8d2f81b13f95b7638fcbcd15f752610a3343f13bdb97fd", + "https://bcr.bazel.build/modules/boost.uuid/1.83.0.bcr.1/MODULE.bazel": "0ec51572f062cfb4795cf57c84a16b1b61890954dead42dd55e8616e09159c37", + "https://bcr.bazel.build/modules/boost.uuid/1.83.0.bcr.1/source.json": "d53f1653d8c2062f42884311de23220573e287b1eb224c465845357ded8c5a89", + "https://bcr.bazel.build/modules/boost.variant2/1.83.0.bcr.1/MODULE.bazel": "c60baa3b8923712a156197ffaf5cf9972bf35e44d00a90f7019a06761f391d3e", + "https://bcr.bazel.build/modules/boost.variant2/1.83.0.bcr.1/source.json": "041f94707bd509bc1f93782ccb6c38d491ac0dca25d26011f2047639d3a2bcd1", + "https://bcr.bazel.build/modules/boost.winapi/1.83.0.bcr.1/MODULE.bazel": "faf78b50dae672a38b77db545a460428cfe47a8d79466455ef397d76037e9e40", + "https://bcr.bazel.build/modules/boost.winapi/1.83.0.bcr.1/source.json": "0957b4dabe425e7f9d8d02db63969b808a280c11388ad7814e50b0779ae592cc", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20211025-d4f1ab9/MODULE.bazel": "6ee6353f8b1a701fe2178e1d925034294971350b6d3ac37e67e5a7d463267834", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20211025-d4f1ab9/source.json": "323bafff99739f6aba35b69a84f0bc04ddb4540a46c1694355f60f073dff3001", + "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", + "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/gflags/2.2.2/MODULE.bazel": "ba6502c3fee189734f359454db8a49b7c08afd7271b32e7c6fc38c2d2e1edbeb", + "https://bcr.bazel.build/modules/gflags/2.2.2/source.json": "b06d93702e18b5d75a69d53464c37ef5c2a9b4e237a8d4f2bf0217b3b0af2bee", + "https://bcr.bazel.build/modules/glog/0.5.0/MODULE.bazel": "f5e4f5ae1c0642c84a06a68c5428a576b28bac2cd1dfa3faf5b6d683c69007a4", + "https://bcr.bazel.build/modules/glog/0.5.0/source.json": "241565699f6a5428189e090297f6f9742259e791908cbace6e9f0eeb8104bd9e", + "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", + "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", + "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", + "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/source.json": "41e9e129f80d8c8bf103a7acc337b76e54fad1214ac0a7084bf24f4cd924b8b4", + "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", + "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", + "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d", + "https://bcr.bazel.build/modules/libevent/2.1.12/MODULE.bazel": "not found", + "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/libunwind/1.8.1/MODULE.bazel": "d9b947b15135786aed51671c22872a93e2637c74caa20e94f2061fd3b1efd6c3", + "https://bcr.bazel.build/modules/libunwind/1.8.1/source.json": "2b4e58e2fa6ed7204862ab1a4e5978e4107340884c3217596ff7910b8dfae919", + "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", + "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", + "https://bcr.bazel.build/modules/platforms/0.0.11/source.json": "f7e188b79ebedebfe75e9e1d098b8845226c7992b307e28e1496f23112e8fc29", + "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", + "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", + "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", + "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", + "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", + "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", + "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", + "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", + "https://bcr.bazel.build/modules/protobuf/27.3/MODULE.bazel": "d94898cbf9d6d25c0edca2521211413506b68a109a6b01776832ed25154d23d7", + "https://bcr.bazel.build/modules/protobuf/27.3/source.json": "d6fdc641a99c600df6eb0fa5b99879ca497dbcf6fd1287372576a83f82dd93b6", + "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", + "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", + "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/source.json": "be4789e951dd5301282729fe3d4938995dc4c1a81c2ff150afc9f1b0504c6022", + "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", + "https://bcr.bazel.build/modules/re2/2023-09-01/source.json": "e044ce89c2883cd957a2969a43e79f7752f9656f6b20050b62f90ede21ec6eb4", + "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", + "https://bcr.bazel.build/modules/rules_cc/0.0.12/MODULE.bazel": "28259f5cb8dd72d7503eb4fd1f9c89ab13759a438b1b78cb9655568d8566d7f9", + "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", + "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", + "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", + "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", + "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", + "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", + "https://bcr.bazel.build/modules/rules_cc/0.1.1/source.json": "d61627377bd7dd1da4652063e368d9366fc9a73920bfa396798ad92172cf645c", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.12.0/MODULE.bazel": "d850fab025ce79a845077035861034393f1cc1efc1d9d58d766272a26ba67def", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.12.0/source.json": "c97ddc022179fe30d1a9b94425d1e56d0a633f72332c55463e584a52ce1b38ac", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.8.0/MODULE.bazel": "e9b4fe0b66c8a7e57bd385a304ea812dcaac3b69b762e2346c808cde26b8cb8d", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", + "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", + "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", + "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", + "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", + "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", + "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", + "https://bcr.bazel.build/modules/rules_jvm_external/5.1/source.json": "5abb45cc9beb27b77aec6a65a11855ef2b55d95dfdc358e9f312b78ae0ba32d5", + "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", + "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", + "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", + "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", + "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", + "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c", + "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", + "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", + "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/source.json": "d57902c052424dfda0e71646cb12668d39c4620ee0544294d9d941e7d12bc3a9", + "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", + "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", + "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", + "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", + "https://bcr.bazel.build/modules/rules_python/0.25.0/source.json": "c45006984eeaa18ad14c006091b264bff620c41952e9184edfe225ea95c3f986", + "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", + "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", + "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.5.3/source.json": "cd53fe968dc8cd98197c052db3db6d82562960c87b61e7a90ee96f8e4e0dda97", + "https://bcr.bazel.build/modules/thrift/0.21.0/MODULE.bazel": "not found", + "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.5/MODULE.bazel": "b93d7035ac14c900dfdf7624e42cfcbfc34415aa8d3c83a6f225867c48d28dad", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.5/source.json": "30c4e5c856087a60d92e2522eafd316c0661671f4478ca94c6b7bd877010210a", + "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", + "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", + "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", + "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", + "https://raw.githubusercontent.com/secretflow/bazel-registry/main/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", + "https://raw.githubusercontent.com/secretflow/bazel-registry/main/modules/leveldb/1.23/MODULE.bazel": "450b5dafff03fd68d2c57544e0cdca1411e9cf42ae599e31fbbbfdf1ddea92b3", + "https://raw.githubusercontent.com/secretflow/bazel-registry/main/modules/leveldb/1.23/source.json": "82a078a44ec4a6c299fe108e87b6d7a0ce56dd46206bfc69495001087d7e2dfb", + "https://raw.githubusercontent.com/secretflow/bazel-registry/main/modules/openssl/3.3.2.bcr.1/MODULE.bazel": "e26204291f98123dd57567b1b558518777787b488d946fbabe181249e1336029", + "https://raw.githubusercontent.com/secretflow/bazel-registry/main/modules/openssl/3.3.2.bcr.1/source.json": "ae3d5c57f40316cdd5d017a5fdc280d5980119556ec61000c0d7d9276b1d8610" + }, + "selectedYankedVersions": {}, + "moduleExtensions": { + "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { + "general": { + "bzlTransitiveDigest": "7ii+gFxWSxHhQPrBxfMEHhtrGvHmBTvsh+KOyGunP/s=", + "usagesDigest": "lTon/N84OKzwOBzHgfdF8vJqtH0gCUX9FqtuRj2zj44=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_config_apple_cc": { + "bzlFile": "@@apple_support~//crosstool:setup.bzl", + "ruleClassName": "_apple_cc_autoconf", + "attributes": {} + }, + "local_config_apple_cc_toolchains": { + "bzlFile": "@@apple_support~//crosstool:setup.bzl", + "ruleClassName": "_apple_cc_autoconf_toolchains", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support~", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@rules_foreign_cc~//foreign_cc:extensions.bzl%tools": { + "general": { + "bzlTransitiveDigest": "f6RBwbUKS3C3FNQJAYhIWLiHaTLQDr1nKJidK7iQTBg=", + "usagesDigest": "rcono0M8vAWDAJxsr93rRa5vX8s73ZexMVAKXNZRFTg=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "cmake-3.23.2-linux-aarch64": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-aarch64.tar.gz" + ], + "sha256": "f2654bf780b53f170bbbec44d8ac67d401d24788e590faa53036a89476efa91e", + "strip_prefix": "cmake-3.23.2-linux-aarch64", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" + } + }, + "rules_foreign_cc_framework_toolchain_macos": { + "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", + "ruleClassName": "framework_toolchain_repository", + "attributes": { + "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:macos_commands.bzl", + "exec_compatible_with": [ + "@platforms//os:macos" + ] + } + }, + "ninja_1.12.1_win": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip" + ], + "sha256": "f550fec705b6d6ff58f2db3c374c2277a37691678d6aba463adcbb129108467a", + "strip_prefix": "", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja.exe\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" + } + }, + "gnumake_src": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "sha256": "dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3", + "strip_prefix": "make-4.4.1", + "urls": [ + "https://mirror.bazel.build/ftpmirror.gnu.org/gnu/make/make-4.4.1.tar.gz", + "http://ftpmirror.gnu.org/gnu/make/make-4.4.1.tar.gz" + ] + } + }, + "gettext_runtime": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "\ncc_import(\n name = \"gettext_runtime\",\n shared_library = \"bin/libintl-8.dll\",\n visibility = [\"//visibility:public\"],\n)\n ", + "sha256": "1f4269c0e021076d60a54e98da6f978a3195013f6de21674ba0edbc339c5b079", + "urls": [ + "https://mirror.bazel.build/download.gnome.org/binaries/win64/dependencies/gettext-runtime_0.18.1.1-2_win64.zip", + "https://download.gnome.org/binaries/win64/dependencies/gettext-runtime_0.18.1.1-2_win64.zip" + ] + } + }, + "cmake_src": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "sha256": "f316b40053466f9a416adf981efda41b160ca859e97f6a484b447ea299ff26aa", + "strip_prefix": "cmake-3.23.2", + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2.tar.gz" + ], + "patches": [ + "@@rules_foreign_cc~//toolchains:cmake-c++11.patch" + ] + } + }, + "bazel_skylib": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz" + ], + "sha256": "f7be3474d42aae265405a592bb7da8e171919d74c16f082a5457840f06054728" + } + }, + "cmake-3.23.2-macos-universal": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-macos-universal.tar.gz" + ], + "sha256": "853a0f9af148c5ef47282ffffee06c4c9f257be2635936755f39ca13c3286c88", + "strip_prefix": "cmake-3.23.2-macos-universal/CMake.app/Contents", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" + } + }, + "ninja_1.12.1_linux-aarch64": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux-aarch64.zip" + ], + "sha256": "5c25c6570b0155e95fce5918cb95f1ad9870df5768653afe128db822301a05a1", + "strip_prefix": "", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" + } + }, + "meson_src": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "exports_files([\"meson.py\"])\n\nfilegroup(\n name = \"runtime\",\n srcs = glob([\"mesonbuild/**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "sha256": "567e533adf255de73a2de35049b99923caf872a455af9ce03e01077e0d384bed", + "strip_prefix": "meson-1.5.1", + "urls": [ + "https://mirror.bazel.build/github.com/mesonbuild/meson/releases/download/1.5.1/meson-1.5.1.tar.gz", + "https://github.com/mesonbuild/meson/releases/download/1.5.1/meson-1.5.1.tar.gz" + ] + } + }, + "rules_foreign_cc_framework_toolchain_freebsd": { + "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", + "ruleClassName": "framework_toolchain_repository", + "attributes": { + "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:freebsd_commands.bzl", + "exec_compatible_with": [ + "@platforms//os:freebsd" + ] + } + }, + "rules_foreign_cc_framework_toolchain_linux": { + "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", + "ruleClassName": "framework_toolchain_repository", + "attributes": { + "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:linux_commands.bzl", + "exec_compatible_with": [ + "@platforms//os:linux" + ] + } + }, + "rules_python": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "sha256": "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", + "strip_prefix": "rules_python-0.23.1", + "url": "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.23.1.tar.gz" + } + }, + "ninja_1.12.1_mac": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-mac.zip" + ], + "sha256": "89a287444b5b3e98f88a945afa50ce937b8ffd1dcc59c555ad9b1baf855298c9", + "strip_prefix": "", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" + } + }, + "pkgconfig_src": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "sha256": "6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591", + "strip_prefix": "pkg-config-0.29.2", + "patches": [ + "@@rules_foreign_cc~//toolchains:pkgconfig-detectenv.patch", + "@@rules_foreign_cc~//toolchains:pkgconfig-makefile-vc.patch", + "@@rules_foreign_cc~//toolchains:pkgconfig-builtin-glib-int-conversion.patch" + ], + "urls": [ + "https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz", + "https://mirror.bazel.build/pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" + ] + } + }, + "ninja_build_src": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "integrity": "sha256-ghvf9Io/aDvEuztvC1/nstZHz2XVKutjMoyRpsbfKFo=", + "strip_prefix": "ninja-1.12.1", + "urls": [ + "https://mirror.bazel.build/github.com/ninja-build/ninja/archive/v1.12.1.tar.gz", + "https://github.com/ninja-build/ninja/archive/v1.12.1.tar.gz" + ] + } + }, + "glib_src": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "\ncc_import(\n name = \"msvc_hdr\",\n hdrs = [\"msvc_recommended_pragmas.h\"],\n visibility = [\"//visibility:public\"],\n)\n ", + "sha256": "bc96f63112823b7d6c9f06572d2ad626ddac7eb452c04d762592197f6e07898e", + "strip_prefix": "glib-2.26.1", + "urls": [ + "https://mirror.bazel.build/download.gnome.org/sources/glib/2.26/glib-2.26.1.tar.gz", + "https://download.gnome.org/sources/glib/2.26/glib-2.26.1.tar.gz" + ] + } + }, + "ninja_1.12.1_linux": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip" + ], + "sha256": "6f98805688d19672bd699fbbfa2c2cf0fc054ac3df1f0e6a47664d963d530255", + "strip_prefix": "", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" + } + }, + "cmake-3.23.2-windows-x86_64": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-windows-x86_64.zip" + ], + "sha256": "2329387f3166b84c25091c86389fb891193967740c9bcf01e7f6d3306f7ffda0", + "strip_prefix": "cmake-3.23.2-windows-x86_64", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake.exe\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake.exe\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" + } + }, + "glib_runtime": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "\nexports_files(\n [\n \"bin/libgio-2.0-0.dll\",\n \"bin/libglib-2.0-0.dll\",\n \"bin/libgmodule-2.0-0.dll\",\n \"bin/libgobject-2.0-0.dll\",\n \"bin/libgthread-2.0-0.dll\",\n ],\n visibility = [\"//visibility:public\"],\n)\n ", + "sha256": "88d857087e86f16a9be651ee7021880b3f7ba050d34a1ed9f06113b8799cb973", + "urls": [ + "https://mirror.bazel.build/download.gnome.org/binaries/win64/glib/2.26/glib_2.26.1-1_win64.zip", + "https://download.gnome.org/binaries/win64/glib/2.26/glib_2.26.1-1_win64.zip" + ] + } + }, + "rules_foreign_cc_framework_toolchains": { + "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", + "ruleClassName": "framework_toolchain_repository_hub", + "attributes": {} + }, + "bazel_features": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "sha256": "ba1282c1aa1d1fffdcf994ab32131d7c7551a9bc960fbf05f42d55a1b930cbfb", + "strip_prefix": "bazel_features-1.15.0", + "url": "https://github.com/bazel-contrib/bazel_features/releases/download/v1.15.0/bazel_features-v1.15.0.tar.gz" + } + }, + "ninja_1.12.1_toolchains": { + "bzlFile": "@@rules_foreign_cc~//toolchains:prebuilt_toolchains_repository.bzl", + "ruleClassName": "prebuilt_toolchains_repository", + "attributes": { + "repos": { + "ninja_1.12.1_linux": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ], + "ninja_1.12.1_linux-aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:linux" + ], + "ninja_1.12.1_mac": [ + "@platforms//cpu:x86_64", + "@platforms//os:macos" + ], + "ninja_1.12.1_mac_aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:macos" + ], + "ninja_1.12.1_win": [ + "@platforms//cpu:x86_64", + "@platforms//os:windows" + ] + }, + "tool": "ninja" + } + }, + "glib_dev": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "build_file_content": "\nload(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\ncc_import(\n name = \"glib_dev\",\n hdrs = glob([\"include/**\"]),\n shared_library = \"@glib_runtime//:bin/libglib-2.0-0.dll\",\n visibility = [\"//visibility:public\"],\n)\n ", + "sha256": "bdf18506df304d38be98a4b3f18055b8b8cca81beabecad0eece6ce95319c369", + "urls": [ + "https://mirror.bazel.build/download.gnome.org/binaries/win64/glib/2.26/glib-dev_2.26.1-1_win64.zip", + "https://download.gnome.org/binaries/win64/glib/2.26/glib-dev_2.26.1-1_win64.zip" + ] + } + }, + "cmake_3.23.2_toolchains": { + "bzlFile": "@@rules_foreign_cc~//toolchains:prebuilt_toolchains_repository.bzl", + "ruleClassName": "prebuilt_toolchains_repository", + "attributes": { + "repos": { + "cmake-3.23.2-linux-aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:linux" + ], + "cmake-3.23.2-linux-x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ], + "cmake-3.23.2-macos-universal": [ + "@platforms//os:macos" + ], + "cmake-3.23.2-windows-i386": [ + "@platforms//cpu:x86_32", + "@platforms//os:windows" + ], + "cmake-3.23.2-windows-x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:windows" + ] + }, + "tool": "cmake" + } + }, + "cmake-3.23.2-windows-i386": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-windows-i386.zip" + ], + "sha256": "6a4fcd6a2315b93cb23c93507efccacc30c449c2bf98f14d6032bb226c582e07", + "strip_prefix": "cmake-3.23.2-windows-i386", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake.exe\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake.exe\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" + } + }, + "cmake-3.23.2-linux-x86_64": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-x86_64.tar.gz" + ], + "sha256": "aaced6f745b86ce853661a595bdac6c5314a60f8181b6912a0a4920acfa32708", + "strip_prefix": "cmake-3.23.2-linux-x86_64", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" + } + }, + "ninja_1.12.1_mac_aarch64": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-mac.zip" + ], + "sha256": "89a287444b5b3e98f88a945afa50ce937b8ffd1dcc59c555ad9b1baf855298c9", + "strip_prefix": "", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" + } + }, + "rules_foreign_cc_framework_toolchain_windows": { + "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", + "ruleClassName": "framework_toolchain_repository", + "attributes": { + "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:windows_commands.bzl", + "exec_compatible_with": [ + "@platforms//os:windows" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_foreign_cc~", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_foreign_cc~", + "rules_foreign_cc", + "rules_foreign_cc~" + ] + ] + } + }, + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "bzlTransitiveDigest": "Kf/7zZzswWR1HLTsIkowkR+hnYBf2occRd9uW0D1KME=", + "usagesDigest": "M8/NqQuC9d+paxu5fLyiB29Pv9UCmhBJfvdJsaVw824=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "python_3_11_s390x-unknown-linux-gnu": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "e477f0749161f9aa7887964f089d9460a539f6b4a8fdab5166f898210e1a87a4", + "patches": [], + "platform": "s390x-unknown-linux-gnu", + "python_version": "3.11.4", + "release_filename": "20230726/cpython-3.11.4+20230726-s390x-unknown-linux-gnu-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-s390x-unknown-linux-gnu-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_11_aarch64-unknown-linux-gnu": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "2e84fc53f4e90e11963281c5c871f593abcb24fc796a50337fa516be99af02fb", + "patches": [], + "platform": "aarch64-unknown-linux-gnu", + "python_version": "3.11.4", + "release_filename": "20230726/cpython-3.11.4+20230726-aarch64-unknown-linux-gnu-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-aarch64-unknown-linux-gnu-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_11_aarch64-apple-darwin": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "cb6d2948384a857321f2aa40fa67744cd9676a330f08b6dad7070bda0b6120a4", + "patches": [], + "platform": "aarch64-apple-darwin", + "python_version": "3.11.4", + "release_filename": "20230726/cpython-3.11.4+20230726-aarch64-apple-darwin-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-aarch64-apple-darwin-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_9": { + "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", + "ruleClassName": "toolchain_aliases", + "attributes": { + "python_version": "3.9.17", + "user_repository_name": "python_3_9" + } + }, + "pythons_hub": { + "bzlFile": "@@rules_python~//python/extensions/private:pythons_hub.bzl", + "ruleClassName": "hub_repo", + "attributes": { + "default_python_version": "3.11", + "toolchain_prefixes": [ + "_0000_python_3_9_", + "_0001_python_3_11_" + ], + "toolchain_python_versions": [ + "3.9", + "3.11" + ], + "toolchain_set_python_version_constraints": [ + "True", + "False" + ], + "toolchain_user_repository_names": [ + "python_3_9", + "python_3_11" + ] + } + }, + "python_3_11_x86_64-pc-windows-msvc": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "878614c03ea38538ae2f758e36c85d2c0eb1eaaca86cd400ff8c76693ee0b3e1", + "patches": [], + "platform": "x86_64-pc-windows-msvc", + "python_version": "3.11.4", + "release_filename": "20230726/cpython-3.11.4+20230726-x86_64-pc-windows-msvc-shared-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-x86_64-pc-windows-msvc-shared-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_9_aarch64-apple-darwin": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "73dbe2d702210b566221da9265acc274ba15275c5d0d1fa327f44ad86cde9aa1", + "patches": [], + "platform": "aarch64-apple-darwin", + "python_version": "3.9.17", + "release_filename": "20230726/cpython-3.9.17+20230726-aarch64-apple-darwin-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-aarch64-apple-darwin-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_9_x86_64-pc-windows-msvc": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "9b9a1e21eff29dcf043cea38180cf8ca3604b90117d00062a7b31605d4157714", + "patches": [], + "platform": "x86_64-pc-windows-msvc", + "python_version": "3.9.17", + "release_filename": "20230726/cpython-3.9.17+20230726-x86_64-pc-windows-msvc-shared-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-x86_64-pc-windows-msvc-shared-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_9_ppc64le-unknown-linux-gnu": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "c591a28d943dce5cf9833e916125fdfbeb3120270c4866ee214493ccb5b83c3c", + "patches": [], + "platform": "ppc64le-unknown-linux-gnu", + "python_version": "3.9.17", + "release_filename": "20230726/cpython-3.9.17+20230726-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-ppc64le-unknown-linux-gnu-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_9_aarch64-unknown-linux-gnu": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "b77012ddaf7e0673e4aa4b1c5085275a06eee2d66f33442b5c54a12b62b96cbe", + "patches": [], + "platform": "aarch64-unknown-linux-gnu", + "python_version": "3.9.17", + "release_filename": "20230726/cpython-3.9.17+20230726-aarch64-unknown-linux-gnu-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-aarch64-unknown-linux-gnu-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_11": { + "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", + "ruleClassName": "toolchain_aliases", + "attributes": { + "python_version": "3.11.4", + "user_repository_name": "python_3_11" + } + }, + "python_3_11_ppc64le-unknown-linux-gnu": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "df7b92ed9cec96b3bb658fb586be947722ecd8e420fb23cee13d2e90abcfcf25", + "patches": [], + "platform": "ppc64le-unknown-linux-gnu", + "python_version": "3.11.4", + "release_filename": "20230726/cpython-3.11.4+20230726-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-ppc64le-unknown-linux-gnu-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_11_x86_64-apple-darwin": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "47e1557d93a42585972772e82661047ca5f608293158acb2778dccf120eabb00", + "patches": [], + "platform": "x86_64-apple-darwin", + "python_version": "3.11.4", + "release_filename": "20230726/cpython-3.11.4+20230726-x86_64-apple-darwin-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-x86_64-apple-darwin-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_versions": { + "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", + "ruleClassName": "multi_toolchain_aliases", + "attributes": { + "python_versions": { + "3.11": "python_3_11", + "3.9": "python_3_9" + } + } + }, + "python_3_9_x86_64-apple-darwin": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "dfe1bea92c94b9cb779288b0b06e39157c5ff7e465cdd24032ac147c2af485c0", + "patches": [], + "platform": "x86_64-apple-darwin", + "python_version": "3.9.17", + "release_filename": "20230726/cpython-3.9.17+20230726-x86_64-apple-darwin-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-x86_64-apple-darwin-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_9_x86_64-unknown-linux-gnu": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "26c4a712b4b8e11ed5c027db5654eb12927c02da4857b777afb98f7a930ce637", + "patches": [], + "platform": "x86_64-unknown-linux-gnu", + "python_version": "3.9.17", + "release_filename": "20230726/cpython-3.9.17+20230726-x86_64-unknown-linux-gnu-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-x86_64-unknown-linux-gnu-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_9_s390x-unknown-linux-gnu": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "01454d7cc7c9c2fccde42ba868c4f372eaaafa48049d49dd94c9cf2875f497e6", + "patches": [], + "platform": "s390x-unknown-linux-gnu", + "python_version": "3.9.17", + "release_filename": "20230726/cpython-3.9.17+20230726-s390x-unknown-linux-gnu-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-s390x-unknown-linux-gnu-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + }, + "python_3_11_x86_64-unknown-linux-gnu": { + "bzlFile": "@@rules_python~//python:repositories.bzl", + "ruleClassName": "python_repository", + "attributes": { + "sha256": "e26247302bc8e9083a43ce9e8dd94905b40d464745b1603041f7bc9a93c65d05", + "patches": [], + "platform": "x86_64-unknown-linux-gnu", + "python_version": "3.11.4", + "release_filename": "20230726/cpython-3.11.4+20230726-x86_64-unknown-linux-gnu-install_only.tar.gz", + "urls": [ + "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-x86_64-unknown-linux-gnu-install_only.tar.gz" + ], + "distutils_content": "", + "strip_prefix": "python", + "coverage_tool": "", + "ignore_root_user_error": false + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_python~", + "bazel_tools", + "bazel_tools" + ] + ] + } + } + } +} diff --git a/Makefile b/Makefile index a2309c8d82..16a10ae2ea 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ include config.mk # 2. Removed -Werror: Not block compilation for non-vital warnings, especially when the # code is tested on newer systems. If the code is used in production, config `config_brpc.sh -werror'. CPPFLAGS+=-DBTHREAD_USE_FAST_PTHREAD_MUTEX -D_GNU_SOURCE -DUSE_SYMBOLIZE -DNO_TCMALLOC -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DNDEBUG -DBRPC_REVISION=\"$(shell ./tools/get_brpc_revision.sh .)\" -CXXFLAGS+=$(CPPFLAGS) -std=c++17 -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -Wno-deprecated-declarations -Wno-unused-but-set-variable +CXXFLAGS+=$(CPPFLAGS) -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -Wno-deprecated-declarations -Wno-unused-but-set-variable CFLAGS=$(CPPFLAGS) -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-unused-parameter -fno-omit-frame-pointer -Wno-deprecated-declarations -Wno-unused-but-set-variable DEBUG_CXXFLAGS = $(filter-out -DNDEBUG,$(CXXFLAGS)) -DUNIT_TEST -DBVAR_NOT_LINK_DEFAULT_VARIABLES DEBUG_CFLAGS = $(filter-out -DNDEBUG,$(CFLAGS)) -DUNIT_TEST diff --git a/example/couchbase_c++/couchbase-cloud-cert.txt b/example/couchbase_c++/couchbase-cloud-cert.txt new file mode 100644 index 0000000000..6a12be36f6 --- /dev/null +++ b/example/couchbase_c++/couchbase-cloud-cert.txt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDFTCCAf2gAwIBAgIRANLVkgOvtaXiQJi0V6qeNtswDQYJKoZIhvcNAQELBQAw +JDESMBAGA1UECgwJQ291Y2hiYXNlMQ4wDAYDVQQLDAVDbG91ZDAeFw0xOTEyMDYy +MjEyNTlaFw0yOTEyMDYyMzEyNTlaMCQxEjAQBgNVBAoMCUNvdWNoYmFzZTEOMAwG +A1UECwwFQ2xvdWQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCfvOIi +enG4Dp+hJu9asdxEMRmH70hDyMXv5ZjBhbo39a42QwR59y/rC/sahLLQuNwqif85 +Fod1DkqgO6Ng3vecSAwyYVkj5NKdycQu5tzsZkghlpSDAyI0xlIPSQjoORA/pCOU +WOpymA9dOjC1bo6rDyw0yWP2nFAI/KA4Z806XeqLREuB7292UnSsgFs4/5lqeil6 +rL3ooAw/i0uxr/TQSaxi1l8t4iMt4/gU+W52+8Yol0JbXBTFX6itg62ppb/Eugmn +mQRMgL67ccZs7cJ9/A0wlXencX2ohZQOR3mtknfol3FH4+glQFn27Q4xBCzVkY9j +KQ20T1LgmGSngBInAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE +FJQOBPvrkU2In1Sjoxt97Xy8+cKNMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0B +AQsFAAOCAQEARgM6XwcXPLSpFdSf0w8PtpNGehmdWijPM3wHb7WZiS47iNen3oq8 +m2mm6V3Z57wbboPpfI+VEzbhiDcFfVnK1CXMC0tkF3fnOG1BDDvwt4jU95vBiNjY +xdzlTP/Z+qr0cnVbGBSZ+fbXstSiRaaAVcqQyv3BRvBadKBkCyPwo+7svQnScQ5P +Js7HEHKVms5tZTgKIw1fbmgR2XHleah1AcANB+MAPBCcTgqurqr5G7W2aPSBLLGA +fRIiVzm7VFLc7kWbp7ENH39HVG6TZzKnfl9zJYeiklo5vQQhGSMhzBsO70z4RRzi +DPFAN/4qZAgD5q3AFNIq2WWADFQGSwVJhg== +-----END CERTIFICATE----- + diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 058c5e2e17..f5234d25c4 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -46,7 +46,7 @@ CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseResponse::metadata_trac bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest(string server, string bucket, CouchbaseMetadataTracking::CollectionManifest manifest){ // Then update the collection manifest with proper locking { - std::unique_lock write_lock(rw_bucket_to_collection_manifest_mutex); + UniqueLock write_lock(rw_bucket_to_collection_manifest_mutex); bucket_to_collection_manifest[server][bucket] = manifest; } @@ -54,7 +54,7 @@ bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest(string s } bool brpc::CouchbaseMetadataTracking::get_bucket_to_collection_manifest(string server, string bucket, CouchbaseMetadataTracking::CollectionManifest *manifest){ - std::shared_lock read_lock(rw_bucket_to_collection_manifest_mutex); + SharedLock read_lock(rw_bucket_to_collection_manifest_mutex); auto it1 = bucket_to_collection_manifest.find(server); if(it1 == bucket_to_collection_manifest.end()){ return false; diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 09b82049a3..71df9ef8e6 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -28,7 +28,9 @@ #include "brpc/pb_compat.h" #include "butil/iobuf.h" #include "butil/strings/string_piece.h" -#include +#include +#include +#include #include using namespace std; @@ -44,6 +46,74 @@ namespace policy { void SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); } +// Simple C++11 compatible reader-writer lock +class ReaderWriterLock { +private: + std::mutex mutex_; + std::condition_variable reader_cv_; + std::condition_variable writer_cv_; + std::atomic reader_count_; + std::atomic writer_active_; + std::atomic waiting_writers_; + +public: + ReaderWriterLock() : reader_count_(0), writer_active_(false), waiting_writers_(0) {} + + void lock_shared() { + std::unique_lock lock(mutex_); + reader_cv_.wait(lock, [this] { return !writer_active_.load() && waiting_writers_.load() == 0; }); + reader_count_.fetch_add(1); + } + + void unlock_shared() { + reader_count_.fetch_sub(1); + if (reader_count_.load() == 0) { + std::lock_guard lock(mutex_); + writer_cv_.notify_one(); + } + } + + void lock() { + std::unique_lock lock(mutex_); + waiting_writers_.fetch_add(1); + writer_cv_.wait(lock, [this] { return !writer_active_.load() && reader_count_.load() == 0; }); + waiting_writers_.fetch_sub(1); + writer_active_.store(true); + } + + void unlock() { + writer_active_.store(false); + std::lock_guard lock(mutex_); + writer_cv_.notify_one(); + reader_cv_.notify_all(); + } +}; + +// RAII helper classes +class SharedLock { +private: + ReaderWriterLock& lock_; +public: + explicit SharedLock(ReaderWriterLock& lock) : lock_(lock) { + lock_.lock_shared(); + } + ~SharedLock() { + lock_.unlock_shared(); + } +}; + +class UniqueLock { +private: + ReaderWriterLock& lock_; +public: + explicit UniqueLock(ReaderWriterLock& lock) : lock_(lock) { + lock_.lock(); + } + ~UniqueLock() { + lock_.unlock(); + } +}; + class CouchbaseMetadataTracking{ public: struct CollectionManifest{ @@ -52,7 +122,7 @@ class CouchbaseMetadataTracking{ }; private: unordered_map> bucket_to_collection_manifest; - shared_mutex rw_bucket_to_collection_manifest_mutex; + ReaderWriterLock rw_bucket_to_collection_manifest_mutex; public: CouchbaseMetadataTracking() {} diff --git a/test/Makefile b/test/Makefile index d48d3e3429..30e196bbce 100644 --- a/test/Makefile +++ b/test/Makefile @@ -19,7 +19,7 @@ NEED_GPERFTOOLS=1 NEED_GTEST=1 include ../config.mk CPPFLAGS+=-DBTHREAD_USE_FAST_PTHREAD_MUTEX -D_GNU_SOURCE -DUSE_SYMBOLIZE -DNO_TCMALLOC -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DUNIT_TEST -Dprivate=public -Dprotected=public -DBVAR_NOT_LINK_DEFAULT_VARIABLES --include sstream_workaround.h -CXXFLAGS+=$(CPPFLAGS) -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++17 +CXXFLAGS+=$(CPPFLAGS) -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++0x # On Darwin, only gtest library is needed, other libraries have been linked in `brpc.dbg.dylib` ifeq ($(SYSTEM),Darwin) diff --git a/test/brpc_http_rpc_protocol_unittest.cpp b/test/brpc_http_rpc_protocol_unittest.cpp index 1f7b8420cc..5a6839c86f 100644 --- a/test/brpc_http_rpc_protocol_unittest.cpp +++ b/test/brpc_http_rpc_protocol_unittest.cpp @@ -20,6 +20,7 @@ // Date: Sun Jul 13 15:04:18 CST 2014 #include +#include #include #include #include @@ -1127,7 +1128,7 @@ class UploadServiceImpl : public ::test::UploadService { private: void check_header(brpc::Controller* cntl) { const std::string* test_header = cntl->http_request().GetHeader(TEST_PROGRESSIVE_HEADER); - CHECK_NE(test_header, nullptr); + GOOGLE_CHECK_NOTNULL(test_header); CHECK_EQ(*test_header, TEST_PROGRESSIVE_HEADER_VAL); } }; diff --git a/test/brpc_load_balancer_unittest.cpp b/test/brpc_load_balancer_unittest.cpp index 951a7e4fd6..07059484d7 100644 --- a/test/brpc_load_balancer_unittest.cpp +++ b/test/brpc_load_balancer_unittest.cpp @@ -21,8 +21,6 @@ #include #include -#include -#include #include #include "bthread/bthread.h" #include "gperftools_helper.h" @@ -407,9 +405,7 @@ TEST_F(LoadBalancerTest, la_sanity) { ValidateLALB(lalb, cur_count); const size_t before_removal = cur_count; - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(ids.begin(), ids.end(), g); + std::random_shuffle(ids.begin(), ids.end()); for (size_t i = 0; i < N / 2; ++i) { const brpc::ServerId id = ids.back(); ids.pop_back(); @@ -542,9 +538,7 @@ TEST_F(LoadBalancerTest, update_while_selection) { } else { removed.assign(ids.begin(), ids.begin() + 255); } - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(removed.begin(), removed.end(), g); + std::random_shuffle(removed.begin(), removed.end()); removed.pop_back(); ASSERT_EQ(removed.size(), lb->RemoveServersInBatch(removed)); ASSERT_EQ(removed.size(), lb->AddServersInBatch(removed)); diff --git a/test/find_cstr_unittest.cpp b/test/find_cstr_unittest.cpp index 6b424a5e8e..340ae0b21f 100644 --- a/test/find_cstr_unittest.cpp +++ b/test/find_cstr_unittest.cpp @@ -16,7 +16,6 @@ // under the License. #include -#include #include #include "butil/find_cstr.h" #include "butil/time.h" @@ -76,9 +75,7 @@ TEST_F(FindCstrTest, perf) { j = 0; } } - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(all_keys.begin(), all_keys.end(), g); + std::random_shuffle(all_keys.begin(), all_keys.end()); int sum = 0; butil::Timer tm; tm.start(); diff --git a/test/flat_map_unittest.cpp b/test/flat_map_unittest.cpp index 834c7e2cb1..a394218ad6 100644 --- a/test/flat_map_unittest.cpp +++ b/test/flat_map_unittest.cpp @@ -18,8 +18,6 @@ #include #include #include -#include -#include #include #include #include @@ -876,9 +874,7 @@ TEST_F(FlatMapTest, perf_cmp_with_map_storing_pointers) { ASSERT_EQ(m1.size(), m2.size()); ASSERT_EQ(m1.size(), m3.size()); - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(r.begin(), r.end(), g); + std::random_shuffle(r.begin(), r.end()); sum = 0; tm.start(); @@ -1257,9 +1253,7 @@ void perf_insert_erase(bool random, const T& value) { } if (random) { - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(keys.begin(), keys.end(), g); + random_shuffle(keys.begin(), keys.end()); } id_map.clear(); @@ -1324,9 +1318,7 @@ void perf_insert_erase(bool random, const T& value) { << "/" << hash_tm.n_elapsed() / keys.size(); if (random) { - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(keys.begin(), keys.end(), g); + random_shuffle(keys.begin(), keys.end()); } id_tm.start(); @@ -1446,9 +1438,7 @@ void perf_seek(const T& value) { hash_map[keys[i]] = value; } - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(keys.begin(), keys.end(), g); + random_shuffle(keys.begin(), keys.end()); long sum = 0; id_tm.start(); diff --git a/test/object_pool_unittest.cpp b/test/object_pool_unittest.cpp index abfdb84602..cfc891aff7 100644 --- a/test/object_pool_unittest.cpp +++ b/test/object_pool_unittest.cpp @@ -16,8 +16,6 @@ // under the License. #include -#include -#include #include "butil/time.h" #include "butil/macros.h" @@ -270,9 +268,7 @@ void* get_and_return_int(void*) { } tm1.stop(); - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(v.begin(), v.end(), g); + std::random_shuffle(v.begin(), v.end()); tm2.start(); for (size_t i = 0; i < v.size(); ++i) { @@ -312,9 +308,7 @@ void* new_and_delete_int(void*) { } tm1.stop(); - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(v2.begin(), v2.end(), g); + std::random_shuffle(v2.begin(), v2.end()); tm2.start(); for (size_t i = 0; i < v2.size(); ++i) { diff --git a/test/resource_pool_unittest.cpp b/test/resource_pool_unittest.cpp index 3a7e213d43..9a56ff3bb5 100644 --- a/test/resource_pool_unittest.cpp +++ b/test/resource_pool_unittest.cpp @@ -16,8 +16,6 @@ // under the License. #include -#include -#include #include "butil/time.h" #include "butil/macros.h" #include "butil/fast_rand.h" @@ -318,9 +316,7 @@ void* get_and_return_int(void*) { } tm1.stop(); - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(v.begin(), v.end(), g); + std::random_shuffle(v.begin(), v.end()); tm2.start(); for (size_t i = 0; i < v.size(); ++i) { @@ -359,9 +355,7 @@ void* new_and_delete_int(void*) { } tm1.stop(); - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(v2.begin(), v2.end(), g); + std::random_shuffle(v2.begin(), v2.end()); tm2.start(); for (size_t i = 0; i < v2.size(); ++i) { From 464cf6907abf824748e4e9dc51d7537cffd08406 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Fri, 3 Oct 2025 04:28:06 +0530 Subject: [PATCH 23/49] Formatted code to google c++ format --- .../couchbase_c++/couchbase-cloud-cert.txt | 20 - example/couchbase_c++/couchbase_client.cpp | 336 ++++--- .../multithreaded_couchbase_client.cpp | 249 +++--- src/brpc/couchbase.cpp | 743 ++++++++++------ src/brpc/couchbase.h | 839 ++++++++++-------- 5 files changed, 1236 insertions(+), 951 deletions(-) delete mode 100644 example/couchbase_c++/couchbase-cloud-cert.txt diff --git a/example/couchbase_c++/couchbase-cloud-cert.txt b/example/couchbase_c++/couchbase-cloud-cert.txt deleted file mode 100644 index 6a12be36f6..0000000000 --- a/example/couchbase_c++/couchbase-cloud-cert.txt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDFTCCAf2gAwIBAgIRANLVkgOvtaXiQJi0V6qeNtswDQYJKoZIhvcNAQELBQAw -JDESMBAGA1UECgwJQ291Y2hiYXNlMQ4wDAYDVQQLDAVDbG91ZDAeFw0xOTEyMDYy -MjEyNTlaFw0yOTEyMDYyMzEyNTlaMCQxEjAQBgNVBAoMCUNvdWNoYmFzZTEOMAwG -A1UECwwFQ2xvdWQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCfvOIi -enG4Dp+hJu9asdxEMRmH70hDyMXv5ZjBhbo39a42QwR59y/rC/sahLLQuNwqif85 -Fod1DkqgO6Ng3vecSAwyYVkj5NKdycQu5tzsZkghlpSDAyI0xlIPSQjoORA/pCOU -WOpymA9dOjC1bo6rDyw0yWP2nFAI/KA4Z806XeqLREuB7292UnSsgFs4/5lqeil6 -rL3ooAw/i0uxr/TQSaxi1l8t4iMt4/gU+W52+8Yol0JbXBTFX6itg62ppb/Eugmn -mQRMgL67ccZs7cJ9/A0wlXencX2ohZQOR3mtknfol3FH4+glQFn27Q4xBCzVkY9j -KQ20T1LgmGSngBInAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE -FJQOBPvrkU2In1Sjoxt97Xy8+cKNMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0B -AQsFAAOCAQEARgM6XwcXPLSpFdSf0w8PtpNGehmdWijPM3wHb7WZiS47iNen3oq8 -m2mm6V3Z57wbboPpfI+VEzbhiDcFfVnK1CXMC0tkF3fnOG1BDDvwt4jU95vBiNjY -xdzlTP/Z+qr0cnVbGBSZ+fbXstSiRaaAVcqQyv3BRvBadKBkCyPwo+7svQnScQ5P -Js7HEHKVms5tZTgKIw1fbmgR2XHleah1AcANB+MAPBCcTgqurqr5G7W2aPSBLLGA -fRIiVzm7VFLc7kWbp7ENH39HVG6TZzKnfl9zJYeiklo5vQQhGSMhzBsO70z4RRzi -DPFAN/4qZAgD5q3AFNIq2WWADFQGSwVJhg== ------END CERTIFICATE----- - diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index ef5e7d8b54..c061a7f464 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -27,14 +27,15 @@ #define RED "\033[31m" #define RESET "\033[0m" -DEFINE_string(server, "cb.dqklxewecglohzwb.cloud.couchbase.com:11207", "IP Address of server"); +DEFINE_string(server, "cb.dqklxewecglohzwb.cloud.couchbase.com:11207", + "IP Address of server"); int main() { // Create CouchbaseOperations instance for high-level operations brpc::CouchbaseOperations couchbase_ops; - std::cout << GREEN << "Using high-level CouchbaseOperations interface" << RESET - << std::endl; + std::cout << GREEN << "Using high-level CouchbaseOperations interface" + << RESET << std::endl; // Ask username and password for authentication std::string username; @@ -55,7 +56,8 @@ int main() { } // Use high-level authentication method - brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate(username, password, FLAGS_server, true, "couchbase-cloud-cert.txt"); + brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( + username, password, FLAGS_server, true, "couchbase-cloud-cert.txt"); if (!auth_result.success) { LOG(ERROR) << "Authentication failed: " << auth_result.error_message; return -1; @@ -79,7 +81,8 @@ int main() { } // Use high-level bucket selection method - brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.SelectBucket(bucket_name); + brpc::CouchbaseOperations::Result bucket_result = + couchbase_ops.SelectBucket(bucket_name); if (!bucket_result.success) { LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; return -1; @@ -89,21 +92,27 @@ int main() { // Add operation using high-level method std::string add_key = "user::test_brpc_binprot"; - std::string add_value = R"({"name": "John Doe", "age": 30, "email": "john@example.com"})"; - - brpc::CouchbaseOperations::Result add_result = couchbase_ops.Add(add_key, add_value); + std::string add_value = + R"({"name": "John Doe", "age": 30, "email": "john@example.com"})"; + + brpc::CouchbaseOperations::Result add_result = + couchbase_ops.Add(add_key, add_value); if (add_result.success) { std::cout << GREEN << "ADD operation successful" << RESET << std::endl; } else { - std::cout << RED << "ADD operation failed: " << add_result.error_message << RESET << std::endl; + std::cout << RED << "ADD operation failed: " << add_result.error_message + << RESET << std::endl; } // Try to ADD the same key again (should fail with key exists) - brpc::CouchbaseOperations::Result add_result2 = couchbase_ops.Add(add_key, add_value); + brpc::CouchbaseOperations::Result add_result2 = + couchbase_ops.Add(add_key, add_value); if (add_result2.success) { - std::cout << GREEN << "Second ADD operation unexpectedly successful" << RESET << std::endl; + std::cout << GREEN << "Second ADD operation unexpectedly successful" + << RESET << std::endl; } else { - std::cout << RED << "Second ADD operation failed as expected: " << add_result2.error_message << RESET << std::endl; + std::cout << RED << "Second ADD operation failed as expected: " + << add_result2.error_message << RESET << std::endl; } // Get operation using high-level method brpc::CouchbaseOperations::Result get_result = couchbase_ops.Get(add_key); @@ -111,62 +120,78 @@ int main() { std::cout << GREEN << "GET operation successful" << RESET << std::endl; std::cout << "GET value: " << get_result.value << std::endl; } else { - std::cout << RED << "GET operation failed: " << get_result.error_message << RESET << std::endl; + std::cout << RED << "GET operation failed: " << get_result.error_message + << RESET << std::endl; } // Add binprot item1 using high-level method std::string item1_key = "binprot_item1"; - brpc::CouchbaseOperations::Result item1_result = couchbase_ops.Add(item1_key, add_value); + brpc::CouchbaseOperations::Result item1_result = + couchbase_ops.Add(item1_key, add_value); if (item1_result.success) { std::cout << GREEN << "ADD binprot item1 successful" << RESET << std::endl; } else { - std::cout << RED << "ADD binprot item1 failed: " << item1_result.error_message << RESET << std::endl; + std::cout << RED + << "ADD binprot item1 failed: " << item1_result.error_message + << RESET << std::endl; } // Add binprot item2 using high-level method std::string item2_key = "binprot_item2"; - brpc::CouchbaseOperations::Result item2_result = couchbase_ops.Add(item2_key, add_value); + brpc::CouchbaseOperations::Result item2_result = + couchbase_ops.Add(item2_key, add_value); if (item2_result.success) { std::cout << GREEN << "ADD binprot item2 successful" << RESET << std::endl; } else { - std::cout << RED << "ADD binprot item2 failed: " << item2_result.error_message << RESET << std::endl; + std::cout << RED + << "ADD binprot item2 failed: " << item2_result.error_message + << RESET << std::endl; } // Add binprot item3 using high-level method std::string item3_key = "binprot_item3"; - brpc::CouchbaseOperations::Result item3_result = couchbase_ops.Add(item3_key, add_value); + brpc::CouchbaseOperations::Result item3_result = + couchbase_ops.Add(item3_key, add_value); if (item3_result.success) { std::cout << GREEN << "ADD binprot item3 successful" << RESET << std::endl; } else { - std::cout << RED << "ADD binprot item3 failed: " << item3_result.error_message << RESET << std::endl; + std::cout << RED + << "ADD binprot item3 failed: " << item3_result.error_message + << RESET << std::endl; } - - - // Perform an UPSERT on the existing key using high-level method + // Perform an UPSERT on the existing key using high-level method std::string upsert_key = "user::test_brpc_binprot"; - std::string upsert_value = R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})"; - brpc::CouchbaseOperations::Result upsert_result = couchbase_ops.Upsert(upsert_key, upsert_value); + std::string upsert_value = + R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})"; + brpc::CouchbaseOperations::Result upsert_result = + couchbase_ops.Upsert(upsert_key, upsert_value); if (upsert_result.success) { - std::cout << GREEN - << "UPSERT operation successful when the document exists in the server" - << RESET << std::endl; + std::cout + << GREEN + << "UPSERT operation successful when the document exists in the server" + << RESET << std::endl; } else { - std::cout << RED - << "UPSERT operation failed when the document exists in the server: " - << upsert_result.error_message << RESET << std::endl; + std::cout + << RED + << "UPSERT operation failed when the document exists in the server: " + << upsert_result.error_message << RESET << std::endl; } // Do UPSERT operation on a new document using high-level method std::string new_upsert_key = "user::test_brpc_new_upsert"; - std::string new_upsert_value = R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})"; - brpc::CouchbaseOperations::Result new_upsert_result = couchbase_ops.Upsert(new_upsert_key, new_upsert_value); + std::string new_upsert_value = + R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})"; + brpc::CouchbaseOperations::Result new_upsert_result = + couchbase_ops.Upsert(new_upsert_key, new_upsert_value); if (new_upsert_result.success) { std::cout << GREEN - << "UPSERT operation successful when the document doesn't exist in the server" + << "UPSERT operation successful when the document doesn't exist " + "in the server" << RESET << std::endl; } else { std::cout << RED - << "UPSERT operation failed when document does not exist in the server: " + << "UPSERT operation failed when document does not exist in the " + "server: " << new_upsert_result.error_message << RESET << std::endl; } @@ -174,9 +199,8 @@ int main() { std::string check_key = "user::test_brpc_new_upsert"; brpc::CouchbaseOperations::Result check_result = couchbase_ops.Get(check_key); if (check_result.success) { - std::cout << GREEN - << "GET after UPSERT operation successful - Value: " << check_result.value - << RESET << std::endl; + std::cout << GREEN << "GET after UPSERT operation successful - Value: " + << check_result.value << RESET << std::endl; } else { std::cout << RED << "GET after UPSERT operation failed: " << check_result.error_message << RESET << std::endl; @@ -184,7 +208,8 @@ int main() { // Delete a non-existent key using high-level method std::string delete_key = "Nonexistent_key"; - brpc::CouchbaseOperations::Result delete_result = couchbase_ops.Delete(delete_key); + brpc::CouchbaseOperations::Result delete_result = + couchbase_ops.Delete(delete_key); if (delete_result.success) { std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; } else { @@ -194,7 +219,8 @@ int main() { // Delete the existing key using high-level method std::string delete_existing_key = "user::test_brpc_binprot"; - brpc::CouchbaseOperations::Result delete_existing_result = couchbase_ops.Delete(delete_existing_key); + brpc::CouchbaseOperations::Result delete_existing_result = + couchbase_ops.Delete(delete_existing_key); if (delete_existing_result.success) { std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; } else { @@ -216,60 +242,78 @@ int main() { // ------------------------------------------------------------------ // Collection-scoped CRUD operations (only if collection id was retrieved) // ------------------------------------------------------------------ - // 1. ADD in collection using high-level method - std::string coll_key = "user::collection_doc"; - std::string coll_value = R"({"type":"collection","op":"add","v":1})"; - brpc::CouchbaseOperations::Result coll_add_result = couchbase_ops.Add(coll_key, coll_value, collection_name); - if (coll_add_result.success) { - std::cout << GREEN << "Collection ADD success" << RESET << std::endl; - } else { - std::cout << RED << "Collection ADD failed: " << coll_add_result.error_message << RESET << std::endl; - } + // 1. ADD in collection using high-level method + std::string coll_key = "user::collection_doc"; + std::string coll_value = R"({"type":"collection","op":"add","v":1})"; + brpc::CouchbaseOperations::Result coll_add_result = + couchbase_ops.Add(coll_key, coll_value, collection_name); + if (coll_add_result.success) { + std::cout << GREEN << "Collection ADD success" << RESET << std::endl; + } else { + std::cout << RED + << "Collection ADD failed: " << coll_add_result.error_message + << RESET << std::endl; + } - // 2. GET from collection using high-level method - brpc::CouchbaseOperations::Result coll_get_result = couchbase_ops.Get(coll_key, collection_name); - if (coll_get_result.success) { - std::cout << GREEN << "Collection GET success value=" << coll_get_result.value << RESET << std::endl; - } else { - std::cout << RED << "Collection GET failed: " << coll_get_result.error_message << RESET << std::endl; - } + // 2. GET from collection using high-level method + brpc::CouchbaseOperations::Result coll_get_result = + couchbase_ops.Get(coll_key, collection_name); + if (coll_get_result.success) { + std::cout << GREEN + << "Collection GET success value=" << coll_get_result.value + << RESET << std::endl; + } else { + std::cout << RED + << "Collection GET failed: " << coll_get_result.error_message + << RESET << std::endl; + } - // 3. UPSERT in collection using high-level method - std::string coll_upsert_value = R"({"type":"collection","op":"upsert","v":2})"; - brpc::CouchbaseOperations::Result coll_upsert_result = couchbase_ops.Upsert(coll_key, coll_upsert_value, collection_name); - if (coll_upsert_result.success) { - std::cout << GREEN << "Collection UPSERT success" << RESET << std::endl; - } else { - std::cout << RED << "Collection UPSERT failed: " << coll_upsert_result.error_message << RESET << std::endl; - } + // 3. UPSERT in collection using high-level method + std::string coll_upsert_value = + R"({"type":"collection","op":"upsert","v":2})"; + brpc::CouchbaseOperations::Result coll_upsert_result = + couchbase_ops.Upsert(coll_key, coll_upsert_value, collection_name); + if (coll_upsert_result.success) { + std::cout << GREEN << "Collection UPSERT success" << RESET << std::endl; + } else { + std::cout << RED << "Collection UPSERT failed: " + << coll_upsert_result.error_message << RESET << std::endl; + } - // 4. GET again to verify upsert using high-level method - brpc::CouchbaseOperations::Result coll_get2_result = couchbase_ops.Get(coll_key, collection_name); - if (coll_get2_result.success) { - std::cout << GREEN << "Collection GET(after upsert) value=" << coll_get2_result.value << RESET << std::endl; - } + // 4. GET again to verify upsert using high-level method + brpc::CouchbaseOperations::Result coll_get2_result = + couchbase_ops.Get(coll_key, collection_name); + if (coll_get2_result.success) { + std::cout << GREEN + << "Collection GET(after upsert) value=" << coll_get2_result.value + << RESET << std::endl; + } - // 5. DELETE from collection using high-level method - brpc::CouchbaseOperations::Result coll_del_result = couchbase_ops.Delete(coll_key, collection_name); - if (coll_del_result.success) { - std::cout << GREEN << "Collection DELETE success" << RESET << std::endl; - } else { - std::cout << RED << "Collection DELETE failed: " << coll_del_result.error_message << RESET << std::endl; - } + // 5. DELETE from collection using high-level method + brpc::CouchbaseOperations::Result coll_del_result = + couchbase_ops.Delete(coll_key, collection_name); + if (coll_del_result.success) { + std::cout << GREEN << "Collection DELETE success" << RESET << std::endl; + } else { + std::cout << RED + << "Collection DELETE failed: " << coll_del_result.error_message + << RESET << std::endl; + } // ------------------------------------------------------------------ // Pipeline Operations Demo // ------------------------------------------------------------------ - std::cout << GREEN << "\n=== Pipeline Operations Demo ===" << RESET << std::endl; - + std::cout << GREEN << "\n=== Pipeline Operations Demo ===" << RESET + << std::endl; + // Begin a new pipeline if (!couchbase_ops.BeginPipeline()) { std::cout << RED << "Failed to begin pipeline" << RESET << std::endl; return -1; } - + std::cout << "Pipeline started. Adding multiple operations..." << std::endl; - + // Add multiple operations to the pipeline std::string pipeline_key1 = "pipeline::doc1"; std::string pipeline_key2 = "pipeline::doc2"; @@ -277,97 +321,135 @@ int main() { std::string pipeline_value1 = R"({"operation": "pipeline_add", "id": 1})"; std::string pipeline_value2 = R"({"operation": "pipeline_upsert", "id": 2})"; std::string pipeline_value3 = R"({"operation": "pipeline_add", "id": 3})"; - + // Pipeline operations - all prepared but not yet executed bool pipeline_success = true; - pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, pipeline_key1, pipeline_value1); - pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, pipeline_key2, pipeline_value2); - pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, pipeline_key3, pipeline_value3); - pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, pipeline_key1); - pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, pipeline_key2); - + pipeline_success &= couchbase_ops.PipelineRequest( + brpc::CouchbaseOperations::ADD, pipeline_key1, pipeline_value1); + pipeline_success &= couchbase_ops.PipelineRequest( + brpc::CouchbaseOperations::UPSERT, pipeline_key2, pipeline_value2); + pipeline_success &= couchbase_ops.PipelineRequest( + brpc::CouchbaseOperations::ADD, pipeline_key3, pipeline_value3); + pipeline_success &= couchbase_ops.PipelineRequest( + brpc::CouchbaseOperations::GET, pipeline_key1); + pipeline_success &= couchbase_ops.PipelineRequest( + brpc::CouchbaseOperations::GET, pipeline_key2); + if (!pipeline_success) { - std::cout << RED << "Failed to add operations to pipeline" << RESET << std::endl; + std::cout << RED << "Failed to add operations to pipeline" << RESET + << std::endl; couchbase_ops.ClearPipeline(); return -1; } - - std::cout << "Added " << couchbase_ops.GetPipelineSize() << " operations to pipeline" << std::endl; - + + std::cout << "Added " << couchbase_ops.GetPipelineSize() + << " operations to pipeline" << std::endl; + // Execute all operations in a single network call std::cout << "Executing pipeline operations..." << std::endl; - std::vector pipeline_results = couchbase_ops.ExecutePipeline(); - + std::vector pipeline_results = + couchbase_ops.ExecutePipeline(); + // Process results in order - std::cout << GREEN << "Pipeline execution completed. Results:" << RESET << std::endl; + std::cout << GREEN << "Pipeline execution completed. Results:" << RESET + << std::endl; for (size_t i = 0; i < pipeline_results.size(); ++i) { const auto& result = pipeline_results[i]; if (result.success) { if (!result.value.empty()) { - std::cout << GREEN << " Operation " << (i+1) << " SUCCESS - Value: " << result.value << RESET << std::endl; + std::cout << GREEN << " Operation " << (i + 1) + << " SUCCESS - Value: " << result.value << RESET << std::endl; } else { - std::cout << GREEN << " Operation " << (i+1) << " SUCCESS" << RESET << std::endl; + std::cout << GREEN << " Operation " << (i + 1) << " SUCCESS" << RESET + << std::endl; } } else { - std::cout << RED << " Operation " << (i+1) << " FAILED: " << result.error_message << RESET << std::endl; + std::cout << RED << " Operation " << (i + 1) + << " FAILED: " << result.error_message << RESET << std::endl; } } - + // Demonstrate pipeline with collection operations - std::cout << GREEN << "\n=== Pipeline with Collection Operations ===" << RESET << std::endl; - + std::cout << GREEN << "\n=== Pipeline with Collection Operations ===" << RESET + << std::endl; + if (!couchbase_ops.BeginPipeline()) { - std::cout << RED << "Failed to begin collection pipeline" << RESET << std::endl; + std::cout << RED << "Failed to begin collection pipeline" << RESET + << std::endl; return -1; } - + std::string coll_pipeline_key1 = "coll_pipeline::doc1"; std::string coll_pipeline_key2 = "coll_pipeline::doc2"; - std::string coll_pipeline_value1 = R"({"collection_operation": "pipeline_add", "id": 1})"; - std::string coll_pipeline_value2 = R"({"collection_operation": "pipeline_upsert", "id": 2})"; - + std::string coll_pipeline_value1 = + R"({"collection_operation": "pipeline_add", "id": 1})"; + std::string coll_pipeline_value2 = + R"({"collection_operation": "pipeline_upsert", "id": 2})"; + // Add collection-scoped operations to pipeline bool coll_pipeline_success = true; - coll_pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, coll_pipeline_key1, coll_pipeline_value1, collection_name); - coll_pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, coll_pipeline_key2, coll_pipeline_value2, collection_name); - coll_pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, coll_pipeline_key1, "", collection_name); - coll_pipeline_success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, coll_pipeline_key1, "", collection_name); - + coll_pipeline_success &= couchbase_ops.PipelineRequest( + brpc::CouchbaseOperations::ADD, coll_pipeline_key1, coll_pipeline_value1, + collection_name); + coll_pipeline_success &= couchbase_ops.PipelineRequest( + brpc::CouchbaseOperations::UPSERT, coll_pipeline_key2, + coll_pipeline_value2, collection_name); + coll_pipeline_success &= couchbase_ops.PipelineRequest( + brpc::CouchbaseOperations::GET, coll_pipeline_key1, "", collection_name); + coll_pipeline_success &= + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + coll_pipeline_key1, "", collection_name); + if (!coll_pipeline_success) { - std::cout << RED << "Failed to add collection operations to pipeline" << RESET << std::endl; + std::cout << RED << "Failed to add collection operations to pipeline" + << RESET << std::endl; couchbase_ops.ClearPipeline(); return -1; } - + // Execute collection pipeline - std::vector coll_pipeline_results = couchbase_ops.ExecutePipeline(); - - std::cout << GREEN << "Collection pipeline execution completed. Results:" << RESET << std::endl; + std::vector coll_pipeline_results = + couchbase_ops.ExecutePipeline(); + + std::cout << GREEN + << "Collection pipeline execution completed. Results:" << RESET + << std::endl; for (size_t i = 0; i < coll_pipeline_results.size(); ++i) { const auto& result = coll_pipeline_results[i]; if (result.success) { if (!result.value.empty()) { - std::cout << GREEN << " Collection Operation " << (i+1) << " SUCCESS - Value: " << result.value << RESET << std::endl; + std::cout << GREEN << " Collection Operation " << (i + 1) + << " SUCCESS - Value: " << result.value << RESET << std::endl; } else { - std::cout << GREEN << " Collection Operation " << (i+1) << " SUCCESS" << RESET << std::endl; + std::cout << GREEN << " Collection Operation " << (i + 1) << " SUCCESS" + << RESET << std::endl; } } else { - std::cout << RED << " Collection Operation " << (i+1) << " FAILED: " << result.error_message << RESET << std::endl; + std::cout << RED << " Collection Operation " << (i + 1) + << " FAILED: " << result.error_message << RESET << std::endl; } } - + // Clean up remaining pipeline documents std::cout << GREEN << "\n=== Cleanup Pipeline Demo ===" << RESET << std::endl; if (couchbase_ops.BeginPipeline()) { - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key1); - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key2); - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key3); - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, coll_pipeline_key2, "", collection_name); - - std::vector cleanup_results = couchbase_ops.ExecutePipeline(); - std::cout << "Cleanup completed (" << cleanup_results.size() << " operations)" << std::endl; + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + pipeline_key1); + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + pipeline_key2); + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + pipeline_key3); + couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + coll_pipeline_key2, "", collection_name); + + std::vector cleanup_results = + couchbase_ops.ExecutePipeline(); + std::cout << "Cleanup completed (" << cleanup_results.size() + << " operations)" << std::endl; } - - std::cout << GREEN << "\n=== All operations completed successfully! ===" << RESET << std::endl; + + std::cout << GREEN + << "\n=== All operations completed successfully! ===" << RESET + << std::endl; return 0; } diff --git a/example/couchbase_c++/multithreaded_couchbase_client.cpp b/example/couchbase_c++/multithreaded_couchbase_client.cpp index 3438317f9b..1065543af0 100644 --- a/example/couchbase_c++/multithreaded_couchbase_client.cpp +++ b/example/couchbase_c++/multithreaded_couchbase_client.cpp @@ -23,8 +23,8 @@ #include #include -#include #include +#include // ANSI color codes #define GREEN "\033[32m" @@ -36,7 +36,8 @@ DEFINE_string(server, "127.0.0.1:11210", "IP Address of server"); DEFINE_string(connection_type, "single", "Connection type"); -DEFINE_int32(operations_per_thread, 50, "Number of operations each thread should perform"); +DEFINE_int32(operations_per_thread, 50, + "Number of operations each thread should perform"); DEFINE_int32(sleep_ms, 100, "Sleep time between operations in milliseconds"); const int NUM_THREADS = 16; @@ -44,141 +45,155 @@ const int THREADS_PER_BUCKET = 4; // Simple global config struct { - std::string username; - std::string password; - std::vector bucket_names; - std::vector collection_names; + std::string username; + std::string password; + std::vector bucket_names; + std::vector collection_names; } g_config; // Thread arguments struct ThreadArgs { - int thread_id; - int bucket_id; - std::string bucket_name; + int thread_id; + int bucket_id; + std::string bucket_name; }; // Simple operation function using high-level API -bool perform_operation(brpc::CouchbaseOperations& couchbase_ops, const std::string& key, const std::string& collection = "_default") { - // Simple ADD operation - std::string value = butil::string_printf(R"({"thread_id": %llu, "timestamp": %lld})", - (unsigned long long)bthread_self(), butil::gettimeofday_us()); - - brpc::CouchbaseOperations::Result result = couchbase_ops.Add(key, value, collection); - return result.success; +bool perform_operation(brpc::CouchbaseOperations& couchbase_ops, + const std::string& key, + const std::string& collection = "_default") { + // Simple ADD operation + std::string value = butil::string_printf( + R"({"thread_id": %llu, "timestamp": %lld})", + (unsigned long long)bthread_self(), butil::gettimeofday_us()); + + brpc::CouchbaseOperations::Result result = + couchbase_ops.Add(key, value, collection); + return result.success; } // Thread worker function using high-level API void* thread_worker(void* arg) { - ThreadArgs* args = static_cast(arg); - - LOG(INFO) << "Thread " << args->thread_id << " starting on bucket " << args->bucket_name; - - // Create CouchbaseOperations instance for this thread - brpc::CouchbaseOperations couchbase_ops; - - // Authentication using high-level method - brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( - g_config.username, g_config.password, FLAGS_server, false, ""); - if (!auth_result.success) { - LOG(ERROR) << "Thread " << args->thread_id << ": Auth failed - " << auth_result.error_message; - return NULL; + ThreadArgs* args = static_cast(arg); + + LOG(INFO) << "Thread " << args->thread_id << " starting on bucket " + << args->bucket_name; + + // Create CouchbaseOperations instance for this thread + brpc::CouchbaseOperations couchbase_ops; + + // Authentication using high-level method + brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( + g_config.username, g_config.password, FLAGS_server, false, ""); + if (!auth_result.success) { + LOG(ERROR) << "Thread " << args->thread_id << ": Auth failed - " + << auth_result.error_message; + return NULL; + } + + // Select bucket using high-level method + brpc::CouchbaseOperations::Result bucket_result = + couchbase_ops.SelectBucket(args->bucket_name); + if (!bucket_result.success) { + LOG(ERROR) << "Thread " << args->thread_id << ": Bucket selection failed - " + << bucket_result.error_message; + return NULL; + } + + LOG(INFO) << "Thread " << args->thread_id << " connected to bucket " + << args->bucket_name; + + // Perform operations + int success_count = 0; + for (int i = 0; i < FLAGS_operations_per_thread; ++i) { + std::string key = + butil::string_printf("thread_%d_op_%d", args->thread_id, i); + + // Choose collection if available, otherwise use default + std::string collection = "_default"; + if (!g_config.collection_names.empty()) { + collection = + g_config.collection_names[i % g_config.collection_names.size()]; } - - // Select bucket using high-level method - brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.SelectBucket(args->bucket_name); - if (!bucket_result.success) { - LOG(ERROR) << "Thread " << args->thread_id << ": Bucket selection failed - " << bucket_result.error_message; - return NULL; + + if (perform_operation(couchbase_ops, key, collection)) { + success_count++; } - - LOG(INFO) << "Thread " << args->thread_id << " connected to bucket " << args->bucket_name; - - // Perform operations - int success_count = 0; - for (int i = 0; i < FLAGS_operations_per_thread; ++i) { - std::string key = butil::string_printf("thread_%d_op_%d", args->thread_id, i); - - // Choose collection if available, otherwise use default - std::string collection = "_default"; - if (!g_config.collection_names.empty()) { - collection = g_config.collection_names[i % g_config.collection_names.size()]; - } - - if (perform_operation(couchbase_ops, key, collection)) { - success_count++; - } - - if (FLAGS_sleep_ms > 0) { - bthread_usleep(FLAGS_sleep_ms * 1000); - } + + if (FLAGS_sleep_ms > 0) { + bthread_usleep(FLAGS_sleep_ms * 1000); } - - LOG(INFO) << "Thread " << args->thread_id << " completed: " << success_count - << "/" << FLAGS_operations_per_thread << " operations successful"; - - return NULL; + } + + LOG(INFO) << "Thread " << args->thread_id << " completed: " << success_count + << "/" << FLAGS_operations_per_thread << " operations successful"; + + return NULL; } // Simple config function void get_config() { - std::cout << CYAN << "=== Simple Multithreaded Couchbase Client ===" << RESET << std::endl; - - std::cout << "Username: "; - std::cin >> g_config.username; - std::cout << "Password: "; - std::cin >> g_config.password; - - std::cout << "Enter 4 bucket names:" << std::endl; - for (int i = 0; i < 4; ++i) { - std::string bucket; - std::cout << "Bucket " << (i+1) << ": "; - std::cin >> bucket; - g_config.bucket_names.push_back(bucket); - } - - int num_collections; - std::cout << "Number of collections (0 for none): "; - std::cin >> num_collections; - - for (int i = 0; i < num_collections; ++i) { - std::string collection; - std::cout << "Collection " << (i+1) << ": "; - std::cin >> collection; - g_config.collection_names.push_back(collection); - } + std::cout << CYAN << "=== Simple Multithreaded Couchbase Client ===" << RESET + << std::endl; + + std::cout << "Username: "; + std::cin >> g_config.username; + std::cout << "Password: "; + std::cin >> g_config.password; + + std::cout << "Enter 4 bucket names:" << std::endl; + for (int i = 0; i < 4; ++i) { + std::string bucket; + std::cout << "Bucket " << (i + 1) << ": "; + std::cin >> bucket; + g_config.bucket_names.push_back(bucket); + } + + int num_collections; + std::cout << "Number of collections (0 for none): "; + std::cin >> num_collections; + + for (int i = 0; i < num_collections; ++i) { + std::string collection; + std::cout << "Collection " << (i + 1) << ": "; + std::cin >> collection; + g_config.collection_names.push_back(collection); + } } int main(int argc, char* argv[]) { - gflags::ParseCommandLineFlags(&argc, &argv, true); - - std::cout << GREEN << "Starting 16 bthreads (4 per bucket)" << RESET << std::endl; - - get_config(); - - // Create bthreads - std::vector threads(NUM_THREADS); - std::vector args(NUM_THREADS); - - for (int i = 0; i < NUM_THREADS; ++i) { - args[i].thread_id = i; - args[i].bucket_id = i / THREADS_PER_BUCKET; - args[i].bucket_name = g_config.bucket_names[args[i].bucket_id]; - - if (bthread_start_background(&threads[i], NULL, thread_worker, &args[i]) != 0) { - LOG(ERROR) << "Failed to create thread " << i; - return -1; - } - - bthread_usleep(50000); // 50ms delay between thread starts - } - - std::cout << "All 16 threads started!" << std::endl; - - // Wait for all threads - for (int i = 0; i < NUM_THREADS; ++i) { - bthread_join(threads[i], NULL); + gflags::ParseCommandLineFlags(&argc, &argv, true); + + std::cout << GREEN << "Starting 16 bthreads (4 per bucket)" << RESET + << std::endl; + + get_config(); + + // Create bthreads + std::vector threads(NUM_THREADS); + std::vector args(NUM_THREADS); + + for (int i = 0; i < NUM_THREADS; ++i) { + args[i].thread_id = i; + args[i].bucket_id = i / THREADS_PER_BUCKET; + args[i].bucket_name = g_config.bucket_names[args[i].bucket_id]; + + if (bthread_start_background(&threads[i], NULL, thread_worker, &args[i]) != + 0) { + LOG(ERROR) << "Failed to create thread " << i; + return -1; } - - std::cout << GREEN << "All threads completed!" << RESET << std::endl; - return 0; + + bthread_usleep(50000); // 50ms delay between thread starts + } + + std::cout << "All 16 threads started!" << std::endl; + + // Wait for all threads + for (int i = 0; i < NUM_THREADS; ++i) { + bthread_join(threads[i], NULL); + } + + std::cout << GREEN << "All threads completed!" << RESET << std::endl; + return 0; } diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index f5234d25c4..5cdb623564 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -32,89 +32,101 @@ namespace brpc { // Couchbase protocol constants namespace { - [[maybe_unused]] constexpr uint32_t APPLE_VBUCKET_COUNT = 64; - constexpr uint32_t DEFAULT_VBUCKET_COUNT = 1024; - constexpr int CONNECTION_ID_SIZE = 33; - constexpr size_t RANDOM_ID_HEX_SIZE = 67; // 33 bytes * 2 + null terminator -} +[[maybe_unused]] constexpr uint32_t APPLE_VBUCKET_COUNT = 64; +constexpr uint32_t DEFAULT_VBUCKET_COUNT = 1024; +constexpr int CONNECTION_ID_SIZE = 33; +constexpr size_t RANDOM_ID_HEX_SIZE = 67; // 33 bytes * 2 + null terminator +} // namespace // Static member definitions -CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseRequest::metadata_tracking = &common_metadata_tracking; -CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseResponse::metadata_tracking = &common_metadata_tracking; - - -bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest(string server, string bucket, CouchbaseMetadataTracking::CollectionManifest manifest){ +CouchbaseMetadataTracking* + CouchbaseOperations::CouchbaseRequest::metadata_tracking = + &common_metadata_tracking; +CouchbaseMetadataTracking* + CouchbaseOperations::CouchbaseResponse::metadata_tracking = + &common_metadata_tracking; + +bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest( + string server, string bucket, + CouchbaseMetadataTracking::CollectionManifest manifest) { // Then update the collection manifest with proper locking { UniqueLock write_lock(rw_bucket_to_collection_manifest_mutex); bucket_to_collection_manifest[server][bucket] = manifest; } - + return true; } -bool brpc::CouchbaseMetadataTracking::get_bucket_to_collection_manifest(string server, string bucket, CouchbaseMetadataTracking::CollectionManifest *manifest){ +bool brpc::CouchbaseMetadataTracking::get_bucket_to_collection_manifest( + string server, string bucket, + CouchbaseMetadataTracking::CollectionManifest* manifest) { SharedLock read_lock(rw_bucket_to_collection_manifest_mutex); auto it1 = bucket_to_collection_manifest.find(server); - if(it1 == bucket_to_collection_manifest.end()){ + if (it1 == bucket_to_collection_manifest.end()) { return false; } auto it2 = it1->second.find(bucket); - if(it2 == it1->second.end()){ + if (it2 == it1->second.end()) { return false; } *manifest = it2->second; return true; } -bool brpc::CouchbaseMetadataTracking::get_manifest_to_collection_id(CouchbaseMetadataTracking::CollectionManifest *manifest, string scope, string collection, uint8_t *collection_id){ - if(manifest == nullptr || collection_id == nullptr){ +bool brpc::CouchbaseMetadataTracking::get_manifest_to_collection_id( + CouchbaseMetadataTracking::CollectionManifest* manifest, string scope, + string collection, uint8_t* collection_id) { + if (manifest == nullptr || collection_id == nullptr) { LOG(ERROR) << "Invalid input: manifest or collection_id is null"; return false; } auto it1 = manifest->scope_to_collectionID_map.find(scope); - if(it1 == manifest->scope_to_collectionID_map.end()){ + if (it1 == manifest->scope_to_collectionID_map.end()) { LOG(ERROR) << "Scope: " << scope << " not found in manifest"; return false; } auto it2 = it1->second.find(collection); - if(it2 == it1->second.end()){ - LOG(ERROR) << "Collection: " << collection << " not found in scope: " << scope; + if (it2 == it1->second.end()) { + LOG(ERROR) << "Collection: " << collection + << " not found in scope: " << scope; return false; } *collection_id = it2->second; return true; } -bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest(const string& json, CouchbaseMetadataTracking::CollectionManifest *manifest) { - if(manifest == nullptr){ +bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest( + const string& json, + CouchbaseMetadataTracking::CollectionManifest* manifest) { + if (manifest == nullptr) { LOG(ERROR) << "Invalid input: manifest is null"; return false; } - + // Clear existing data manifest->uid.clear(); manifest->scope_to_collectionID_map.clear(); - + if (json.empty()) { LOG(ERROR) << "JSON string is empty"; return false; } - + // Parse JSON using RapidJSON BUTIL_RAPIDJSON_NAMESPACE::Document document; document.Parse(json.c_str()); - + if (document.HasParseError()) { LOG(ERROR) << "Failed to parse JSON: " << document.GetParseError(); return false; } - + if (!document.IsObject()) { LOG(ERROR) << "JSON root is not an object"; return false; } - + // Extract uid if (document.HasMember("uid") && document["uid"].IsString()) { manifest->uid = document["uid"].GetString(); @@ -122,87 +134,97 @@ bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest(const string& LOG(ERROR) << "Missing or invalid 'uid' field in JSON"; return false; } - + // Extract scopes if (!document.HasMember("scopes") || !document["scopes"].IsArray()) { LOG(ERROR) << "Missing or invalid 'scopes' field in JSON"; return false; } - + const BUTIL_RAPIDJSON_NAMESPACE::Value& scopes = document["scopes"]; for (BUTIL_RAPIDJSON_NAMESPACE::SizeType i = 0; i < scopes.Size(); ++i) { const BUTIL_RAPIDJSON_NAMESPACE::Value& scope = scopes[i]; - + if (!scope.IsObject()) { LOG(ERROR) << "Scope at index " << i << " is not an object"; return false; } - + // Extract scope name if (!scope.HasMember("name") || !scope["name"].IsString()) { LOG(ERROR) << "Missing or invalid 'name' field in scope at index " << i; return false; } string scope_name = scope["name"].GetString(); - + // Extract collections if (!scope.HasMember("collections") || !scope["collections"].IsArray()) { - LOG(ERROR) << "Missing or invalid 'collections' field in scope '" << scope_name << "'"; + LOG(ERROR) << "Missing or invalid 'collections' field in scope '" + << scope_name << "'"; return false; } - + const BUTIL_RAPIDJSON_NAMESPACE::Value& collections = scope["collections"]; unordered_map collection_map; - - for (BUTIL_RAPIDJSON_NAMESPACE::SizeType j = 0; j < collections.Size(); ++j) { + + for (BUTIL_RAPIDJSON_NAMESPACE::SizeType j = 0; j < collections.Size(); + ++j) { const BUTIL_RAPIDJSON_NAMESPACE::Value& collection = collections[j]; - + if (!collection.IsObject()) { - LOG(ERROR) << "Collection at index " << j << " in scope '" << scope_name << "' is not an object"; + LOG(ERROR) << "Collection at index " << j << " in scope '" << scope_name + << "' is not an object"; return false; } - + // Extract collection name if (!collection.HasMember("name") || !collection["name"].IsString()) { - LOG(ERROR) << "Missing or invalid 'name' field in collection at index " << j << " in scope '" << scope_name << "'"; + LOG(ERROR) << "Missing or invalid 'name' field in collection at index " + << j << " in scope '" << scope_name << "'"; return false; } string collection_name = collection["name"].GetString(); - + // Extract collection uid (hex string) if (!collection.HasMember("uid") || !collection["uid"].IsString()) { - LOG(ERROR) << "Missing or invalid 'uid' field in collection '" << collection_name << "' in scope '" << scope_name << "'"; + LOG(ERROR) << "Missing or invalid 'uid' field in collection '" + << collection_name << "' in scope '" << scope_name << "'"; return false; } string collection_uid_str = collection["uid"].GetString(); - + // Convert hex string to uint8_t uint8_t collection_id = 0; try { // Convert hex string to integer unsigned long uid_val = std::stoul(collection_uid_str, nullptr, 16); if (uid_val > 255) { - LOG(ERROR) << "Collection uid '" << collection_uid_str << "' exceeds uint8_t range in collection '" << collection_name << "' in scope '" << scope_name << "'"; + LOG(ERROR) << "Collection uid '" << collection_uid_str + << "' exceeds uint8_t range in collection '" + << collection_name << "' in scope '" << scope_name << "'"; return false; } collection_id = static_cast(uid_val); } catch (const std::exception& e) { - LOG(ERROR) << "Failed to parse collection uid '" << collection_uid_str << "' as hex in collection '" << collection_name << "' in scope '" << scope_name << "': " << e.what(); + LOG(ERROR) << "Failed to parse collection uid '" << collection_uid_str + << "' as hex in collection '" << collection_name + << "' in scope '" << scope_name << "': " << e.what(); return false; } - + // Add to collection map collection_map[collection_name] = collection_id; } - + // Add scope and its collections to manifest manifest->scope_to_collectionID_map[scope_name] = std::move(collection_map); } - + return true; } -uint32_t CouchbaseOperations::CouchbaseRequest::hash_crc32(const char* key, size_t key_length) { +uint32_t CouchbaseOperations::CouchbaseRequest::hash_crc32(const char* key, + size_t key_length) { static const uint32_t crc32tab[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, @@ -269,7 +291,9 @@ void CouchbaseOperations::CouchbaseRequest::SharedCtor() { void CouchbaseOperations::CouchbaseRequest::SharedDtor() {} -void CouchbaseOperations::CouchbaseRequest::SetCachedSize(int size) const { _cached_size_ = size; } +void CouchbaseOperations::CouchbaseRequest::SetCachedSize(int size) const { + _cached_size_ = size; +} void CouchbaseOperations::CouchbaseRequest::Clear() { _buf.clear(); @@ -278,7 +302,8 @@ void CouchbaseOperations::CouchbaseRequest::Clear() { // Support for scope level collections will be added in future. // Get the Scope ID for a given scope name -// bool CouchbaseOperations::CouchbaseRequest::GetScopeId(const butil::StringPiece& scope_name) { +// bool CouchbaseOperations::CouchbaseRequest::GetScopeId(const +// butil::StringPiece& scope_name) { // if (scope_name.empty()) { // LOG(ERROR) << "Empty scope name"; // return false; @@ -305,7 +330,8 @@ void CouchbaseOperations::CouchbaseRequest::Clear() { // return true; // } -bool CouchbaseOperations::CouchbaseRequest::SelectBucketRequest(const butil::StringPiece& bucket_name) { +bool CouchbaseOperations::CouchbaseRequest::SelectBucketRequest( + const butil::StringPiece& bucket_name) { if (bucket_name.empty()) { LOG(ERROR) << "Empty bucket name"; return false; @@ -359,7 +385,8 @@ bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { // Generate a random connection ID as hex string unsigned char raw_id[CONNECTION_ID_SIZE]; FILE* urandom = fopen("/dev/urandom", "rb"); - if (!urandom || fread(raw_id, 1, CONNECTION_ID_SIZE, urandom) != CONNECTION_ID_SIZE) { + if (!urandom || + fread(raw_id, 1, CONNECTION_ID_SIZE, urandom) != CONNECTION_ID_SIZE) { if (urandom) fclose(urandom); LOG(ERROR) << "Failed to generate random connection id"; return false; @@ -414,8 +441,8 @@ bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { return true; } -bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest(const butil::StringPiece& username, - const butil::StringPiece& password) { +bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest( + const butil::StringPiece& username, const butil::StringPiece& password) { if (username.empty() || password.empty()) { LOG(ERROR) << "Empty username or password"; return false; @@ -441,7 +468,7 @@ bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest(const butil::Str 0, 0}; std::string auth_str; - auth_str.reserve(sizeof(header) + sizeof(kPlainAuthCommand) - 1 + + auth_str.reserve(sizeof(header) + sizeof(kPlainAuthCommand) - 1 + username.size() * 2 + password.size() + 2); auth_str.append(reinterpret_cast(&header), sizeof(header)); auth_str.append(kPlainAuthCommand, sizeof(kPlainAuthCommand) - 1); @@ -513,13 +540,16 @@ size_t CouchbaseOperations::CouchbaseRequest::ByteSizeLong() const { return total_size; } -void CouchbaseOperations::CouchbaseRequest::MergeFrom(const CouchbaseRequest& from) { +void CouchbaseOperations::CouchbaseRequest::MergeFrom( + const CouchbaseRequest& from) { CHECK_NE(&from, this); _buf.append(from._buf); _pipelined_count += from._pipelined_count; } -bool CouchbaseOperations::CouchbaseRequest::IsInitialized() const { return _pipelined_count != 0; } +bool CouchbaseOperations::CouchbaseRequest::IsInitialized() const { + return _pipelined_count != 0; +} void CouchbaseOperations::CouchbaseRequest::Swap(CouchbaseRequest* other) { if (other != this) { @@ -529,7 +559,8 @@ void CouchbaseOperations::CouchbaseRequest::Swap(CouchbaseRequest* other) { } } -::google::protobuf::Metadata CouchbaseOperations::CouchbaseRequest::GetMetadata() const { +::google::protobuf::Metadata +CouchbaseOperations::CouchbaseRequest::GetMetadata() const { ::google::protobuf::Metadata metadata{}; metadata.descriptor = CouchbaseRequestBase::descriptor(); metadata.reflection = nullptr; @@ -538,10 +569,11 @@ ::google::protobuf::Metadata CouchbaseOperations::CouchbaseRequest::GetMetadata( void CouchbaseOperations::CouchbaseResponse::SharedCtor() { _cached_size_ = 0; } - void CouchbaseOperations::CouchbaseResponse::SharedDtor() {} -void CouchbaseOperations::CouchbaseResponse::SetCachedSize(int size) const { _cached_size_ = size; } +void CouchbaseOperations::CouchbaseResponse::SetCachedSize(int size) const { + _cached_size_ = size; +} void CouchbaseOperations::CouchbaseResponse::Clear() {} @@ -578,13 +610,16 @@ size_t CouchbaseOperations::CouchbaseResponse::ByteSizeLong() const { return total_size; } -void CouchbaseOperations::CouchbaseResponse::MergeFrom(const CouchbaseResponse& from) { +void CouchbaseOperations::CouchbaseResponse::MergeFrom( + const CouchbaseResponse& from) { CHECK_NE(&from, this); _err = from._err; _buf.append(from._buf); } -bool CouchbaseOperations::CouchbaseResponse::IsInitialized() const { return !_buf.empty(); } +bool CouchbaseOperations::CouchbaseResponse::IsInitialized() const { + return !_buf.empty(); +} void CouchbaseOperations::CouchbaseResponse::Swap(CouchbaseResponse* other) { if (other != this) { @@ -593,7 +628,8 @@ void CouchbaseOperations::CouchbaseResponse::Swap(CouchbaseResponse* other) { } } -::google::protobuf::Metadata CouchbaseOperations::CouchbaseResponse::GetMetadata() const { +::google::protobuf::Metadata +CouchbaseOperations::CouchbaseResponse::GetMetadata() const { ::google::protobuf::Metadata metadata{}; metadata.descriptor = CouchbaseResponseBase::descriptor(); metadata.reflection = nullptr; @@ -738,9 +774,8 @@ std::string CouchbaseOperations::CouchbaseResponse::format_error_message( // MUST NOT have extras. // MUST have key. // MUST NOT have value. -bool CouchbaseOperations::CouchbaseRequest::GetOrDelete(uint8_t command, - const butil::StringPiece& key, - uint8_t coll_id) { +bool CouchbaseOperations::CouchbaseRequest::GetOrDelete( + uint8_t command, const butil::StringPiece& key, uint8_t coll_id) { // Collection ID uint8_t collection_id = coll_id; uint16_t VBucket_id = hash_crc32(key.data(), key.size()); @@ -769,85 +804,103 @@ bool CouchbaseOperations::CouchbaseRequest::GetOrDelete(uint8_t command, return true; } -bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, brpc::CouchbaseMetadataTracking *metadata_tracking, - brpc::Channel* channel, const string& server, const string& selected_bucket){ - if(collection_name.empty()){ +bool get_cached_or_fetch_collection_id( + string collection_name, uint8_t* coll_id, + brpc::CouchbaseMetadataTracking* metadata_tracking, brpc::Channel* channel, + const string& server, const string& selected_bucket) { + if (collection_name.empty()) { LOG(ERROR) << "Empty collection name"; return false; } - - if(channel == nullptr){ + + if (channel == nullptr) { LOG(ERROR) << "No channel found, make sure to call Authenticate() first"; return false; } - if(server.empty()){ + if (server.empty()) { LOG(ERROR) << "Server is empty, make sure to call Authenticate() first"; return false; } - if(selected_bucket.empty()){ + if (selected_bucket.empty()) { LOG(ERROR) << "No bucket selected, make sure to call SelectBucket() first"; return false; } - + brpc::CouchbaseMetadataTracking::CollectionManifest manifest; - if(!metadata_tracking->get_bucket_to_collection_manifest(server, selected_bucket, &manifest)){ - LOG(INFO) << "No cached collection manifest found for bucket " << selected_bucket << " on server " << server << ", fetching from server"; + if (!metadata_tracking->get_bucket_to_collection_manifest( + server, selected_bucket, &manifest)) { + LOG(INFO) << "No cached collection manifest found for bucket " + << selected_bucket << " on server " << server + << ", fetching from server"; // No cached manifest found, fetch from server CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; CouchbaseOperations::CouchbaseResponse temp_get_manifest_response; brpc::Controller temp_cntl; temp_get_manifest_request.GetCollectionManifest(); channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, - &temp_get_manifest_response, NULL); + &temp_get_manifest_response, NULL); if (temp_cntl.Failed()) { - LOG(ERROR) << "Failed to get collection manifest for bucket " << selected_bucket << " on server " << server - << ": " << temp_cntl.ErrorText(); + LOG(ERROR) << "Failed to get collection manifest for bucket " + << selected_bucket << " on server " << server << ": " + << temp_cntl.ErrorText(); return false; } string manifest_json; if (!temp_get_manifest_response.PopManifest(&manifest_json)) { - LOG(ERROR) << "Failed to parse response for collection Manifest in bucket " << selected_bucket << " on server " << server - << ": " << temp_get_manifest_response.LastError(); + LOG(ERROR) + << "Failed to parse response for collection Manifest in bucket " + << selected_bucket << " on server " << server << ": " + << temp_get_manifest_response.LastError(); return false; - } - else{ + } else { // convert JSON to manifest structure - if(!metadata_tracking->json_to_collection_manifest(manifest_json, &manifest)){ - LOG(ERROR) << "Failed to parse collection manifest JSON for bucket " << selected_bucket << " on server " << server; + if (!metadata_tracking->json_to_collection_manifest(manifest_json, + &manifest)) { + LOG(ERROR) << "Failed to parse collection manifest JSON for bucket " + << selected_bucket << " on server " << server; return false; } // Cache the collection manifest - if(!metadata_tracking->set_bucket_to_collection_manifest(server, selected_bucket, manifest)){ - LOG(ERROR) << "Failed to cache collection ID for collection " << collection_name << " in bucket " << selected_bucket << " on server " << server; + if (!metadata_tracking->set_bucket_to_collection_manifest( + server, selected_bucket, manifest)) { + LOG(ERROR) << "Failed to cache collection ID for collection " + << collection_name << " in bucket " << selected_bucket + << " on server " << server; return false; } return true; - } - } - else{ - if(!metadata_tracking->get_manifest_to_collection_id(&manifest, "_default", collection_name, coll_id)){ + } + } else { + if (!metadata_tracking->get_manifest_to_collection_id( + &manifest, "_default", collection_name, coll_id)) { return false; } return true; } } -bool CouchbaseOperations::CouchbaseRequest::GetRequest(const butil::StringPiece& key, string collection_name, - brpc::Channel* channel, const string& server, const string& bucket) { - uint8_t coll_id = 0; // default collection ID - if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ +bool CouchbaseOperations::CouchbaseRequest::GetRequest( + const butil::StringPiece& key, string collection_name, + brpc::Channel* channel, const string& server, const string& bucket) { + uint8_t coll_id = 0; // default collection ID + if (collection_name != "_default") { + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } return GetOrDelete(policy::CB_BINARY_GET, key, coll_id); } -bool CouchbaseOperations::CouchbaseRequest::DeleteRequest(const butil::StringPiece& key, string collection_name, - brpc::Channel* channel, const string& server, const string& bucket) { - uint8_t coll_id = 0; // default collection ID - if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ +bool CouchbaseOperations::CouchbaseRequest::DeleteRequest( + const butil::StringPiece& key, string collection_name, + brpc::Channel* channel, const string& server, const string& bucket) { + uint8_t coll_id = 0; // default collection ID + if (collection_name != "_default") { + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } @@ -876,7 +929,8 @@ BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); // const uint8_t FLUSH_EXTRAS = (timeout == 0 ? 0 : 4); // FlushHeaderWithExtras header_with_extras = { // {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_FLUSH, 0, FLUSH_EXTRAS, -// policy::CB_BINARY_RAW_BYTES, 0, butil::HostToNet32(FLUSH_EXTRAS), 0, 0}, +// policy::CB_BINARY_RAW_BYTES, 0, butil::HostToNet32(FLUSH_EXTRAS), 0, +// 0}, // butil::HostToNet32(timeout)}; // if (FLUSH_EXTRAS == 0) { // if (_buf.append(&header_with_extras.header, @@ -904,8 +958,9 @@ BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); // 0| Flags | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, uint32_t* flags, - uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, + uint32_t* flags, + uint64_t* cas_value) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -976,8 +1031,9 @@ bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, uint32_ return true; } -bool CouchbaseOperations::CouchbaseResponse::PopGet(std::string* value, uint32_t* flags, - uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopGet(std::string* value, + uint32_t* flags, + uint64_t* cas_value) { butil::IOBuf tmp; if (PopGet(&tmp, flags, cas_value)) { tmp.copy_to(value); @@ -1018,10 +1074,10 @@ const size_t STORE_EXTRAS = // 4| Expiration | // +---------------+---------------+---------------+---------------+ // Total 8 bytes -bool CouchbaseOperations::CouchbaseRequest::Store(uint8_t command, const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value, - uint8_t coll_id) { +bool CouchbaseOperations::CouchbaseRequest::Store( + uint8_t command, const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, uint32_t exptime, + uint64_t cas_value, uint8_t coll_id) { // add collection id // uint16_t collection_id = 0x00; uint8_t collection_id = coll_id; @@ -1057,7 +1113,8 @@ bool CouchbaseOperations::CouchbaseRequest::Store(uint8_t command, const butil:: // MUST NOT have extras // MUST NOT have key // MUST NOT have value -bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, + uint64_t* cas_value) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1102,7 +1159,9 @@ bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, uint64_t* return true; } -const char* CouchbaseOperations::CouchbaseResponse::couchbase_binary_command_to_string(uint8_t cmd) { +const char* +CouchbaseOperations::CouchbaseResponse::couchbase_binary_command_to_string( + uint8_t cmd) { switch (cmd) { case 0x1f: return "CB_HELLO_SELECT_FEATURES"; @@ -1191,14 +1250,16 @@ const char* CouchbaseOperations::CouchbaseResponse::couchbase_binary_command_to_ } } -bool CouchbaseOperations::CouchbaseRequest::UpsertRequest(const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value, - string collection_name, - brpc::Channel* channel, const string& server, const string& bucket) { - uint8_t coll_id = 0; // default collection ID - if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ +bool CouchbaseOperations::CouchbaseRequest::UpsertRequest( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name, brpc::Channel* channel, const string& server, + const string& bucket) { + uint8_t coll_id = 0; // default collection ID + if (collection_name != "_default") { + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } @@ -1255,7 +1316,7 @@ bool CouchbaseOperations::CouchbaseRequest::GetCollectionManifest() { } // bool RefreshCollectionManifest(brpc::Channel* channel) { -// // first fetch the manifest +// // first fetch the manifest // // then compare the UID with the cached one // CouchbaseRequest temp_get_manifest_request; // CouchbaseResponse temp_get_manifest_response; @@ -1264,94 +1325,117 @@ bool CouchbaseOperations::CouchbaseRequest::GetCollectionManifest() { // channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, // &temp_get_manifest_response, NULL); // if (temp_cntl.Failed()) { -// LOG(ERROR) << "Failed to get collection manifest: " << temp_cntl.ErrorText(); -// return false; +// LOG(ERROR) << "Failed to get collection manifest: " << +// temp_cntl.ErrorText(); return false; // } // string manifest_json; // if (!temp_get_manifest_response.PopManifest(&manifest_json)) { -// LOG(ERROR) << "Failed to parse response for collection Manifest: " << temp_get_manifest_response.LastError(); -// return false; +// LOG(ERROR) << "Failed to parse response for collection Manifest: " << +// temp_get_manifest_response.LastError(); return false; // } // // Compare the UID with the cached one // // If they are different, refresh the cache // brpc::CouchbaseMetadataTracking::CollectionManifest manifest; -// if(!common_metadata_tracking.json_to_collection_manifest(manifest_json, &manifest)){ +// if(!common_metadata_tracking.json_to_collection_manifest(manifest_json, +// &manifest)){ // LOG(ERROR) << "Failed to parse collection manifest JSON"; // return false; // } // brpc::CouchbaseMetadataTracking::ChannelInfo temp_channel_info; -// common_metadata_tracking.get_channel_info_for_thread(bthread_self(), &temp_channel_info); -// if(temp_channel_info.server.empty() || temp_channel_info.selected_bucket.empty()){ -// LOG(ERROR) << "No channel info found for this thread, make sure to call Authenticate() and SelectBucket() first"; -// return false; +// common_metadata_tracking.get_channel_info_for_thread(bthread_self(), +// &temp_channel_info); if(temp_channel_info.server.empty() || +// temp_channel_info.selected_bucket.empty()){ +// LOG(ERROR) << "No channel info found for this thread, make sure to call +// Authenticate() and SelectBucket() first"; return false; // } // brpc::CouchbaseMetadataTracking::CollectionManifest cached_manifest; -// if(!common_metadata_tracking.get_bucket_to_collection_manifest(bthread_self(), temp_channel_info.server, temp_channel_info.selected_bucket, &cached_manifest)){ +// if(!common_metadata_tracking.get_bucket_to_collection_manifest(bthread_self(), +// temp_channel_info.server, temp_channel_info.selected_bucket, +// &cached_manifest)){ // // No cached manifest found, set the new one -// if(!common_metadata_tracking.set_bucket_to_collection_manifest(bthread_self(), manifest)){ -// LOG(ERROR) << "Failed to cache collection manifest for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; -// return false; +// if(!common_metadata_tracking.set_bucket_to_collection_manifest(bthread_self(), +// manifest)){ +// LOG(ERROR) << "Failed to cache collection manifest for bucket " << +// temp_channel_info.selected_bucket << " on server " << +// temp_channel_info.server; return false; // } -// LOG(INFO) << "Cached collection manifest for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; -// return true; +// LOG(INFO) << "Cached collection manifest for bucket " << +// temp_channel_info.selected_bucket << " on server " << +// temp_channel_info.server; return true; // } // if(manifest.uid != cached_manifest.uid) { -// LOG(INFO) << "Collection manifest has changed for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; -// if(!common_metadata_tracking.set_bucket_to_collection_manifest(bthread_self(), manifest)){ -// LOG(ERROR) << "Failed to update cached collection manifest for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; -// return false; +// LOG(INFO) << "Collection manifest has changed for bucket " << +// temp_channel_info.selected_bucket << " on server " << +// temp_channel_info.server; +// if(!common_metadata_tracking.set_bucket_to_collection_manifest(bthread_self(), +// manifest)){ +// LOG(ERROR) << "Failed to update cached collection manifest for bucket " +// << temp_channel_info.selected_bucket << " on server " << +// temp_channel_info.server; return false; // } -// LOG(INFO) << "Updated cached collection manifest for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; +// LOG(INFO) << "Updated cached collection manifest for bucket " << +// temp_channel_info.selected_bucket << " on server " << +// temp_channel_info.server; // } // else{ -// LOG(INFO) << "Collection manifest is up-to-date for bucket " << temp_channel_info.selected_bucket << " on server " << temp_channel_info.server; +// LOG(INFO) << "Collection manifest is up-to-date for bucket " << +// temp_channel_info.selected_bucket << " on server " << +// temp_channel_info.server; // } // return true; // } -bool CouchbaseOperations::CouchbaseRequest::AddRequest(const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value, - string collection_name, - brpc::Channel* channel, const string& server, const string& bucket) { - uint8_t coll_id = 0; // default collection ID - if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ +bool CouchbaseOperations::CouchbaseRequest::AddRequest( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name, brpc::Channel* channel, const string& server, + const string& bucket) { + uint8_t coll_id = 0; // default collection ID + if (collection_name != "_default") { + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; - } + } } return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, coll_id); } // Warning: Not tested -// bool CouchbaseOperations::CouchbaseRequest::ReplaceRequest(const butil::StringPiece& key, -// const butil::StringPiece& value, uint32_t flags, -// uint32_t exptime, uint64_t cas_value, +// bool CouchbaseOperations::CouchbaseRequest::ReplaceRequest(const +// butil::StringPiece& key, +// const butil::StringPiece& value, uint32_t +// flags, uint32_t exptime, uint64_t cas_value, // string collection_name, -// brpc::Channel* channel, const string& server, const string& bucket) { +// brpc::Channel* channel, const string& server, +// const string& bucket) { // uint8_t coll_id = 0; // default collection ID // if(collection_name != "_default"){ -// if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ +// if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, +// metadata_tracking, channel, server, bucket)){ // return false; // } // } -// return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, cas_value, +// return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, +// cas_value, // coll_id); // } -bool CouchbaseOperations::CouchbaseRequest::AppendRequest(const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value, - string collection_name, - brpc::Channel* channel, const string& server, const string& bucket) { +bool CouchbaseOperations::CouchbaseRequest::AppendRequest( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name, brpc::Channel* channel, const string& server, + const string& bucket) { if (value.empty()) { LOG(ERROR) << "value to append must be non-empty"; return false; } - uint8_t coll_id = 0; // default collection ID - if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ + uint8_t coll_id = 0; // default collection ID + if (collection_name != "_default") { + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } @@ -1359,18 +1443,20 @@ bool CouchbaseOperations::CouchbaseRequest::AppendRequest(const butil::StringPie coll_id); } -bool CouchbaseOperations::CouchbaseRequest::PrependRequest(const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, - uint32_t exptime, uint64_t cas_value, - string collection_name, - brpc::Channel* channel, const string& server, const string& bucket) { +bool CouchbaseOperations::CouchbaseRequest::PrependRequest( + const butil::StringPiece& key, const butil::StringPiece& value, + uint32_t flags, uint32_t exptime, uint64_t cas_value, + string collection_name, brpc::Channel* channel, const string& server, + const string& bucket) { if (value.empty()) { LOG(ERROR) << "value to prepend must be non-empty"; return false; } - uint8_t coll_id = 0; // default collection ID - if(collection_name != "_default"){ - if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)){ + uint8_t coll_id = 0; // default collection ID + if (collection_name != "_default") { + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } @@ -1385,7 +1471,8 @@ bool CouchbaseOperations::CouchbaseResponse::PopAdd(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_ADD, cas_value); } // Warning: Not tested -// bool CouchbaseOperations::CouchbaseResponse::PopReplace(uint64_t* cas_value) { +// bool CouchbaseOperations::CouchbaseResponse::PopReplace(uint64_t* cas_value) +// { // return PopStore(policy::CB_BINARY_REPLACE, cas_value); // } bool CouchbaseOperations::CouchbaseResponse::PopAppend(uint64_t* cas_value) { @@ -1394,16 +1481,19 @@ bool CouchbaseOperations::CouchbaseResponse::PopAppend(uint64_t* cas_value) { bool CouchbaseOperations::CouchbaseResponse::PopPrepend(uint64_t* cas_value) { return PopStore(policy::CB_BINARY_PREPEND, cas_value); } -bool CouchbaseOperations::CouchbaseResponse::PopSelectBucket(uint64_t* cas_value, std::string bucket_name) { - if(PopStore(policy::CB_SELECT_BUCKET, cas_value) == false){ +bool CouchbaseOperations::CouchbaseResponse::PopSelectBucket( + uint64_t* cas_value, std::string bucket_name) { + if (PopStore(policy::CB_SELECT_BUCKET, cas_value) == false) { LOG(ERROR) << "Failed to select bucket: " << _err; return false; } - // Note: Bucket tracking is now handled at CouchbaseOperations level, not per-thread + // Note: Bucket tracking is now handled at CouchbaseOperations level, not + // per-thread return true; } // Collection-related response method -bool CouchbaseOperations::CouchbaseResponse::PopCollectionId(uint8_t* collection_id) { +bool CouchbaseOperations::CouchbaseResponse::PopCollectionId( + uint8_t* collection_id) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1473,7 +1563,8 @@ bool CouchbaseOperations::CouchbaseResponse::PopCollectionId(uint8_t* collection return true; } -bool CouchbaseOperations::CouchbaseResponse::PopManifest(std::string* manifest_json) { +bool CouchbaseOperations::CouchbaseResponse::PopManifest( + std::string* manifest_json) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1495,10 +1586,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopManifest(std::string* manifest_j if (header.status != 0) { // handle error case - if(header.extras_length != 0){ + if (header.extras_length != 0) { LOG(ERROR) << "Get Collections Manifest response must not have extras"; } - if(header.key_length != 0){ + if (header.key_length != 0) { LOG(ERROR) << "Get Collections Manifest response must not have key"; } _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); @@ -1508,7 +1599,8 @@ bool CouchbaseOperations::CouchbaseResponse::PopManifest(std::string* manifest_j if (value_size > 0) { std::string err_msg; _buf.cutn(&err_msg, value_size); - _err = format_error_message(header.status, "Get Collections Manifest", err_msg); + _err = format_error_message(header.status, "Get Collections Manifest", + err_msg); } else { _err = format_error_message(header.status, "Get Collections Manifest"); } @@ -1516,7 +1608,8 @@ bool CouchbaseOperations::CouchbaseResponse::PopManifest(std::string* manifest_j } // Success case: the manifest should be in the value section - size_t value_size = header.total_body_length - header.extras_length - header.key_length; + size_t value_size = + header.total_body_length - header.extras_length - header.key_length; if (value_size == 0) { butil::string_printf(&_err, "No manifest data in response"); _buf.pop_front(sizeof(header) + header.total_body_length); @@ -1528,7 +1621,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopManifest(std::string* manifest_j // Read the manifest JSON from the value section _buf.cutn(manifest_json, value_size); - + _err.clear(); return true; } @@ -1561,9 +1654,9 @@ const size_t INCR_EXTRAS = // 16| Expiration | // +---------------+---------------+---------------+---------------+ // Total 20 bytes -bool CouchbaseOperations::CouchbaseRequest::Counter(uint8_t command, const butil::StringPiece& key, - uint64_t delta, uint64_t initial_value, - uint32_t exptime) { +bool CouchbaseOperations::CouchbaseRequest::Counter( + uint8_t command, const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime) { IncrHeaderWithExtras header_with_extras = { {policy::CB_MAGIC_REQUEST, command, butil::HostToNet16(key.size()), INCR_EXTRAS, policy::CB_BINARY_RAW_BYTES, 0, @@ -1582,21 +1675,27 @@ bool CouchbaseOperations::CouchbaseRequest::Counter(uint8_t command, const butil } // Warning: Not tested -// bool CouchbaseOperations::CouchbaseRequest::IncrementRequest(const butil::StringPiece& key, uint64_t delta, +// bool CouchbaseOperations::CouchbaseRequest::IncrementRequest(const +// butil::StringPiece& key, uint64_t delta, // uint64_t initial_value, uint32_t exptime, // string collection_name, -// brpc::Channel* channel, const string& server, const string& bucket) { -// // Note: Counter method doesn't seem to use collection_name, may need to be updated if collection support is needed -// return Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, +// brpc::Channel* channel, const string& +// server, const string& bucket) { +// // Note: Counter method doesn't seem to use collection_name, may need to be +// updated if collection support is needed return +// Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, // exptime); // } -// bool CouchbaseOperations::CouchbaseRequest::DecrementRequest(const butil::StringPiece& key, uint64_t delta, +// bool CouchbaseOperations::CouchbaseRequest::DecrementRequest(const +// butil::StringPiece& key, uint64_t delta, // uint64_t initial_value, uint32_t exptime, // string collection_name, -// brpc::Channel* channel, const string& server, const string& bucket) { -// // Note: Counter method doesn't seem to use collection_name, may need to be updated if collection support is needed -// return Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, +// brpc::Channel* channel, const string& +// server, const string& bucket) { +// // Note: Counter method doesn't seem to use collection_name, may need to be +// updated if collection support is needed return +// Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, // exptime); // } @@ -1611,8 +1710,9 @@ bool CouchbaseOperations::CouchbaseRequest::Counter(uint8_t command, const butil // | | // +---------------+---------------+---------------+---------------+ // Total 8 bytes -bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, uint64_t* new_value, - uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, + uint64_t* new_value, + uint64_t* cas_value) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1668,10 +1768,12 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, uint64_ } // Warning: Not tested -// bool CouchbaseOperations::CouchbaseResponse::PopIncrement(uint64_t* new_value, uint64_t* cas_value) { +// bool CouchbaseOperations::CouchbaseResponse::PopIncrement(uint64_t* +// new_value, uint64_t* cas_value) { // return PopCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); // } -// bool CouchbaseOperations::CouchbaseResponse::PopDecrement(uint64_t* new_value, uint64_t* cas_value) { +// bool CouchbaseOperations::CouchbaseResponse::PopDecrement(uint64_t* +// new_value, uint64_t* cas_value) { // return PopCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); // } @@ -1698,9 +1800,11 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, uint64_ // +---------------+---------------+---------------+---------------+ // Total 4 bytes // Warning: Not tested -// bool CouchbaseOperations::CouchbaseRequest::TouchRequest(const butil::StringPiece& key, uint32_t exptime, +// bool CouchbaseOperations::CouchbaseRequest::TouchRequest(const +// butil::StringPiece& key, uint32_t exptime, // string collection_name, -// brpc::Channel* channel, const string& server, const string& bucket) { +// brpc::Channel* channel, const string& server, +// const string& bucket) { // TouchHeaderWithExtras header_with_extras = { // {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_TOUCH, // butil::HostToNet16(key.size()), TOUCH_EXTRAS, @@ -1791,13 +1895,16 @@ bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { return true; } -CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, string collection_name) { - //create CouchbaseRequest and CouchbaseResponse objects and then using the channel which is created for this thread in authenticate() use it to call() +CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, + string collection_name) { + // create CouchbaseRequest and CouchbaseResponse objects and then using the + // channel which is created for this thread in authenticate() use it to call() CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.GetRequest(key, collection_name, channel, server_address, selected_bucket) == false){ + if (request.GetRequest(key, collection_name, channel, server_address, + selected_bucket) == false) { LOG(ERROR) << "Failed to create Get request for key: " << key; result.success = false; result.value = ""; @@ -1805,7 +1912,8 @@ CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, string c } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to get key: " << key << " from Couchbase: " << cntl.ErrorText(); + LOG(ERROR) << "Failed to get key: " << key + << " from Couchbase: " << cntl.ErrorText(); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); @@ -1814,7 +1922,7 @@ CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, string c string value; uint32_t flags = 0; uint64_t cas = 0; - if(response.PopGet(&value, &flags, &cas) == false){ + if (response.PopGet(&value, &flags, &cas) == false) { result.success = false; result.value = ""; result.error_message = response.LastError(); @@ -1826,12 +1934,14 @@ CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, string c return result; } -CouchbaseOperations::Result CouchbaseOperations::Upsert(const string& key, const string& value, string collection_name) { +CouchbaseOperations::Result CouchbaseOperations::Upsert( + const string& key, const string& value, string collection_name) { CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ + if (request.UpsertRequest(key, value, 0, 0, 0, collection_name, channel, + server_address, selected_bucket) == false) { LOG(ERROR) << "Failed to create Upsert request for key: " << key; result.success = false; result.value = ""; @@ -1839,13 +1949,14 @@ CouchbaseOperations::Result CouchbaseOperations::Upsert(const string& key, const } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to upsert key: " << key << " to Couchbase: " << cntl.ErrorText(); + LOG(ERROR) << "Failed to upsert key: " << key + << " to Couchbase: " << cntl.ErrorText(); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); return result; } - if(response.PopUpsert(NULL) == false){ + if (response.PopUpsert(NULL) == false) { result.success = false; result.value = ""; result.error_message = response.LastError(); @@ -1857,12 +1968,14 @@ CouchbaseOperations::Result CouchbaseOperations::Upsert(const string& key, const return result; } -CouchbaseOperations::Result CouchbaseOperations::Delete(const string& key, string collection_name) { +CouchbaseOperations::Result CouchbaseOperations::Delete( + const string& key, string collection_name) { CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.DeleteRequest(key, collection_name, channel, server_address, selected_bucket) == false){ + if (request.DeleteRequest(key, collection_name, channel, server_address, + selected_bucket) == false) { LOG(ERROR) << "Failed to create Delete request for key: " << key; result.success = false; result.value = ""; @@ -1870,13 +1983,14 @@ CouchbaseOperations::Result CouchbaseOperations::Delete(const string& key, strin } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to delete key: " << key << " from Couchbase: " << cntl.ErrorText(); + LOG(ERROR) << "Failed to delete key: " << key + << " from Couchbase: " << cntl.ErrorText(); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); return result; } - if(response.PopDelete() == false){ + if (response.PopDelete() == false) { result.success = false; result.value = ""; result.error_message = response.LastError(); @@ -1888,12 +2002,15 @@ CouchbaseOperations::Result CouchbaseOperations::Delete(const string& key, strin return result; } -CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, const string& value, string collection_name) { +CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, + const string& value, + string collection_name) { CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.AddRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ + if (request.AddRequest(key, value, 0, 0, 0, collection_name, channel, + server_address, selected_bucket) == false) { LOG(ERROR) << "Failed to create Add request for key: " << key; result.success = false; result.value = ""; @@ -1901,13 +2018,14 @@ CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, const st } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to add key: " << key << " to Couchbase: " << cntl.ErrorText(); + LOG(ERROR) << "Failed to add key: " << key + << " to Couchbase: " << cntl.ErrorText(); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); return result; } - if(response.PopAdd(NULL) == false){ + if (response.PopAdd(NULL) == false) { result.success = false; result.value = ""; result.error_message = response.LastError(); @@ -1919,25 +2037,30 @@ CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, const st return result; } -CouchbaseOperations::Result CouchbaseOperations::Authenticate(const string& username, const string& password, const string& server_address, bool enable_ssl, string path_to_cert) { +CouchbaseOperations::Result CouchbaseOperations::Authenticate( + const string& username, const string& password, + const string& server_address, bool enable_ssl, string path_to_cert) { // Create a channel to the Couchbase server brpc::ChannelOptions options; options.protocol = brpc::PROTOCOL_COUCHBASE; options.connection_type = "single"; - options.timeout_ms = 1000; // 1 second + options.timeout_ms = 1000; // 1 second options.max_retry = 3; - //enable_ssl - if(enable_ssl){ + // enable_ssl + if (enable_ssl) { brpc::ChannelSSLOptions* ssl_options = options.mutable_ssl_options(); - ssl_options->sni_name = server_address; - ssl_options->verify.verify_depth = 1; // Enable certificate verification, to disable SSL set it to 0 - ssl_options->verify.ca_file_path = path_to_cert; // Path to your downloaded TLS certificate + ssl_options->sni_name = server_address; + ssl_options->verify.verify_depth = + 1; // Enable certificate verification, to disable SSL set it to 0 + ssl_options->verify.ca_file_path = + path_to_cert; // Path to your downloaded TLS certificate } CouchbaseOperations::Result result; brpc::Channel* new_channel = new brpc::Channel(); if (new_channel->Init(server_address.c_str(), &options) != 0) { - LOG(ERROR) << "Failed to initialize Couchbase channel to " << server_address; + LOG(ERROR) << "Failed to initialize Couchbase channel to " + << server_address; delete new_channel; result.success = false; result.value = ""; @@ -1948,8 +2071,10 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate(const string& user CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; - if(request.AuthenticateRequest(username.c_str(), password.c_str()) == false){ - LOG(ERROR) << "Failed to create Authenticate request for user: " << username; + if (request.AuthenticateRequest(username.c_str(), password.c_str()) == + false) { + LOG(ERROR) << "Failed to create Authenticate request for user: " + << username; delete new_channel; result.success = false; return result; @@ -1970,26 +2095,29 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate(const string& user return result; } -CouchbaseOperations::Result CouchbaseOperations::SelectBucket(const string& bucket_name) { +CouchbaseOperations::Result CouchbaseOperations::SelectBucket( + const string& bucket_name) { CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.SelectBucketRequest(bucket_name.c_str()) == false){ - LOG(ERROR) << "Failed to create Select Bucket request for bucket: " << bucket_name; + if (request.SelectBucketRequest(bucket_name.c_str()) == false) { + LOG(ERROR) << "Failed to create Select Bucket request for bucket: " + << bucket_name; result.success = false; result.value = ""; return result; } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to select bucket: " << bucket_name << " from Couchbase: " << cntl.ErrorText(); + LOG(ERROR) << "Failed to select bucket: " << bucket_name + << " from Couchbase: " << cntl.ErrorText(); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); return result; } - if(response.PopSelectBucket(NULL, bucket_name) == false){ + if (response.PopSelectBucket(NULL, bucket_name) == false) { result.success = false; result.value = ""; result.error_message = response.LastError(); @@ -2003,12 +2131,14 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket(const string& buck } // Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Replace(const string& key, const string& value, string collection_name) { +// CouchbaseOperations::Result CouchbaseOperations::Replace(const string& key, +// const string& value, string collection_name) { // CouchbaseRequest request; // CouchbaseResponse response; // brpc::Controller cntl; // CouchbaseOperations::Result result; -// if(request.ReplaceRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ +// if(request.ReplaceRequest(key, value, 0, 0, 0, collection_name, channel, +// server_address, selected_bucket) == false){ // LOG(ERROR) << "Failed to create Replace request for key: " << key; // result.success = false; // result.value = ""; @@ -2016,9 +2146,8 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket(const string& buck // } // channel->CallMethod(NULL, &cntl, &request, &response, NULL); // if (cntl.Failed()) { -// LOG(ERROR) << "Failed to replace key: " << key << " to Couchbase: " << cntl.ErrorText(); -// result.success = false; -// result.value = ""; +// LOG(ERROR) << "Failed to replace key: " << key << " to Couchbase: " << +// cntl.ErrorText(); result.success = false; result.value = ""; // result.error_message = cntl.ErrorText(); // return result; // } @@ -2035,12 +2164,14 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket(const string& buck // return result; // } -CouchbaseOperations::Result CouchbaseOperations::Append(const string& key, const string& value, string collection_name) { +CouchbaseOperations::Result CouchbaseOperations::Append( + const string& key, const string& value, string collection_name) { CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.AppendRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ + if (request.AppendRequest(key, value, 0, 0, 0, collection_name, channel, + server_address, selected_bucket) == false) { LOG(ERROR) << "Failed to create Append request for key: " << key; result.success = false; result.value = ""; @@ -2048,14 +2179,15 @@ CouchbaseOperations::Result CouchbaseOperations::Append(const string& key, const } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to append to key: " << key << " to Couchbase: " << cntl.ErrorText(); + LOG(ERROR) << "Failed to append to key: " << key + << " to Couchbase: " << cntl.ErrorText(); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); return result; } uint64_t cas_value; - if(response.PopAppend(&cas_value) == false){ + if (response.PopAppend(&cas_value) == false) { result.success = false; result.value = ""; result.error_message = response.LastError(); @@ -2067,12 +2199,14 @@ CouchbaseOperations::Result CouchbaseOperations::Append(const string& key, const return result; } -CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, const string& value, string collection_name) { +CouchbaseOperations::Result CouchbaseOperations::Prepend( + const string& key, const string& value, string collection_name) { CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.PrependRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false){ + if (request.PrependRequest(key, value, 0, 0, 0, collection_name, channel, + server_address, selected_bucket) == false) { LOG(ERROR) << "Failed to create Prepend request for key: " << key; result.success = false; result.value = ""; @@ -2080,14 +2214,15 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, cons } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to prepend to key: " << key << " to Couchbase: " << cntl.ErrorText(); + LOG(ERROR) << "Failed to prepend to key: " << key + << " to Couchbase: " << cntl.ErrorText(); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); return result; } uint64_t cas_value; - if(response.PopPrepend(&cas_value) == false){ + if (response.PopPrepend(&cas_value) == false) { result.success = false; result.value = ""; result.error_message = response.LastError(); @@ -2100,12 +2235,15 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, cons } // Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Increment(const string& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name) { +// CouchbaseOperations::Result CouchbaseOperations::Increment(const string& key, +// uint64_t delta, uint64_t initial_value, uint32_t exptime, string +// collection_name) { // CouchbaseRequest request; // CouchbaseResponse response; // brpc::Controller cntl; // CouchbaseOperations::Result result; -// if(request.IncrementRequest(key, delta, initial_value, exptime, collection_name, channel, server_address, selected_bucket) == false){ +// if(request.IncrementRequest(key, delta, initial_value, exptime, +// collection_name, channel, server_address, selected_bucket) == false){ // LOG(ERROR) << "Failed to create Increment request for key: " << key; // result.success = false; // result.value = ""; @@ -2113,9 +2251,8 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, cons // } // channel->CallMethod(NULL, &cntl, &request, &response, NULL); // if (cntl.Failed()) { -// LOG(ERROR) << "Failed to increment key: " << key << " in Couchbase: " << cntl.ErrorText(); -// result.success = false; -// result.value = ""; +// LOG(ERROR) << "Failed to increment key: " << key << " in Couchbase: " << +// cntl.ErrorText(); result.success = false; result.value = ""; // result.error_message = cntl.ErrorText(); // return result; // } @@ -2133,12 +2270,15 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, cons // } // Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Decrement(const string& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name) { +// CouchbaseOperations::Result CouchbaseOperations::Decrement(const string& key, +// uint64_t delta, uint64_t initial_value, uint32_t exptime, string +// collection_name) { // CouchbaseRequest request; // CouchbaseResponse response; // brpc::Controller cntl; // CouchbaseOperations::Result result; -// if(request.DecrementRequest(key, delta, initial_value, exptime, collection_name, channel, server_address, selected_bucket) == false){ +// if(request.DecrementRequest(key, delta, initial_value, exptime, +// collection_name, channel, server_address, selected_bucket) == false){ // LOG(ERROR) << "Failed to create Decrement request for key: " << key; // result.success = false; // result.value = ""; @@ -2146,9 +2286,8 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, cons // } // channel->CallMethod(NULL, &cntl, &request, &response, NULL); // if (cntl.Failed()) { -// LOG(ERROR) << "Failed to decrement key: " << key << " in Couchbase: " << cntl.ErrorText(); -// result.success = false; -// result.value = ""; +// LOG(ERROR) << "Failed to decrement key: " << key << " in Couchbase: " << +// cntl.ErrorText(); result.success = false; result.value = ""; // result.error_message = cntl.ErrorText(); // return result; // } @@ -2166,12 +2305,14 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, cons // } // Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Touch(const string& key, uint32_t exptime, string collection_name) { +// CouchbaseOperations::Result CouchbaseOperations::Touch(const string& key, +// uint32_t exptime, string collection_name) { // CouchbaseRequest request; // CouchbaseResponse response; // brpc::Controller cntl; // CouchbaseOperations::Result result; -// if(request.TouchRequest(key, exptime, collection_name, channel, server_address, selected_bucket) == false){ +// if(request.TouchRequest(key, exptime, collection_name, channel, +// server_address, selected_bucket) == false){ // LOG(ERROR) << "Failed to create Touch request for key: " << key; // result.success = false; // result.value = ""; @@ -2179,9 +2320,8 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend(const string& key, cons // } // channel->CallMethod(NULL, &cntl, &request, &response, NULL); // if (cntl.Failed()) { -// LOG(ERROR) << "Failed to touch key: " << key << " in Couchbase: " << cntl.ErrorText(); -// result.success = false; -// result.value = ""; +// LOG(ERROR) << "Failed to touch key: " << key << " in Couchbase: " << +// cntl.ErrorText(); result.success = false; result.value = ""; // result.error_message = cntl.ErrorText(); // return result; // } @@ -2234,7 +2374,7 @@ CouchbaseOperations::Result CouchbaseOperations::Version() { CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(request.VersionRequest() == false){ + if (request.VersionRequest() == false) { LOG(ERROR) << "Failed to create Version request"; result.success = false; result.value = ""; @@ -2249,7 +2389,7 @@ CouchbaseOperations::Result CouchbaseOperations::Version() { return result; } string version; - if(response.PopVersion(&version) == false){ + if (response.PopVersion(&version) == false) { result.success = false; result.value = ""; result.error_message = response.LastError(); @@ -2266,56 +2406,71 @@ bool CouchbaseOperations::BeginPipeline() { LOG(WARNING) << "Pipeline already active. Call ClearPipeline() first."; return false; } - + // Clear any previous state while (!pipeline_operations_queue.empty()) { pipeline_operations_queue.pop(); } pipeline_request.Clear(); - + pipeline_active = true; return true; } -bool CouchbaseOperations::PipelineRequest(pipeline_operation_type op_type, const string& key, const string& value, string collection_name) { +bool CouchbaseOperations::PipelineRequest(pipeline_operation_type op_type, + const string& key, + const string& value, + string collection_name) { if (!pipeline_active) { LOG(ERROR) << "Pipeline not active. Call BeginPipeline() first."; return false; } - - switch(op_type){ + + switch (op_type) { case GET: - if(pipeline_request.GetRequest(key, collection_name, channel, server_address, selected_bucket)== false){ + if (pipeline_request.GetRequest(key, collection_name, channel, + server_address, + selected_bucket) == false) { return false; } pipeline_operations_queue.push(GET); break; case UPSERT: - if(pipeline_request.UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket)== false){ + if (pipeline_request.UpsertRequest(key, value, 0, 0, 0, collection_name, + channel, server_address, + selected_bucket) == false) { return false; } pipeline_operations_queue.push(UPSERT); break; case ADD: - if(pipeline_request.AddRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket)== false){ + if (pipeline_request.AddRequest(key, value, 0, 0, 0, collection_name, + channel, server_address, + selected_bucket) == false) { return false; } pipeline_operations_queue.push(ADD); break; case APPEND: - if(pipeline_request.AppendRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket)== false){ + if (pipeline_request.AppendRequest(key, value, 0, 0, 0, collection_name, + channel, server_address, + selected_bucket) == false) { return false; } pipeline_operations_queue.push(APPEND); break; case PREPEND: - if(pipeline_request.PrependRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket)== false){ + if (pipeline_request.PrependRequest(key, value, 0, 0, 0, collection_name, + channel, server_address, + selected_bucket) == false) { return false; } pipeline_operations_queue.push(PREPEND); break; case DELETE: - if(pipeline_request.DeleteRequest(key, collection_name, channel, server_address, selected_bucket)== false){ + if (pipeline_request.DeleteRequest(key, collection_name, channel, + server_address, + selected_bucket) == false) { return false; } pipeline_operations_queue.push(DELETE); @@ -2328,45 +2483,45 @@ bool CouchbaseOperations::PipelineRequest(pipeline_operation_type op_type, const } vector CouchbaseOperations::ExecutePipeline() { vector results; - + if (!pipeline_active || pipeline_operations_queue.empty()) { LOG(ERROR) << "No pipeline active or no operations queued"; return results; } - + brpc::Controller cntl; channel->CallMethod(NULL, &cntl, &pipeline_request, &pipeline_response, NULL); - + if (cntl.Failed()) { LOG(ERROR) << "Pipeline execution failed: " << cntl.ErrorText(); // Create failure results for all operations size_t op_count = pipeline_operations_queue.size(); results.reserve(op_count); - + CouchbaseOperations::Result failure_result; failure_result.success = false; failure_result.error_message = cntl.ErrorText(); - + for (size_t i = 0; i < op_count; ++i) { results.push_back(failure_result); } - + ClearPipeline(); return results; } - + // Process each operation in the order they were added - CouchbaseOperations::CouchbaseResponse *response = &pipeline_response; - while(!pipeline_operations_queue.empty()){ + CouchbaseOperations::CouchbaseResponse* response = &pipeline_response; + while (!pipeline_operations_queue.empty()) { CouchbaseOperations::Result result; pipeline_operation_type op_type = pipeline_operations_queue.front(); pipeline_operations_queue.pop(); - switch(op_type){ + switch (op_type) { case GET: { string value; uint32_t flags = 0; uint64_t cas = 0; - if(response->PopGet(&value, &flags, &cas) == false){ + if (response->PopGet(&value, &flags, &cas) == false) { result.success = false; result.value = ""; result.error_message = response->LastError(); @@ -2378,7 +2533,7 @@ vector CouchbaseOperations::ExecutePipeline() { break; } case UPSERT: { - if(response->PopUpsert(NULL) == false){ + if (response->PopUpsert(NULL) == false) { result.success = false; result.value = ""; result.error_message = response->LastError(); @@ -2390,7 +2545,7 @@ vector CouchbaseOperations::ExecutePipeline() { break; } case ADD: { - if(response->PopAdd(NULL) == false){ + if (response->PopAdd(NULL) == false) { result.success = false; result.value = ""; result.error_message = response->LastError(); @@ -2403,7 +2558,7 @@ vector CouchbaseOperations::ExecutePipeline() { } case APPEND: { uint64_t cas_value; - if(response->PopAppend(&cas_value) == false){ + if (response->PopAppend(&cas_value) == false) { result.success = false; result.value = ""; result.error_message = response->LastError(); @@ -2416,7 +2571,7 @@ vector CouchbaseOperations::ExecutePipeline() { } case PREPEND: { uint64_t cas_value; - if(response->PopPrepend(&cas_value) == false){ + if (response->PopPrepend(&cas_value) == false) { result.success = false; result.value = ""; result.error_message = response->LastError(); @@ -2428,7 +2583,7 @@ vector CouchbaseOperations::ExecutePipeline() { break; } case DELETE: { - if(response->PopDelete() == false){ + if (response->PopDelete() == false) { result.success = false; result.value = ""; result.error_message = response->LastError(); @@ -2448,10 +2603,10 @@ vector CouchbaseOperations::ExecutePipeline() { break; } } - + pipeline_active = false; pipeline_request.Clear(); - + return results; } @@ -2463,4 +2618,4 @@ bool CouchbaseOperations::ClearPipeline() { pipeline_active = false; return true; } -}// namespace brpc \ No newline at end of file +} // namespace brpc \ No newline at end of file diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 71df9ef8e6..2b95011720 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -20,21 +20,26 @@ #endif -#include +#include + +#include +#include +#include #include +#include +#include #include "brpc/nonreflectable_message.h" -#include #include "brpc/pb_compat.h" #include "butil/iobuf.h" #include "butil/strings/string_piece.h" -#include -#include -#include -#include using namespace std; -namespace google { namespace protobuf { class Message; } } +namespace google { +namespace protobuf { +class Message; +} +} // namespace google namespace brpc { @@ -42,416 +47,464 @@ namespace brpc { class InputMessageBase; class Controller; namespace policy { - void ProcessCouchbaseResponse(InputMessageBase* msg); - void SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); -} +void ProcessCouchbaseResponse(InputMessageBase* msg); +void SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, + const google::protobuf::Message* request); +} // namespace policy // Simple C++11 compatible reader-writer lock class ReaderWriterLock { -private: - std::mutex mutex_; - std::condition_variable reader_cv_; - std::condition_variable writer_cv_; - std::atomic reader_count_; - std::atomic writer_active_; - std::atomic waiting_writers_; - -public: - ReaderWriterLock() : reader_count_(0), writer_active_(false), waiting_writers_(0) {} - - void lock_shared() { - std::unique_lock lock(mutex_); - reader_cv_.wait(lock, [this] { return !writer_active_.load() && waiting_writers_.load() == 0; }); - reader_count_.fetch_add(1); - } - - void unlock_shared() { - reader_count_.fetch_sub(1); - if (reader_count_.load() == 0) { - std::lock_guard lock(mutex_); - writer_cv_.notify_one(); - } - } - - void lock() { - std::unique_lock lock(mutex_); - waiting_writers_.fetch_add(1); - writer_cv_.wait(lock, [this] { return !writer_active_.load() && reader_count_.load() == 0; }); - waiting_writers_.fetch_sub(1); - writer_active_.store(true); - } - - void unlock() { - writer_active_.store(false); - std::lock_guard lock(mutex_); - writer_cv_.notify_one(); - reader_cv_.notify_all(); + private: + std::mutex mutex_; + std::condition_variable reader_cv_; + std::condition_variable writer_cv_; + std::atomic reader_count_; + std::atomic writer_active_; + std::atomic waiting_writers_; + + public: + ReaderWriterLock() + : reader_count_(0), writer_active_(false), waiting_writers_(0) {} + + void lock_shared() { + std::unique_lock lock(mutex_); + reader_cv_.wait(lock, [this] { + return !writer_active_.load() && waiting_writers_.load() == 0; + }); + reader_count_.fetch_add(1); + } + + void unlock_shared() { + reader_count_.fetch_sub(1); + if (reader_count_.load() == 0) { + std::lock_guard lock(mutex_); + writer_cv_.notify_one(); } + } + + void lock() { + std::unique_lock lock(mutex_); + waiting_writers_.fetch_add(1); + writer_cv_.wait(lock, [this] { + return !writer_active_.load() && reader_count_.load() == 0; + }); + waiting_writers_.fetch_sub(1); + writer_active_.store(true); + } + + void unlock() { + writer_active_.store(false); + std::lock_guard lock(mutex_); + writer_cv_.notify_one(); + reader_cv_.notify_all(); + } }; // RAII helper classes class SharedLock { -private: - ReaderWriterLock& lock_; -public: - explicit SharedLock(ReaderWriterLock& lock) : lock_(lock) { - lock_.lock_shared(); - } - ~SharedLock() { - lock_.unlock_shared(); - } + private: + ReaderWriterLock& lock_; + + public: + explicit SharedLock(ReaderWriterLock& lock) : lock_(lock) { + lock_.lock_shared(); + } + ~SharedLock() { lock_.unlock_shared(); } }; class UniqueLock { -private: - ReaderWriterLock& lock_; -public: - explicit UniqueLock(ReaderWriterLock& lock) : lock_(lock) { - lock_.lock(); + private: + ReaderWriterLock& lock_; + + public: + explicit UniqueLock(ReaderWriterLock& lock) : lock_(lock) { lock_.lock(); } + ~UniqueLock() { lock_.unlock(); } +}; + +class CouchbaseMetadataTracking { + public: + struct CollectionManifest { + string uid; // uid of the manifest, it can be used to track if the manifest + // is updated + unordered_map> + scope_to_collectionID_map; // scope -> (collection -> collection_id) + }; + + private: + unordered_map> + bucket_to_collection_manifest; + ReaderWriterLock rw_bucket_to_collection_manifest_mutex; + + public: + CouchbaseMetadataTracking() {} + ~CouchbaseMetadataTracking() { bucket_to_collection_manifest.clear(); } + bool set_bucket_to_collection_manifest(string server, string bucket, + CollectionManifest manifest); + + bool get_bucket_to_collection_manifest(string server, string bucket, + CollectionManifest* manifest); + bool get_manifest_to_collection_id(CollectionManifest* manifest, string scope, + string collection, uint8_t* collection_id); + + bool json_to_collection_manifest(const string& json, + CollectionManifest* manifest); +} static common_metadata_tracking; +class CouchbaseOperations { + public: + enum pipeline_operation_type { + GET = 1, + UPSERT = 2, + ADD = 3, + REPLACE = 4, + APPEND = 5, + PREPEND = 6, + DELETE = 7 + }; + struct Result { + bool success; + string error_message; + string value; + }; + Result Get(const string& key, string collection_name = "_default"); + Result Upsert(const string& key, const string& value, + string collection_name = "_default"); + Result Add(const string& key, const string& value, + string collection_name = "_default"); + // Warning: Not tested + // Result Replace(const string& key, const string& value, string + // collection_name = "_default"); + Result Append(const string& key, const string& value, + string collection_name = "_default"); + Result Prepend(const string& key, const string& value, + string collection_name = "_default"); + Result Delete(const string& key, string collection_name = "_default"); + // Warning: Not tested + // Result Increment(const string& key, uint64_t delta, uint64_t initial_value, + // uint32_t exptime, string collection_name = "_default"); Result + // Decrement(const string& key, uint64_t delta, uint64_t initial_value, + // uint32_t exptime, string collection_name = "_default"); Result Touch(const + // string& key, uint32_t exptime, string collection_name = "_default"); Result + // Flush(uint32_t timeout = 0); + Result Version(); + Result Authenticate(const string& username, const string& password, + const string& server_address, bool enable_ssl, + string path_to_cert); + Result SelectBucket(const string& bucket_name); + + // Pipeline management + bool BeginPipeline(); + bool PipelineRequest(pipeline_operation_type op_type, const string& key, + const string& value = "", + string collection_name = "_default"); + vector ExecutePipeline(); // Return by value instead of pointer + bool ClearPipeline(); + + // Pipeline status + bool IsPipelineActive() const { return pipeline_active; } + size_t GetPipelineSize() const { return pipeline_operations_queue.size(); } + + CouchbaseOperations() : pipeline_active(false) {} + ~CouchbaseOperations() {} + + private: + // Friend functions to allow access to private nested classes + friend bool get_cached_or_fetch_collection_id( + string collection_name, uint8_t* coll_id, + brpc::CouchbaseMetadataTracking* metadata_tracking, + brpc::Channel* channel, const string& server, + const string& selected_bucket); + friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); + friend void policy::SerializeCouchbaseRequest( + butil::IOBuf* buf, Controller* cntl, + const google::protobuf::Message* request); + + brpc::Channel* channel; + string server_address; + string selected_bucket; + + class CouchbaseRequest : public NonreflectableMessage { + private: + static brpc::CouchbaseMetadataTracking* metadata_tracking; + int _pipelined_count; + butil::IOBuf _buf; + mutable int _cached_size_; + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + bool GetOrDelete(uint8_t command, const butil::StringPiece& key, + uint8_t coll_id = 0); + bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, + uint64_t initial_value, uint32_t exptime); + + bool Store(uint8_t command, const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value, uint8_t coll_id = 0); + uint32_t hash_crc32(const char* key, size_t key_length); + + public: + CouchbaseRequest() : NonreflectableMessage() { + metadata_tracking = &common_metadata_tracking; + SharedCtor(); } - ~UniqueLock() { - lock_.unlock(); + ~CouchbaseRequest() { SharedDtor(); } + CouchbaseRequest(const CouchbaseRequest& from) + : NonreflectableMessage(from) { + SharedCtor(); + MergeFrom(from); } -}; -class CouchbaseMetadataTracking{ - public: - struct CollectionManifest{ - string uid; //uid of the manifest, it can be used to track if the manifest is updated - unordered_map> scope_to_collectionID_map; //scope -> (collection -> collection_id) - }; - private: - unordered_map> bucket_to_collection_manifest; - ReaderWriterLock rw_bucket_to_collection_manifest_mutex; - - public: - CouchbaseMetadataTracking() {} - ~CouchbaseMetadataTracking() { - bucket_to_collection_manifest.clear(); + inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { + CopyFrom(from); + return *this; } - bool set_bucket_to_collection_manifest(string server, string bucket, CollectionManifest manifest); - - bool get_bucket_to_collection_manifest(string server, string bucket, CollectionManifest *manifest); - bool get_manifest_to_collection_id(CollectionManifest *manifest, string scope, string collection, uint8_t *collection_id); - bool json_to_collection_manifest(const string& json, CollectionManifest *manifest); -} static common_metadata_tracking; -class CouchbaseOperations { - public: - enum pipeline_operation_type { - GET = 1, - UPSERT = 2, - ADD = 3, - REPLACE = 4, - APPEND = 5, - PREPEND = 6, - DELETE = 7 - }; - struct Result { - bool success; - string error_message; - string value; - }; - Result Get(const string& key, string collection_name = "_default"); - Result Upsert(const string& key, const string& value, string collection_name = "_default"); - Result Add(const string& key, const string& value, string collection_name = "_default"); - // Warning: Not tested - // Result Replace(const string& key, const string& value, string collection_name = "_default"); - Result Append(const string& key, const string& value, string collection_name = "_default"); - Result Prepend(const string& key, const string& value, string collection_name = "_default"); - Result Delete(const string& key, string collection_name = "_default"); - // Warning: Not tested - // Result Increment(const string& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); - // Result Decrement(const string& key, uint64_t delta, uint64_t initial_value, uint32_t exptime, string collection_name = "_default"); - // Result Touch(const string& key, uint32_t exptime, string collection_name = "_default"); - // Result Flush(uint32_t timeout = 0); - Result Version(); - Result Authenticate(const string& username, const string& password, const string& server_address, bool enable_ssl, string path_to_cert); - Result SelectBucket(const string& bucket_name); - - // Pipeline management - bool BeginPipeline(); - bool PipelineRequest(pipeline_operation_type op_type, const string& key, const string& value = "", string collection_name = "_default"); - vector ExecutePipeline(); // Return by value instead of pointer - bool ClearPipeline(); - - // Pipeline status - bool IsPipelineActive() const { return pipeline_active; } - size_t GetPipelineSize() const { return pipeline_operations_queue.size(); } - - CouchbaseOperations() : pipeline_active(false) {} - ~CouchbaseOperations() {} - private: - // Friend functions to allow access to private nested classes - friend bool get_cached_or_fetch_collection_id(string collection_name, uint8_t *coll_id, brpc::CouchbaseMetadataTracking *metadata_tracking, brpc::Channel* channel, const string& server, const string& selected_bucket); - friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); - friend void policy::SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); - - brpc::Channel* channel; - string server_address; - string selected_bucket; - - class CouchbaseRequest : public NonreflectableMessage { - private: - static brpc::CouchbaseMetadataTracking *metadata_tracking; - int _pipelined_count; - butil::IOBuf _buf; - mutable int _cached_size_; - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - bool GetOrDelete(uint8_t command, const butil::StringPiece& key, - uint8_t coll_id = 0); - bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, - uint64_t initial_value, uint32_t exptime); - - bool Store(uint8_t command, const butil::StringPiece& key, - const butil::StringPiece& value, uint32_t flags, uint32_t exptime, - uint64_t cas_value, uint8_t coll_id = 0); - uint32_t hash_crc32(const char* key, size_t key_length); - - public: - CouchbaseRequest() : NonreflectableMessage() { - metadata_tracking = &common_metadata_tracking; - SharedCtor(); - } - ~CouchbaseRequest() { SharedDtor(); } - CouchbaseRequest(const CouchbaseRequest& from) : NonreflectableMessage(from) { - SharedCtor(); - MergeFrom(from); - } - - inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { - CopyFrom(from); - return *this; - } - - bool SelectBucketRequest(const butil::StringPiece& bucket_name); - bool AuthenticateRequest(const butil::StringPiece& username, - const butil::StringPiece& password); - bool HelloRequest(); - - // Using GetCollectionManifest instead of fetching collection ID directly - // bool GetCollectionId(const butil::StringPiece& scope_name, - // const butil::StringPiece& collection_name); - - bool GetScopeId(const butil::StringPiece& scope_name); - - bool GetCollectionManifest(); - - bool RefreshCollectionManifest(); - - // Collection-aware document operations - bool GetRequest(const butil::StringPiece& key, string collection_name = "_default", - brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - - bool UpsertRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default", - brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + bool SelectBucketRequest(const butil::StringPiece& bucket_name); + bool AuthenticateRequest(const butil::StringPiece& username, + const butil::StringPiece& password); + bool HelloRequest(); - bool AddRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default", - brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + // Using GetCollectionManifest instead of fetching collection ID directly + // bool GetCollectionId(const butil::StringPiece& scope_name, + // const butil::StringPiece& collection_name); - // Warning: Not tested - // bool ReplaceRequest(const butil::StringPiece& key, const butil::StringPiece& value, - // uint32_t flags, uint32_t exptime, uint64_t cas_value, - // string collection_name = "_default", - // brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + bool GetScopeId(const butil::StringPiece& scope_name); - bool AppendRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default", - brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); + bool GetCollectionManifest(); + + bool RefreshCollectionManifest(); - bool PrependRequest(const butil::StringPiece& key, const butil::StringPiece& value, - uint32_t flags, uint32_t exptime, uint64_t cas_value, + // Collection-aware document operations + bool GetRequest(const butil::StringPiece& key, string collection_name = "_default", - brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - - bool DeleteRequest(const butil::StringPiece& key, string collection_name = "_default", - brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - - // Warning: Not tested - // bool FlushRequest(uint32_t timeout); - - // bool IncrementRequest(const butil::StringPiece& key, uint64_t delta, - // uint64_t initial_value, uint32_t exptime, string collection_name = "_default", - // brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - // bool DecrementRequest(const butil::StringPiece& key, uint64_t delta, - // uint64_t initial_value, uint32_t exptime, string collection_name = "_default", - // brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - - // bool TouchRequest(const butil::StringPiece& key, uint32_t exptime, - // string collection_name = "_default", - // brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - - bool VersionRequest(); - - int pipelined_count() const { return _pipelined_count; } - - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - void Swap(CouchbaseRequest* other); - void MergeFrom(const CouchbaseRequest& from) override; - void Clear() override; - bool IsInitialized() const PB_527_OVERRIDE; - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - }; + brpc::Channel* channel = nullptr, const string& server = "", + const string& bucket = ""); + + bool UpsertRequest(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value, + string collection_name = "_default", + brpc::Channel* channel = nullptr, + const string& server = "", const string& bucket = ""); + + bool AddRequest(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value, + string collection_name = "_default", + brpc::Channel* channel = nullptr, const string& server = "", + const string& bucket = ""); + + // Warning: Not tested + // bool ReplaceRequest(const butil::StringPiece& key, const + // butil::StringPiece& value, + // uint32_t flags, uint32_t exptime, uint64_t cas_value, + // string collection_name = "_default", + // brpc::Channel* channel = nullptr, const string& server = "", + // const string& bucket = ""); + + bool AppendRequest(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value, + string collection_name = "_default", + brpc::Channel* channel = nullptr, + const string& server = "", const string& bucket = ""); + + bool PrependRequest(const butil::StringPiece& key, + const butil::StringPiece& value, uint32_t flags, + uint32_t exptime, uint64_t cas_value, + string collection_name = "_default", + brpc::Channel* channel = nullptr, + const string& server = "", const string& bucket = ""); + + bool DeleteRequest(const butil::StringPiece& key, + string collection_name = "_default", + brpc::Channel* channel = nullptr, + const string& server = "", const string& bucket = ""); - class CouchbaseResponse : public NonreflectableMessage { - private: - string _err; - static brpc::CouchbaseMetadataTracking *metadata_tracking; - butil::IOBuf _buf; - mutable int _cached_size_; - bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); - bool PopStore(uint8_t command, uint64_t* cas_value); - - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - - public: - CouchbaseResponse() : NonreflectableMessage() { - SharedCtor(); - } - ~CouchbaseResponse() { SharedDtor(); } - CouchbaseResponse(const CouchbaseResponse& from) : NonreflectableMessage(from) { - metadata_tracking = &common_metadata_tracking; - SharedCtor(); - MergeFrom(from); - } - inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { - CopyFrom(from); - return *this; - } - - // the status codes are from Couchbase Binary Protocol documentation, - // for original reference of status codes visit - // https://github.com/couchbase/kv_engine/blob/master/include/mcbp/protocol/status.h - enum Status { - STATUS_SUCCESS = 0x00, - STATUS_KEY_ENOENT = 0x01, - STATUS_KEY_EEXISTS = 0x02, - STATUS_E2BIG = 0x03, - STATUS_EINVAL = 0x04, - STATUS_NOT_STORED = 0x05, - STATUS_DELTA_BADVAL = 0x06, - STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, - STATUS_AUTH_ERROR = 0x20, - STATUS_AUTH_CONTINUE = 0x21, - STATUS_ERANGE = 0x22, - STATUS_ROLLBACK = 0x23, - STATUS_EACCESS = 0x24, - STATUS_NOT_INITIALIZED = 0x25, - STATUS_UNKNOWN_COMMAND = 0x81, - STATUS_ENOMEM = 0x82, - STATUS_NOT_SUPPORTED = 0x83, - STATUS_EINTERNAL = 0x84, - STATUS_EBUSY = 0x85, - STATUS_ETMPFAIL = 0x86, - STATUS_UNKNOWN_COLLECTION = 0x88, - STATUS_NO_COLLECTIONS_MANIFEST = 0x89, - STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, - STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, - STATUS_UNKNOWN_SCOPE = 0x8c, - STATUS_DCP_STREAM_ID_INVALID = 0x8d, - STATUS_DURABILITY_INVALID_LEVEL = 0xa0, - STATUS_DURABILITY_IMPOSSIBLE = 0xa1, - STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, - STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, - STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, - STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, - STATUS_SUBDOC_PATH_MISMATCH = 0xc1, - STATUS_SUBDOC_PATH_EINVAL = 0xc2, - STATUS_SUBDOC_PATH_E2BIG = 0xc3, - STATUS_SUBDOC_DOC_E2DEEP = 0xc4, - STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, - STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, - STATUS_SUBDOC_NUM_E2BIG = 0xc7, - STATUS_SUBDOC_DELTA_E2BIG = 0xc8, - STATUS_SUBDOC_PATH_EEXISTS = 0xc9, - STATUS_SUBDOC_VALUE_E2DEEP = 0xca, - STATUS_SUBDOC_INVALID_COMBO = 0xcb, - STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, - STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, - STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, - STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, - STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, - STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, - STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, - STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, - STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, - STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, - STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, - STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, - STATUS_XATTR_EINVAL = 0xe0 - }; - const char* couchbase_binary_command_to_string(uint8_t cmd); - void MergeFrom(const CouchbaseResponse& from) override; - void Clear() override; - bool IsInitialized() const PB_527_OVERRIDE; - - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - static const char* status_str(Status); - - // Helper method to format error messages with status codes - static string format_error_message(uint16_t status_code, - const string& operation, - const string& error_msg = ""); - - // Add methods to handle response parsing - void Swap(CouchbaseResponse* other); - bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool PopGet(string* value, uint32_t* flags, uint64_t* cas_value); - const string& LastError() const { return _err; } - bool PopUpsert(uint64_t* cas_value); - bool PopAdd(uint64_t* cas_value); - // Warning: Not tested - // bool PopReplace(uint64_t* cas_value); - bool PopAppend(uint64_t* cas_value); - bool PopPrepend(uint64_t* cas_value); - bool PopSelectBucket(uint64_t* cas_value, std::string bucket_name); - - // Collection-related response methods - bool PopCollectionId(uint8_t* collection_id); - - bool PopManifest(std::string* manifest_json); - - bool PopDelete(); - // Warning: Not tested - // bool PopFlush(); - // bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); - // bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); - // bool PopTouch(); - bool PopVersion(string* version); + // Warning: Not tested + // bool FlushRequest(uint32_t timeout); + + // bool IncrementRequest(const butil::StringPiece& key, uint64_t delta, + // uint64_t initial_value, uint32_t exptime, string + // collection_name = "_default", brpc::Channel* channel = + // nullptr, const string& server = "", const string& bucket = + // ""); + // bool DecrementRequest(const butil::StringPiece& key, uint64_t delta, + // uint64_t initial_value, uint32_t exptime, string + // collection_name = "_default", brpc::Channel* channel = + // nullptr, const string& server = "", const string& bucket = + // ""); + + // bool TouchRequest(const butil::StringPiece& key, uint32_t exptime, + // string collection_name = "_default", + // brpc::Channel* channel = nullptr, const string& server = "", + // const string& bucket = ""); + + bool VersionRequest(); + + int pipelined_count() const { return _pipelined_count; } + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + void Swap(CouchbaseRequest* other); + void MergeFrom(const CouchbaseRequest& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream* + output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; + }; + + class CouchbaseResponse : public NonreflectableMessage { + private: + string _err; + static brpc::CouchbaseMetadataTracking* metadata_tracking; + butil::IOBuf _buf; + mutable int _cached_size_; + bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); + bool PopStore(uint8_t command, uint64_t* cas_value); + + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const PB_425_OVERRIDE; + + public: + CouchbaseResponse() : NonreflectableMessage() { + SharedCtor(); + } + ~CouchbaseResponse() { SharedDtor(); } + CouchbaseResponse(const CouchbaseResponse& from) + : NonreflectableMessage(from) { + metadata_tracking = &common_metadata_tracking; + SharedCtor(); + MergeFrom(from); + } + inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { + CopyFrom(from); + return *this; + } + + // the status codes are from Couchbase Binary Protocol documentation, + // for original reference of status codes visit + // https://github.com/couchbase/kv_engine/blob/master/include/mcbp/protocol/status.h + enum Status { + STATUS_SUCCESS = 0x00, + STATUS_KEY_ENOENT = 0x01, + STATUS_KEY_EEXISTS = 0x02, + STATUS_E2BIG = 0x03, + STATUS_EINVAL = 0x04, + STATUS_NOT_STORED = 0x05, + STATUS_DELTA_BADVAL = 0x06, + STATUS_VBUCKET_BELONGS_TO_ANOTHER_SERVER = 0x07, + STATUS_AUTH_ERROR = 0x20, + STATUS_AUTH_CONTINUE = 0x21, + STATUS_ERANGE = 0x22, + STATUS_ROLLBACK = 0x23, + STATUS_EACCESS = 0x24, + STATUS_NOT_INITIALIZED = 0x25, + STATUS_UNKNOWN_COMMAND = 0x81, + STATUS_ENOMEM = 0x82, + STATUS_NOT_SUPPORTED = 0x83, + STATUS_EINTERNAL = 0x84, + STATUS_EBUSY = 0x85, + STATUS_ETMPFAIL = 0x86, + STATUS_UNKNOWN_COLLECTION = 0x88, + STATUS_NO_COLLECTIONS_MANIFEST = 0x89, + STATUS_CANNOT_APPLY_COLLECTIONS_MANIFEST = 0x8a, + STATUS_COLLECTIONS_MANIFEST_IS_AHEAD = 0x8b, + STATUS_UNKNOWN_SCOPE = 0x8c, + STATUS_DCP_STREAM_ID_INVALID = 0x8d, + STATUS_DURABILITY_INVALID_LEVEL = 0xa0, + STATUS_DURABILITY_IMPOSSIBLE = 0xa1, + STATUS_SYNC_WRITE_IN_PROGRESS = 0xa2, + STATUS_SYNC_WRITE_AMBIGUOUS = 0xa3, + STATUS_SYNC_WRITE_RE_COMMIT_IN_PROGRESS = 0xa4, + STATUS_SUBDOC_PATH_NOT_FOUND = 0xc0, + STATUS_SUBDOC_PATH_MISMATCH = 0xc1, + STATUS_SUBDOC_PATH_EINVAL = 0xc2, + STATUS_SUBDOC_PATH_E2BIG = 0xc3, + STATUS_SUBDOC_DOC_E2DEEP = 0xc4, + STATUS_SUBDOC_VALUE_CANTINSERT = 0xc5, + STATUS_SUBDOC_DOC_NOT_JSON = 0xc6, + STATUS_SUBDOC_NUM_E2BIG = 0xc7, + STATUS_SUBDOC_DELTA_E2BIG = 0xc8, + STATUS_SUBDOC_PATH_EEXISTS = 0xc9, + STATUS_SUBDOC_VALUE_E2DEEP = 0xca, + STATUS_SUBDOC_INVALID_COMBO = 0xcb, + STATUS_SUBDOC_MULTI_PATH_FAILURE = 0xcc, + STATUS_SUBDOC_SUCCESS_DELETED = 0xcd, + STATUS_SUBDOC_XATTR_INVALID_FLAG_COMBO = 0xce, + STATUS_SUBDOC_XATTR_INVALID_KEY_COMBO = 0xcf, + STATUS_SUBDOC_XATTR_UNKNOWN_MACRO = 0xd0, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR = 0xd1, + STATUS_SUBDOC_XATTR_CANT_MODIFY_VATTR = 0xd2, + STATUS_SUBDOC_MULTI_PATH_FAILURE_DELETED = 0xd3, + STATUS_SUBDOC_INVALID_XATTR_ORDER = 0xd4, + STATUS_SUBDOC_XATTR_UNKNOWN_VATTR_MACRO = 0xd5, + STATUS_SUBDOC_CAN_ONLY_REVIVE_DELETED_DOCUMENTS = 0xd6, + STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, + STATUS_XATTR_EINVAL = 0xe0 }; + const char* couchbase_binary_command_to_string(uint8_t cmd); + void MergeFrom(const CouchbaseResponse& from) override; + void Clear() override; + bool IsInitialized() const PB_527_OVERRIDE; + + size_t ByteSizeLong() const override; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; + void SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream* + output) const PB_310_OVERRIDE; + int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } + + ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; + + butil::IOBuf& raw_buffer() { return _buf; } + const butil::IOBuf& raw_buffer() const { return _buf; } + static const char* status_str(Status); + + // Helper method to format error messages with status codes + static string format_error_message(uint16_t status_code, + const string& operation, + const string& error_msg = ""); + + // Add methods to handle response parsing + void Swap(CouchbaseResponse* other); + bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); + bool PopGet(string* value, uint32_t* flags, uint64_t* cas_value); + const string& LastError() const { return _err; } + bool PopUpsert(uint64_t* cas_value); + bool PopAdd(uint64_t* cas_value); + // Warning: Not tested + // bool PopReplace(uint64_t* cas_value); + bool PopAppend(uint64_t* cas_value); + bool PopPrepend(uint64_t* cas_value); + bool PopSelectBucket(uint64_t* cas_value, std::string bucket_name); - // Pipeline management - per instance - queue pipeline_operations_queue; - CouchbaseRequest pipeline_request; - CouchbaseResponse pipeline_response; - bool pipeline_active; + // Collection-related response methods + bool PopCollectionId(uint8_t* collection_id); + + bool PopManifest(std::string* manifest_json); + + bool PopDelete(); + // Warning: Not tested + // bool PopFlush(); + // bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); + // bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); + // bool PopTouch(); + bool PopVersion(string* version); + }; + + // Pipeline management - per instance + queue pipeline_operations_queue; + CouchbaseRequest pipeline_request; + CouchbaseResponse pipeline_response; + bool pipeline_active; }; } // namespace brpc \ No newline at end of file From a66208ee0fdd4d785f0a1943cbe950623f78a395 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Sat, 4 Oct 2025 18:05:06 +0530 Subject: [PATCH 24/49] Introduced local cache per-instance of CouchbaseOperations and added functionality to handle server side manifest updates. --- example/couchbase_c++/couchbase_client.cpp | 26 +- src/brpc/couchbase.cpp | 715 ++++++++++++++------- src/brpc/couchbase.h | 57 +- 3 files changed, 519 insertions(+), 279 deletions(-) diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index c061a7f464..f8ec367314 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -27,7 +27,7 @@ #define RED "\033[31m" #define RESET "\033[0m" -DEFINE_string(server, "cb.dqklxewecglohzwb.cloud.couchbase.com:11207", +DEFINE_string(server, "localhost:11210", "IP Address of server"); int main() { @@ -38,8 +38,8 @@ int main() { << RESET << std::endl; // Ask username and password for authentication - std::string username; - std::string password; + std::string username="Administrator"; + std::string password="password"; while (username.empty() || password.empty()) { std::cout << "Enter Couchbase username: "; std::cin >> username; @@ -56,8 +56,9 @@ int main() { } // Use high-level authentication method + // when connecting to capella use couchbase_ops.Authenticate(username, password, FLAGS_server, true, "path/to/cert.txt"); brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( - username, password, FLAGS_server, true, "couchbase-cloud-cert.txt"); + username, password, FLAGS_server); if (!auth_result.success) { LOG(ERROR) << "Authentication failed: " << auth_result.error_message; return -1; @@ -69,7 +70,7 @@ int main() { << RESET << std::endl; // Select bucket - std::string bucket_name; + std::string bucket_name="testing"; while (bucket_name.empty()) { std::cout << "Enter Couchbase bucket name: "; std::cin >> bucket_name; @@ -233,12 +234,12 @@ int main() { const std::string scope_name = "_default"; // default scope std::string collection_name = "testing_collection"; // target collection // enter collection name as user input - std::cout << "Enter collection name (default 'testing_collection'): "; - std::string user_input; - std::cin >> user_input; - if (!user_input.empty()) { - collection_name = user_input; - } + // std::cout << "Enter collection name (default 'testing_collection'): "; + // std::string user_input; + // std::cin >> user_input; + // if (!user_input.empty()) { + // collection_name = user_input; + // } // ------------------------------------------------------------------ // Collection-scoped CRUD operations (only if collection id was retrieved) // ------------------------------------------------------------------ @@ -254,7 +255,8 @@ int main() { << "Collection ADD failed: " << coll_add_result.error_message << RESET << std::endl; } - + // std::cout << "Delete the collection."; + // std::cin >> collection_name; // 2. GET from collection using high-level method brpc::CouchbaseOperations::Result coll_get_result = couchbase_ops.Get(coll_key, collection_name); diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 5cdb623564..e60651c421 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -804,7 +804,8 @@ bool CouchbaseOperations::CouchbaseRequest::GetOrDelete( return true; } -bool get_cached_or_fetch_collection_id( +// collectionID fetching either from the metadata cache or if doesn't exist then fetch from the server. +bool CouchbaseOperations::CouchbaseRequest::get_cached_or_fetch_collection_id( string collection_name, uint8_t* coll_id, brpc::CouchbaseMetadataTracking* metadata_tracking, brpc::Channel* channel, const string& server, const string& selected_bucket) { @@ -812,7 +813,6 @@ bool get_cached_or_fetch_collection_id( LOG(ERROR) << "Empty collection name"; return false; } - if (channel == nullptr) { LOG(ERROR) << "No channel found, make sure to call Authenticate() first"; return false; @@ -827,6 +827,7 @@ bool get_cached_or_fetch_collection_id( } brpc::CouchbaseMetadataTracking::CollectionManifest manifest; + //check if the server/bucket exists in the cached collection manifest if (!metadata_tracking->get_bucket_to_collection_manifest( server, selected_bucket, &manifest)) { LOG(INFO) << "No cached collection manifest found for bucket " @@ -860,7 +861,7 @@ bool get_cached_or_fetch_collection_id( << selected_bucket << " on server " << server; return false; } - // Cache the collection manifest + // Cache the collection manifest in global cache if (!metadata_tracking->set_bucket_to_collection_manifest( server, selected_bucket, manifest)) { LOG(ERROR) << "Failed to cache collection ID for collection " @@ -868,26 +869,58 @@ bool get_cached_or_fetch_collection_id( << " on server " << server; return false; } + //copy the data to the local cache of the CouchbaseOperations instance which called this. + (*local_collection_manifest_cache)[selected_bucket] = manifest; + if( !metadata_tracking->get_manifest_to_collection_id( + &manifest, "_default", collection_name, coll_id)) { + return false; + } + // Collection does exist and the collection id is stored in coll_id return true; } } else { + // check if collection name to id mapping exists. if (!metadata_tracking->get_manifest_to_collection_id( &manifest, "_default", collection_name, coll_id)) { - return false; + // Just to verify that the collectionID does not exist in the manifest + // refresh manifest from server and try again + if(!RefreshCollectionManifest(channel, server, selected_bucket)) { + return false; + } + // get the reference to latest manifest fetched from server + if (!metadata_tracking->get_bucket_to_collection_manifest( + server, selected_bucket, &manifest)) { + return false; + } + // re-verify the collectionID in the latest manifest. + if( !metadata_tracking->get_manifest_to_collection_id( + &manifest, "_default", collection_name, coll_id)) { + return false; + } } return true; } } + bool CouchbaseOperations::CouchbaseRequest::GetRequest( const butil::StringPiece& key, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket)) { - return false; + // check if the local cache is empty or not. + if(local_collection_manifest_cache->empty()){ + // if local cache is empty, goto global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } + } + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + // if not check in the global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } } } return GetOrDelete(policy::CB_BINARY_GET, key, coll_id); @@ -898,10 +931,19 @@ bool CouchbaseOperations::CouchbaseRequest::DeleteRequest( brpc::Channel* channel, const string& server, const string& bucket) { uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket)) { - return false; + // check if the local cache is empty or not. + if(local_collection_manifest_cache->empty()){ + // if local cache is empty, goto global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } + } + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + // if not check in the global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } } } return GetOrDelete(policy::CB_BINARY_DELETE, key, coll_id); @@ -984,6 +1026,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, LOG_IF(ERROR, header.key_length != 0) << "GET response must not have key"; const int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; + _status_code = header.status; if (value_size < 0) { butil::string_printf(&_err, "value_size=%d is non-negative", value_size); return false; @@ -1137,6 +1180,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, (int)header.key_length; if (header.status != (uint16_t)STATUS_SUCCESS) { _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); + _status_code = header.status; if (value_size > 0) { std::string error_msg; _buf.cutn(&error_msg, value_size); @@ -1257,10 +1301,19 @@ bool CouchbaseOperations::CouchbaseRequest::UpsertRequest( const string& bucket) { uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket)) { - return false; + // check if the local cache is empty or not. + if(local_collection_manifest_cache->empty()){ + // if local cache is empty, goto global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } + } + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + // if not check in the global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } } } return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value, @@ -1315,75 +1368,76 @@ bool CouchbaseOperations::CouchbaseRequest::GetCollectionManifest() { return true; } -// bool RefreshCollectionManifest(brpc::Channel* channel) { -// // first fetch the manifest -// // then compare the UID with the cached one -// CouchbaseRequest temp_get_manifest_request; -// CouchbaseResponse temp_get_manifest_response; -// brpc::Controller temp_cntl; -// temp_get_manifest_request.GetCollectionManifest(); -// channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, -// &temp_get_manifest_response, NULL); -// if (temp_cntl.Failed()) { -// LOG(ERROR) << "Failed to get collection manifest: " << -// temp_cntl.ErrorText(); return false; -// } -// string manifest_json; -// if (!temp_get_manifest_response.PopManifest(&manifest_json)) { -// LOG(ERROR) << "Failed to parse response for collection Manifest: " << -// temp_get_manifest_response.LastError(); return false; -// } -// // Compare the UID with the cached one -// // If they are different, refresh the cache -// brpc::CouchbaseMetadataTracking::CollectionManifest manifest; -// if(!common_metadata_tracking.json_to_collection_manifest(manifest_json, -// &manifest)){ -// LOG(ERROR) << "Failed to parse collection manifest JSON"; -// return false; -// } -// brpc::CouchbaseMetadataTracking::ChannelInfo temp_channel_info; -// common_metadata_tracking.get_channel_info_for_thread(bthread_self(), -// &temp_channel_info); if(temp_channel_info.server.empty() || -// temp_channel_info.selected_bucket.empty()){ -// LOG(ERROR) << "No channel info found for this thread, make sure to call -// Authenticate() and SelectBucket() first"; return false; -// } -// brpc::CouchbaseMetadataTracking::CollectionManifest cached_manifest; -// if(!common_metadata_tracking.get_bucket_to_collection_manifest(bthread_self(), -// temp_channel_info.server, temp_channel_info.selected_bucket, -// &cached_manifest)){ -// // No cached manifest found, set the new one -// if(!common_metadata_tracking.set_bucket_to_collection_manifest(bthread_self(), -// manifest)){ -// LOG(ERROR) << "Failed to cache collection manifest for bucket " << -// temp_channel_info.selected_bucket << " on server " << -// temp_channel_info.server; return false; -// } -// LOG(INFO) << "Cached collection manifest for bucket " << -// temp_channel_info.selected_bucket << " on server " << -// temp_channel_info.server; return true; -// } -// if(manifest.uid != cached_manifest.uid) { -// LOG(INFO) << "Collection manifest has changed for bucket " << -// temp_channel_info.selected_bucket << " on server " << -// temp_channel_info.server; -// if(!common_metadata_tracking.set_bucket_to_collection_manifest(bthread_self(), -// manifest)){ -// LOG(ERROR) << "Failed to update cached collection manifest for bucket " -// << temp_channel_info.selected_bucket << " on server " << -// temp_channel_info.server; return false; -// } -// LOG(INFO) << "Updated cached collection manifest for bucket " << -// temp_channel_info.selected_bucket << " on server " << -// temp_channel_info.server; -// } -// else{ -// LOG(INFO) << "Collection manifest is up-to-date for bucket " << -// temp_channel_info.selected_bucket << " on server " << -// temp_channel_info.server; -// } -// return true; -// } +bool CouchbaseOperations::CouchbaseRequest::RefreshCollectionManifest(brpc::Channel* channel, const string& server, const string& bucket) { + // first fetch the manifest + // then compare the UID with the cached one + if (channel == nullptr) { + LOG(ERROR) << "No channel found, make sure to call Authenticate() first"; + return false; + } + if (server.empty()) { + LOG(ERROR) << "Server is empty, make sure to call Authenticate() first"; + return false; + } + if (bucket.empty()) { + LOG(ERROR) << "No bucket selected, make sure to call SelectBucket() first"; + return false; + } + CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; + CouchbaseOperations::CouchbaseResponse temp_get_manifest_response; + brpc::Controller temp_cntl; + temp_get_manifest_request.GetCollectionManifest(); + channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, + &temp_get_manifest_response, NULL); + if (temp_cntl.Failed()) { + LOG(ERROR) << "Failed to get collection manifest: bRPC controller error" << + temp_cntl.ErrorText(); return false; + } + string manifest_json; + if (!temp_get_manifest_response.PopManifest(&manifest_json)) { + LOG(ERROR) << "Failed to parse response for refreshing collection Manifest: " << + temp_get_manifest_response.LastError(); return false; + } + // Compare the UID with the cached one + // If they are different, refresh the cache + brpc::CouchbaseMetadataTracking::CollectionManifest manifest; + if(!common_metadata_tracking.json_to_collection_manifest(manifest_json, + &manifest)){ + LOG(ERROR) << "Failed to parse collection manifest JSON"; + return false; + } + brpc::CouchbaseMetadataTracking::CollectionManifest cached_manifest; + if(!common_metadata_tracking.get_bucket_to_collection_manifest(server, bucket, + &cached_manifest)){ + // No cached manifest found, set the new one + if(!common_metadata_tracking.set_bucket_to_collection_manifest(server, bucket, + manifest)){ + LOG(ERROR) << "Failed to cache collection manifest for bucket " << + bucket << " on server " << server; return false; + } + LOG(INFO) << "Cached collection manifest for bucket " << + bucket << " on server " << server; return true; + } + if(manifest.uid != cached_manifest.uid) { + LOG(INFO) << "Collection manifest has changed for bucket " << + bucket << " on server " << server; + if(!common_metadata_tracking.set_bucket_to_collection_manifest(server, bucket, + manifest)){ + LOG(ERROR) << "Failed to update cached collection manifest for bucket " + << bucket << " on server " << server; return false; + } + LOG(INFO) << "Updated cached collection manifest for bucket " << + bucket << " on server " << server; + // also update the local cache + (*local_collection_manifest_cache)[bucket] = manifest; + return true; + } + else{ + LOG(INFO) << "Collection manifest is already up-to-date for bucket " << + bucket << " on server " << server; + return false; + } +} bool CouchbaseOperations::CouchbaseRequest::AddRequest( const butil::StringPiece& key, const butil::StringPiece& value, @@ -1392,10 +1446,19 @@ bool CouchbaseOperations::CouchbaseRequest::AddRequest( const string& bucket) { uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket)) { - return false; + // check if the local cache is empty or not. + if(local_collection_manifest_cache->empty()){ + // if local cache is empty, goto global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } + } + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + // if not check in the global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } } } return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, @@ -1433,10 +1496,19 @@ bool CouchbaseOperations::CouchbaseRequest::AppendRequest( } uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket)) { - return false; + // check if the local cache is empty or not. + if(!local_collection_manifest_cache->empty()){ + // if local cache is empty, goto global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } + } + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + // if not check in the global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } } } return Store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value, @@ -1454,10 +1526,19 @@ bool CouchbaseOperations::CouchbaseRequest::PrependRequest( } uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket)) { - return false; + // check if the local cache is empty or not. + if(!local_collection_manifest_cache->empty()){ + // if local cache is empty, goto global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } + } + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + // if not check in the global cache or fetch from server + if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + return false; + } } } return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, cas_value, @@ -1895,145 +1976,272 @@ bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { return true; } +bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, + const string& value, string collection_name, + CouchbaseOperations::Result *result, brpc::Channel *channel, + const string& server, const string& bucket, + CouchbaseOperations::CouchbaseRequest *request, + CouchbaseOperations::CouchbaseResponse *response) { + if(channel == nullptr){ + LOG(ERROR)<< "No channel found, make sure to call Authenticate() first"; + result->error_message = "No channel found, make sure to call Authenticate() first"; + return false; + } + if(server.empty()){ + LOG(ERROR)<< "Server is empty, make sure to call Authenticate() first"; + result->error_message = "Server is empty, make sure to call Authenticate() first"; + return false; + } + if(bucket.empty()){ + LOG(ERROR)<< "No bucket selected, make sure to call SelectBucket() first"; + result->error_message = "No bucket selected, make sure to call SelectBucket() first"; + return false; + } + brpc::Controller cntl; + switch(op_type){ + case CouchbaseOperations::GET: + request->GetRequest(key, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::UPSERT: + request->UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::ADD: + request->AddRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::APPEND: + request->AppendRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::PREPEND: + request->PrependRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::DELETE: + request->DeleteRequest(key, collection_name, channel, server, bucket); + break; + default: + LOG(ERROR) << "Unsupported operation type"; + result->success = false; + result->value = ""; + result->error_message = "Unsupported operation type"; + return false; + } + channel->CallMethod(NULL, &cntl, request, response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Failed to perform operation on key: " << key + << " to Couchbase: " << cntl.ErrorText(); + result->success = false; + result->value = ""; + result->error_message = cntl.ErrorText(); + return false; + } + if(op_type == CouchbaseOperations::GET) { + string value; + uint32_t flags = 0; + uint64_t cas = 0; + if (response->PopGet(&value, &flags, &cas) == false) { + result->success = false; + result->value = ""; + result->error_message = response->LastError(); + result->status_code = response->_status_code; + return false; + } + // Successfully got the value + result->success = true; + result->value = value; + result->status_code = 0; + return true; + } + else{ + // For other operations, just check if it was successful + if (response->LastError().empty() && response->_status_code == 0) { + result->success = true; + result->value = ""; + result->status_code = 0; + return true; + } else { + result->success = false; + result->value = ""; + result->error_message = response->LastError(); + result->status_code = response->_status_code; + return false; + } + } +} CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, string collection_name) { // create CouchbaseRequest and CouchbaseResponse objects and then using the // channel which is created for this thread in authenticate() use it to call() - CouchbaseRequest request; + CouchbaseRequest request(&local_bucket_to_collection_manifest); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if (request.GetRequest(key, collection_name, channel, server_address, - selected_bucket) == false) { - LOG(ERROR) << "Failed to create Get request for key: " << key; - result.success = false; - result.value = ""; + if(!sendRequest(CouchbaseOperations::GET, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ + if(result.status_code == 0x88) { + // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. + // and the client have a stale copy of collection manifest. + // In this case, we need to refresh the collection manifest and retry the operation. + if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { + LOG(ERROR) << "Failed to refresh collection manifest"; + result.success = false; + result.value = ""; + result.error_message = "Failed to refresh collection manifest"; + return result; + } + // check if the colelction id is available in the local cache now. + uint8_t coll_id = 0; // default collection ID + if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ + LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + result.success = false; + result.value = ""; + result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; + return result; + } + // only reachable if the collection manifest now has the collectionID + sendRequest(CouchbaseOperations::GET, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response); + return result; + } return result; } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Failed to get key: " << key - << " from Couchbase: " << cntl.ErrorText(); - result.success = false; - result.value = ""; - result.error_message = cntl.ErrorText(); - return result; + return result; +} + +bool CouchbaseOperations::CouchbaseRequest::getLocalCachedCollectionId(const string& bucket, const string& scope, + const string& collection, + uint8_t* collection_id) { + if (bucket.empty() || scope.empty() || collection.empty()) { + LOG(ERROR) << "Bucket, scope, and collection names must be non-empty"; + return false; } - string value; - uint32_t flags = 0; - uint64_t cas = 0; - if (response.PopGet(&value, &flags, &cas) == false) { - result.success = false; - result.value = ""; - result.error_message = response.LastError(); - return result; + auto it = local_collection_manifest_cache->find(bucket); + if(it != local_collection_manifest_cache->end()) { + CouchbaseMetadataTracking::CollectionManifest& manifest = it->second; + if(manifest.scope_to_collectionID_map.find(scope)!= manifest.scope_to_collectionID_map.end()){ + auto& collection_map = manifest.scope_to_collectionID_map[scope]; + if(collection_map.find(collection) != collection_map.end()){ + *collection_id = collection_map[collection]; + return true; + } + else{ + LOG(ERROR) << "Collection name not found in local cache: " << collection; + return false; + } + } + else{ + LOG(ERROR) << "Scope name not found in local cache: " << scope; + return false; + } + } + else{ + LOG(ERROR) << "Bucket name not found in local cache: " << bucket; + return false; } - // Successfully got the value - result.success = true; - result.value = value; - return result; } CouchbaseOperations::Result CouchbaseOperations::Upsert( const string& key, const string& value, string collection_name) { - CouchbaseRequest request; + CouchbaseRequest request(&local_bucket_to_collection_manifest); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if (request.UpsertRequest(key, value, 0, 0, 0, collection_name, channel, - server_address, selected_bucket) == false) { - LOG(ERROR) << "Failed to create Upsert request for key: " << key; - result.success = false; - result.value = ""; - return result; - } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Failed to upsert key: " << key - << " to Couchbase: " << cntl.ErrorText(); - result.success = false; - result.value = ""; - result.error_message = cntl.ErrorText(); - return result; - } - if (response.PopUpsert(NULL) == false) { - result.success = false; - result.value = ""; - result.error_message = response.LastError(); + if(!sendRequest(CouchbaseOperations::UPSERT, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ + if(result.status_code == 0x88) { + // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. + // and the client have a stale copy of collection manifest. + // In this case, we need to refresh the collection manifest and retry the operation. + if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { + LOG(ERROR) << "Failed to refresh collection manifest"; + result.success = false; + result.value = ""; + result.error_message = "Failed to refresh collection manifest"; + return result; + } + // check if the colelction id is available in the local cache now. + uint8_t coll_id = 0; // default collection ID + if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ + LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + result.success = false; + result.value = ""; + result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; + return result; + } + // only reachable if the collection manifest now has the collectionID + sendRequest(CouchbaseOperations::UPSERT, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); + return result; + } return result; } - // Successfully upserted the value - result.success = true; - result.value = ""; return result; } CouchbaseOperations::Result CouchbaseOperations::Delete( const string& key, string collection_name) { - CouchbaseRequest request; + CouchbaseRequest request(&local_bucket_to_collection_manifest); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if (request.DeleteRequest(key, collection_name, channel, server_address, - selected_bucket) == false) { - LOG(ERROR) << "Failed to create Delete request for key: " << key; - result.success = false; - result.value = ""; - return result; - } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Failed to delete key: " << key - << " from Couchbase: " << cntl.ErrorText(); - result.success = false; - result.value = ""; - result.error_message = cntl.ErrorText(); - return result; - } - if (response.PopDelete() == false) { - result.success = false; - result.value = ""; - result.error_message = response.LastError(); + if(!sendRequest(CouchbaseOperations::DELETE, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ + if(result.status_code == 0x88) { + // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. + // and the client have a stale copy of collection manifest. + // In this case, we need to refresh the collection manifest and retry the operation. + if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { + LOG(ERROR) << "Failed to refresh collection manifest"; + result.success = false; + result.value = ""; + result.error_message = "Failed to refresh collection manifest"; + return result; + } + // check if the colelction id is available in the local cache now. + uint8_t coll_id = 0; // default collection ID + if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ + LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + result.success = false; + result.value = ""; + result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; + return result; + } + // only reachable if the collection manifest now has the collectionID + sendRequest(CouchbaseOperations::DELETE, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response); + return result; + } return result; } - // Successfully deleted the value - result.success = true; - result.value = ""; return result; } CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, const string& value, string collection_name) { - CouchbaseRequest request; + CouchbaseRequest request(&local_bucket_to_collection_manifest); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if (request.AddRequest(key, value, 0, 0, 0, collection_name, channel, - server_address, selected_bucket) == false) { - LOG(ERROR) << "Failed to create Add request for key: " << key; - result.success = false; - result.value = ""; - return result; - } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Failed to add key: " << key - << " to Couchbase: " << cntl.ErrorText(); - result.success = false; - result.value = ""; - result.error_message = cntl.ErrorText(); - return result; - } - if (response.PopAdd(NULL) == false) { - result.success = false; - result.value = ""; - result.error_message = response.LastError(); + if(!sendRequest(CouchbaseOperations::ADD, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ + if(result.status_code == 0x88) { + // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. + // and the client have a stale copy of collection manifest. + // In this case, we need to refresh the collection manifest and retry the operation. + if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { + LOG(ERROR) << "Failed to refresh collection manifest"; + result.success = false; + result.value = ""; + result.error_message = "Failed to refresh collection manifest"; + return result; + } + // check if the colelction id is available in the local cache now. + uint8_t coll_id = 0; // default collection ID + if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ + LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + result.success = false; + result.value = ""; + result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; + return result; + } + // only reachable if the collection manifest now has the collectionID + sendRequest(CouchbaseOperations::ADD, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); + return result; + } return result; } - // Successfully added the value - result.success = true; - result.value = ""; return result; } @@ -2092,6 +2300,7 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate( channel = new_channel; this->server_address = server_address; result.success = true; + result.status_code = 0; return result; } @@ -2121,12 +2330,14 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket( result.success = false; result.value = ""; result.error_message = response.LastError(); + result.status_code = response._status_code; return result; } // Successfully selected the bucket selected_bucket = bucket_name; result.success = true; result.value = ""; + result.status_code = 0; return result; } @@ -2166,71 +2377,73 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket( CouchbaseOperations::Result CouchbaseOperations::Append( const string& key, const string& value, string collection_name) { - CouchbaseRequest request; + CouchbaseRequest request(&local_bucket_to_collection_manifest); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if (request.AppendRequest(key, value, 0, 0, 0, collection_name, channel, - server_address, selected_bucket) == false) { - LOG(ERROR) << "Failed to create Append request for key: " << key; - result.success = false; - result.value = ""; - return result; - } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Failed to append to key: " << key - << " to Couchbase: " << cntl.ErrorText(); - result.success = false; - result.value = ""; - result.error_message = cntl.ErrorText(); - return result; - } - uint64_t cas_value; - if (response.PopAppend(&cas_value) == false) { - result.success = false; - result.value = ""; - result.error_message = response.LastError(); + if(!sendRequest(CouchbaseOperations::APPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ + if(result.status_code == 0x88) { + // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. + // and the client have a stale copy of collection manifest. + // In this case, we need to refresh the collection manifest and retry the operation. + if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { + LOG(ERROR) << "Failed to refresh collection manifest"; + result.success = false; + result.value = ""; + result.error_message = "Failed to refresh collection manifest"; + return result; + } + // check if the colelction id is available in the local cache now. + uint8_t coll_id = 0; // default collection ID + if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ + LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + result.success = false; + result.value = ""; + result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; + return result; + } + // only reachable if the collection manifest now has the collectionID + sendRequest(CouchbaseOperations::APPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); + return result; + } return result; } - // Successfully appended the value - result.success = true; - result.value = ""; return result; } CouchbaseOperations::Result CouchbaseOperations::Prepend( const string& key, const string& value, string collection_name) { - CouchbaseRequest request; + CouchbaseRequest request(&local_bucket_to_collection_manifest); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if (request.PrependRequest(key, value, 0, 0, 0, collection_name, channel, - server_address, selected_bucket) == false) { - LOG(ERROR) << "Failed to create Prepend request for key: " << key; - result.success = false; - result.value = ""; - return result; - } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); - if (cntl.Failed()) { - LOG(ERROR) << "Failed to prepend to key: " << key - << " to Couchbase: " << cntl.ErrorText(); - result.success = false; - result.value = ""; - result.error_message = cntl.ErrorText(); - return result; - } - uint64_t cas_value; - if (response.PopPrepend(&cas_value) == false) { - result.success = false; - result.value = ""; - result.error_message = response.LastError(); + if(!sendRequest(CouchbaseOperations::PREPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ + if(result.status_code == 0x88) { + // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. + // and the client have a stale copy of collection manifest. + // In this case, we need to refresh the collection manifest and retry the operation. + if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { + LOG(ERROR) << "Failed to refresh collection manifest"; + result.success = false; + result.value = ""; + result.error_message = "Failed to refresh collection manifest"; + return result; + } + // check if the colelction id is available in the local cache now. + uint8_t coll_id = 0; // default collection ID + if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ + LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + result.success = false; + result.value = ""; + result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; + return result; + } + // only reachable if the collection manifest now has the collectionID + sendRequest(CouchbaseOperations::PREPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); + return result; + } return result; } - // Successfully prepended the value - result.success = true; - result.value = ""; return result; } @@ -2393,11 +2606,13 @@ CouchbaseOperations::Result CouchbaseOperations::Version() { result.success = false; result.value = ""; result.error_message = response.LastError(); + result.status_code = response._status_code; return result; } // Successfully got version result.success = true; result.value = version; + result.status_code = 0; return result; } @@ -2417,7 +2632,7 @@ bool CouchbaseOperations::BeginPipeline() { return true; } -bool CouchbaseOperations::PipelineRequest(pipeline_operation_type op_type, +bool CouchbaseOperations::PipelineRequest(operation_type op_type, const string& key, const string& value, string collection_name) { @@ -2514,7 +2729,7 @@ vector CouchbaseOperations::ExecutePipeline() { CouchbaseOperations::CouchbaseResponse* response = &pipeline_response; while (!pipeline_operations_queue.empty()) { CouchbaseOperations::Result result; - pipeline_operation_type op_type = pipeline_operations_queue.front(); + operation_type op_type = pipeline_operations_queue.front(); pipeline_operations_queue.pop(); switch (op_type) { case GET: { @@ -2525,6 +2740,7 @@ vector CouchbaseOperations::ExecutePipeline() { result.success = false; result.value = ""; result.error_message = response->LastError(); + result.status_code = response->_status_code; } else { result.success = true; result.value = value; @@ -2537,6 +2753,7 @@ vector CouchbaseOperations::ExecutePipeline() { result.success = false; result.value = ""; result.error_message = response->LastError(); + result.status_code = response->_status_code; } else { result.success = true; result.value = ""; @@ -2549,6 +2766,7 @@ vector CouchbaseOperations::ExecutePipeline() { result.success = false; result.value = ""; result.error_message = response->LastError(); + result.status_code = response->_status_code; } else { result.success = true; result.value = ""; @@ -2562,6 +2780,7 @@ vector CouchbaseOperations::ExecutePipeline() { result.success = false; result.value = ""; result.error_message = response->LastError(); + result.status_code = response->_status_code; } else { result.success = true; result.value = ""; @@ -2575,6 +2794,7 @@ vector CouchbaseOperations::ExecutePipeline() { result.success = false; result.value = ""; result.error_message = response->LastError(); + result.status_code = response->_status_code; } else { result.success = true; result.value = ""; @@ -2587,6 +2807,7 @@ vector CouchbaseOperations::ExecutePipeline() { result.success = false; result.value = ""; result.error_message = response->LastError(); + result.status_code = response->_status_code; } else { result.success = true; result.value = ""; diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 2b95011720..a0fbc17c1b 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -35,12 +35,6 @@ #include "butil/strings/string_piece.h" using namespace std; -namespace google { -namespace protobuf { -class Message; -} -} // namespace google - namespace brpc { // Forward declarations for friend functions @@ -152,7 +146,7 @@ class CouchbaseMetadataTracking { } static common_metadata_tracking; class CouchbaseOperations { public: - enum pipeline_operation_type { + enum operation_type { GET = 1, UPSERT = 2, ADD = 3, @@ -165,6 +159,7 @@ class CouchbaseOperations { bool success; string error_message; string value; + uint16_t status_code; // 0x00 if success }; Result Get(const string& key, string collection_name = "_default"); Result Upsert(const string& key, const string& value, @@ -188,13 +183,13 @@ class CouchbaseOperations { // Flush(uint32_t timeout = 0); Result Version(); Result Authenticate(const string& username, const string& password, - const string& server_address, bool enable_ssl, - string path_to_cert); + const string& server_address, bool enable_ssl=false, + string path_to_cert=""); Result SelectBucket(const string& bucket_name); // Pipeline management bool BeginPipeline(); - bool PipelineRequest(pipeline_operation_type op_type, const string& key, + bool PipelineRequest(operation_type op_type, const string& key, const string& value = "", string collection_name = "_default"); vector ExecutePipeline(); // Return by value instead of pointer @@ -206,23 +201,20 @@ class CouchbaseOperations { CouchbaseOperations() : pipeline_active(false) {} ~CouchbaseOperations() {} + bool get_local_cached_collection_id(const string& bucket, const string& scope, + const string& collection, uint8_t* coll_id); private: - // Friend functions to allow access to private nested classes - friend bool get_cached_or_fetch_collection_id( - string collection_name, uint8_t* coll_id, - brpc::CouchbaseMetadataTracking* metadata_tracking, - brpc::Channel* channel, const string& server, - const string& selected_bucket); friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); friend void policy::SerializeCouchbaseRequest( butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); - brpc::Channel* channel; string server_address; string selected_bucket; + unordered_map local_bucket_to_collection_manifest; + class CouchbaseRequest : public NonreflectableMessage { private: static brpc::CouchbaseMetadataTracking* metadata_tracking; @@ -241,8 +233,13 @@ class CouchbaseOperations { const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, uint8_t coll_id = 0); uint32_t hash_crc32(const char* key, size_t key_length); - + unordered_map *local_collection_manifest_cache; public: + CouchbaseRequest(unordered_map *local_cache_reference) : NonreflectableMessage() { + metadata_tracking = &common_metadata_tracking; + local_collection_manifest_cache = local_cache_reference; + SharedCtor(); + } CouchbaseRequest() : NonreflectableMessage() { metadata_tracking = &common_metadata_tracking; SharedCtor(); @@ -272,7 +269,18 @@ class CouchbaseOperations { bool GetCollectionManifest(); - bool RefreshCollectionManifest(); + bool getLocalCachedCollectionId(const string& bucket, const string& scope, + const string& collection, uint8_t* coll_id); + + bool get_cached_or_fetch_collection_id( + string collection_name, uint8_t* coll_id, + brpc::CouchbaseMetadataTracking* metadata_tracking, + brpc::Channel* channel, const string& server, + const string& selected_bucket); + + bool RefreshCollectionManifest(brpc::Channel* channel, + const string& server, + const string& bucket); // Collection-aware document operations bool GetRequest(const butil::StringPiece& key, @@ -374,6 +382,8 @@ class CouchbaseOperations { void SetCachedSize(int size) const PB_425_OVERRIDE; public: + uint16_t _status_code; + CouchbaseResponse() : NonreflectableMessage() { SharedCtor(); } @@ -500,8 +510,15 @@ class CouchbaseOperations { bool PopVersion(string* version); }; + friend bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, + const string& value, string collection_name, + CouchbaseOperations::Result *result, brpc::Channel *channel, + const string& server, const string& bucket, + CouchbaseRequest *request, + CouchbaseResponse *response); + // Pipeline management - per instance - queue pipeline_operations_queue; + queue pipeline_operations_queue; CouchbaseRequest pipeline_request; CouchbaseResponse pipeline_response; bool pipeline_active; From 8384ad87c8e476854d3c26ee7f6db9138b3b97f2 Mon Sep 17 00:00:00 2001 From: Giriraj Singh Date: Sat, 4 Oct 2025 18:07:30 +0530 Subject: [PATCH 25/49] Delete MODULE.bazel.lock Unnecessary file --- MODULE.bazel.lock | 992 ---------------------------------------------- 1 file changed, 992 deletions(-) delete mode 100644 MODULE.bazel.lock diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock deleted file mode 100644 index a57f5aba60..0000000000 --- a/MODULE.bazel.lock +++ /dev/null @@ -1,992 +0,0 @@ -{ - "lockFileVersion": 11, - "registryFileHashes": { - "https://baidu.github.io/babylon/registry/bazel_registry.json": "not found", - "https://baidu.github.io/babylon/registry/modules/libevent/2.1.12/MODULE.bazel": "1ba8ed9390cd17472cd0495f0bdfdc4d2de62781fd6ef157b32682f07033bd4d", - "https://baidu.github.io/babylon/registry/modules/libevent/2.1.12/source.json": "97292d6719ae1313ace6f2f049aea985a9bd34570583bba96fe72e3fe722aaa1", - "https://baidu.github.io/babylon/registry/modules/thrift/0.21.0/MODULE.bazel": "8d28c0adba6b3010c8f4e8297039f890286fe3429584559d4332bfbb105a9bb0", - "https://baidu.github.io/babylon/registry/modules/thrift/0.21.0/source.json": "9b0a0c088bad57ec8bdef2e4d1c33cc541d7005dfb6387c03bc64b42a0d6f4cc", - "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", - "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", - "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", - "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/source.json": "14892cc698e02ffedf4967546e6bedb7245015906888d3465fcf27c90a26da10", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", - "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", - "https://bcr.bazel.build/modules/apple_support/1.17.1/source.json": "6b2b8c74d14e8d485528a938e44bdb72a5ba17632b9e14ef6e68a5ee96c8347f", - "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", - "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", - "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", - "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", - "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", - "https://bcr.bazel.build/modules/bazel_features/1.19.0/source.json": "d7bf14517c1b25b9d9c580b0f8795fceeae08a7590f507b76aace528e941375d", - "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", - "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", - "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", - "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", - "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", - "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", - "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", - "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", - "https://bcr.bazel.build/modules/boost.algorithm/1.83.0/MODULE.bazel": "f5afbca8fe26f73016c040ae2525c8cfdfb7c7f8a0b2aa7412e08447a8bf76b4", - "https://bcr.bazel.build/modules/boost.algorithm/1.83.0/source.json": "cbc02be90eedd45da42f55b7e352a3aeac56ae72f62237308b5e6e1733b2f2c5", - "https://bcr.bazel.build/modules/boost.array/1.83.0.bcr.1/MODULE.bazel": "ffe2956a8f5d90839e710cb298f0a342acc5a1d8fdfeee9578cae9657df87e38", - "https://bcr.bazel.build/modules/boost.array/1.83.0.bcr.1/source.json": "6d3925fef402c04bd7ea463203c1892c6ab96351c2264566e4bc9e6543fc777f", - "https://bcr.bazel.build/modules/boost.array/1.83.0/MODULE.bazel": "76bc954c31d8c840d89f347ec3d8f4542fc74229e9ab9d74b7e21342a8becd39", - "https://bcr.bazel.build/modules/boost.assert/1.83.0.bcr.1/MODULE.bazel": "abc07418c37c6e1876f56d959ff1b7cdcac70c8b7236bbacc69ab66686968910", - "https://bcr.bazel.build/modules/boost.assert/1.83.0.bcr.1/source.json": "05d5c7ef77352f32264b9ca625df8ca742b78a94d4b057347a54204d6cd9ded6", - "https://bcr.bazel.build/modules/boost.assert/1.83.0/MODULE.bazel": "7b132fe0760e15733253280bd4b9e0c8867b6a8e27322d3def361cab7a729ac7", - "https://bcr.bazel.build/modules/boost.bind/1.83.0.bcr.1/MODULE.bazel": "c226a7152112ec5329aa831fd7daab296db2e8bf1a427e05c33bb770c885e044", - "https://bcr.bazel.build/modules/boost.bind/1.83.0.bcr.1/source.json": "1ad06dbc6d538e539abaeb58d647d787b63c2b0eb54f61168c8501d420cdd691", - "https://bcr.bazel.build/modules/boost.bind/1.83.0/MODULE.bazel": "11e5db47f917eb4a0238e89bc0eda2b41a979682c679cc21209dbd347c3f4723", - "https://bcr.bazel.build/modules/boost.concept_check/1.83.0.bcr.1/MODULE.bazel": "987ddeeca7c552f7c15557ed9f1e1de26e976d1380002cd41e56ea0ef7e1d25e", - "https://bcr.bazel.build/modules/boost.concept_check/1.83.0.bcr.1/source.json": "0a5cb9c8a8b424c04a717b2cbf483f1ee576611b3370550369671ea4f5852e3c", - "https://bcr.bazel.build/modules/boost.concept_check/1.83.0/MODULE.bazel": "a783fc53e77b42e8baa39a4e8f031d0dc2c87f2da386569bb9d46bc12a652c66", - "https://bcr.bazel.build/modules/boost.config/1.83.0.bcr.1/MODULE.bazel": "a9049b46df8ba7d4a895122cfbda4cf71ad01f441fad62117931cfafecba35d1", - "https://bcr.bazel.build/modules/boost.config/1.83.0.bcr.1/source.json": "4810a18d8d9827bf2822ca8e3e1491cfb882657224bc2def70b334120415ccc2", - "https://bcr.bazel.build/modules/boost.config/1.83.0/MODULE.bazel": "beb2e8899be8dc7e3fc2d5b6d5302e665f42be85046e60a955bcbd7ba8174bf8", - "https://bcr.bazel.build/modules/boost.container_hash/1.83.0.bcr.1/MODULE.bazel": "baf420d50979d18d247681bb4f3e097c054f314c2a3c98c14f1b750871329ad7", - "https://bcr.bazel.build/modules/boost.container_hash/1.83.0.bcr.1/source.json": "f3248fbd4a2cd88333343bd78f317e3f7b085b6a2a31adab26f402083bdbdb26", - "https://bcr.bazel.build/modules/boost.container_hash/1.83.0/MODULE.bazel": "be8328ca4dcbe7832438eedf35607f277dd7b4372d75996f5287f40867d1cd2e", - "https://bcr.bazel.build/modules/boost.conversion/1.83.0.bcr.1/MODULE.bazel": "d05ea5d39c0399ddd0331922e64a6bb8bdfe40b83cb6d4ecee62d7cb68f6c36b", - "https://bcr.bazel.build/modules/boost.conversion/1.83.0.bcr.1/source.json": "7b91cda19dcf46d35b47fd30ea9a6226c880d27f6abd51f8107c5015fec9bd46", - "https://bcr.bazel.build/modules/boost.conversion/1.83.0/MODULE.bazel": "d7c3ae92bb0e806da3174eeba584a66aa02a6677a3768eec62f89c7159b48723", - "https://bcr.bazel.build/modules/boost.core/1.83.0.bcr.1/MODULE.bazel": "6aa5d86d84dbeefeda0f5e3ac596db7426ecb6f6ebdf69383c2963292ee568cf", - "https://bcr.bazel.build/modules/boost.core/1.83.0.bcr.1/source.json": "62f0884fe4d287b72146c3355df6d737b016f312bb3bf9c1bbf41fd84400b046", - "https://bcr.bazel.build/modules/boost.core/1.83.0/MODULE.bazel": "769d1faa7dc89832c1a435110a0b447b2e280b3da740ca1811711418fc2e476e", - "https://bcr.bazel.build/modules/boost.describe/1.83.0.bcr.1/MODULE.bazel": "04f902525e3f53a7eec9e10bb58623d5b249afaa4f245917cc827f63933351ba", - "https://bcr.bazel.build/modules/boost.describe/1.83.0.bcr.1/source.json": "b3201219ad62723a684c98d1f13448346774d5979adf99d88583765150a9095c", - "https://bcr.bazel.build/modules/boost.describe/1.83.0/MODULE.bazel": "63ca2aa1c8f62e8a187767d294529f1a762d0f6a4bc1b6081c3e1af63458bac4", - "https://bcr.bazel.build/modules/boost.detail/1.83.0.bcr.2/MODULE.bazel": "0016cbb9f265ebd4efcbb5f1aff14ccce5754bd98c3d9ff375d3a39f4173f8de", - "https://bcr.bazel.build/modules/boost.detail/1.83.0.bcr.2/source.json": "0044cf130a00ccde75322449d2a525e1ca95af2bd61b49873125943c4d3ff7ad", - "https://bcr.bazel.build/modules/boost.detail/1.83.0/MODULE.bazel": "9bbd10b5c30d0c17908fd0c94e9a26f4899540d1c1a97088911d1195feb98da9", - "https://bcr.bazel.build/modules/boost.dynamic_bitset/1.83.0.bcr.1/MODULE.bazel": "2093a9d09159b97ada67ed370492f61c775ae7df5018c8ba9c69053ed7813dab", - "https://bcr.bazel.build/modules/boost.dynamic_bitset/1.83.0.bcr.1/source.json": "c8470feff0d94ef0c950dfa6b214cc119bbaae95355e27838b9d295c7b222b21", - "https://bcr.bazel.build/modules/boost.exception/1.83.0/MODULE.bazel": "3cc2e06995cb92e329449793e389051035ec3516180aaec0bbfd36efc6c90d7b", - "https://bcr.bazel.build/modules/boost.exception/1.83.0/source.json": "09d071c13a55cd740d2354729eb02cc716a2673c6e6030725690af16f32ad6fa", - "https://bcr.bazel.build/modules/boost.function/1.83.0.bcr.1/MODULE.bazel": "b555cccb955cc4bd8d548f81ec29fdd33284ae04afd9ed92b254abe6357a5ed7", - "https://bcr.bazel.build/modules/boost.function/1.83.0.bcr.1/source.json": "a85e42738a1fd7eb3df45ca31fe213ca23832adb0032a7bcf9cd8dac42445f86", - "https://bcr.bazel.build/modules/boost.function/1.83.0/MODULE.bazel": "fa8cf838f76289180210f17e5f5effb51b91ce0be0a2cc72a2b134e53006bbd6", - "https://bcr.bazel.build/modules/boost.function_types/1.83.0.bcr.1/MODULE.bazel": "266d931a312bafb39053b6a0260e59882c1389e8126f9dc5b3f37a1c2b66a152", - "https://bcr.bazel.build/modules/boost.function_types/1.83.0.bcr.1/source.json": "c421295de00ee97f4d605d8a873ce002b64d9e4226f61db710ceaaf2ff1c581a", - "https://bcr.bazel.build/modules/boost.function_types/1.83.0/MODULE.bazel": "8974442d5a0929ef7df5f63fa5d49221b2274cd19f2be0ca91b99d2538b74d64", - "https://bcr.bazel.build/modules/boost.functional/1.83.0.bcr.1/MODULE.bazel": "8e0f52ea1e5f1d34442347c1ddcb202ce2cc610175226df58770c69d7267b4a8", - "https://bcr.bazel.build/modules/boost.functional/1.83.0.bcr.1/source.json": "6a3c58ca81f5226221ac1565aa3a7c93e5d04b27d2f9bd71e1b113ff19f5a658", - "https://bcr.bazel.build/modules/boost.functional/1.83.0/MODULE.bazel": "f6c5ec6684a05d5c235a9141a417c1c18eee1af1783a30f7f3f19bd3020f128b", - "https://bcr.bazel.build/modules/boost.fusion/1.83.0.bcr.1/MODULE.bazel": "1a651840d5710e5f2b882535f9fd7a18d6932c24559185362a3dfe0ca3702ee4", - "https://bcr.bazel.build/modules/boost.fusion/1.83.0.bcr.1/source.json": "8977203271c1f1db3f329e5cdd4960d26060ef13bdc7e47820b77f61d2369909", - "https://bcr.bazel.build/modules/boost.fusion/1.83.0/MODULE.bazel": "394a7dd5bb57baad66c30db1cf871c2b7d05129637945c04c3b26e11bd0cb4de", - "https://bcr.bazel.build/modules/boost.integer/1.83.0.bcr.1/MODULE.bazel": "29655d7e043ba79065ed8fb8b8b99133470ff240f4606bebb6085d2c856578df", - "https://bcr.bazel.build/modules/boost.integer/1.83.0.bcr.1/source.json": "92d1105b5e005c47b10c09b42dfbc86374a069b62055c7471eb8546545ee534b", - "https://bcr.bazel.build/modules/boost.integer/1.83.0/MODULE.bazel": "2bbbc2e0e01f901ce84ef66859635e3ad0b645eeb3a133e243af8e750f4db54d", - "https://bcr.bazel.build/modules/boost.io/1.83.0.bcr.1/MODULE.bazel": "ce3b42ee33880f7250263a158c51b6c33c38ccbff754b78703c8af3f842e1cb1", - "https://bcr.bazel.build/modules/boost.io/1.83.0.bcr.1/source.json": "94e74cc68d9ba9821aa35c207a943c8140856fbc494b3524d3729f0caa9e97db", - "https://bcr.bazel.build/modules/boost.io/1.83.0/MODULE.bazel": "47457a879a81af920b096b76affd242f898fdf406aec12c62c6772a84aa0e0a0", - "https://bcr.bazel.build/modules/boost.iterator/1.83.0.bcr.1/MODULE.bazel": "6ac7783146113154a83e841bfa4449105718006378edff7fbfc53aeddf00812b", - "https://bcr.bazel.build/modules/boost.iterator/1.83.0.bcr.1/source.json": "16e10bfe39dd3096147160668219e7a9bb8a42bbce5bd6a8a9bfcb9c428f7912", - "https://bcr.bazel.build/modules/boost.iterator/1.83.0/MODULE.bazel": "a526477248e085c73e8059708b61b9238536f923ede8c310e979aec4672f2e5d", - "https://bcr.bazel.build/modules/boost.move/1.83.0.bcr.1/MODULE.bazel": "e901ceb363762eb989a7a53418fe7ce2acf8f542a413109105184a4e22e38096", - "https://bcr.bazel.build/modules/boost.move/1.83.0.bcr.1/source.json": "978a8ff249833ab0764a7c2a83a9bda299960112613f22c1ee3bb3ec97fbddbf", - "https://bcr.bazel.build/modules/boost.move/1.83.0/MODULE.bazel": "9294f124718b42857e0eabde467fd78b2b9f8844529749fc1ffe27bc44d23940", - "https://bcr.bazel.build/modules/boost.mp11/1.83.0.bcr.1/MODULE.bazel": "cf0d62cfca8865a1e1672953bd5b08cf5cb95e3aae819427565606305dd29165", - "https://bcr.bazel.build/modules/boost.mp11/1.83.0.bcr.1/source.json": "554cee46c279651fc3ffca71d947e8b99cb26bd87661debe7946eb21b61031f7", - "https://bcr.bazel.build/modules/boost.mp11/1.83.0/MODULE.bazel": "5b47c08596fd8b0f1cf5e183b22012a8380bacbdf68535de0a469a26bcb32f47", - "https://bcr.bazel.build/modules/boost.mpl/1.83.0.bcr.1/MODULE.bazel": "9cd97b224f9c1d8ffefdf48a7163767ca3968fbdfbc76f057caefbb4054be33e", - "https://bcr.bazel.build/modules/boost.mpl/1.83.0.bcr.1/source.json": "2ad7f4a254613fe2c0722d6127d9b366dbd19dcd8e44ad23f3ff66bf48163019", - "https://bcr.bazel.build/modules/boost.mpl/1.83.0/MODULE.bazel": "3110db5c9c5000140076797d61592f8abffd50e91e04a153ee24bf47d30e38ec", - "https://bcr.bazel.build/modules/boost.numeric_conversion/1.83.0.bcr.1/MODULE.bazel": "1920d71f656a0433947579a86c22e0dd897816eadee3ae20d5c51dab9505deb3", - "https://bcr.bazel.build/modules/boost.numeric_conversion/1.83.0.bcr.1/source.json": "fc35a1dbb8810812f05fa54667f0edeac142f5fc144f9426236694a1d01e875c", - "https://bcr.bazel.build/modules/boost.numeric_conversion/1.83.0/MODULE.bazel": "34fbcfe4eab607de5285b543f63142042f98c0895d765476f251c7bb45acf33d", - "https://bcr.bazel.build/modules/boost.optional/1.83.0.bcr.1/MODULE.bazel": "971e786a19e31dfbd8d311dff3bbffc7c1e16f348e019dfca50620fd9c475092", - "https://bcr.bazel.build/modules/boost.optional/1.83.0.bcr.1/source.json": "486e1173f8b24120078f072f434a79fb3aa8c6b0dafbc19c56b7780ea2336595", - "https://bcr.bazel.build/modules/boost.optional/1.83.0/MODULE.bazel": "d2d9c9afa139aa075212e451570aee269d73d5ccda31ab0ab33a9a4b406c0cf7", - "https://bcr.bazel.build/modules/boost.predef/1.83.0.bcr.1/MODULE.bazel": "0dd34241f892423a9dcd20e1256b6a1c787d60849e9576c724a71607ad8a1955", - "https://bcr.bazel.build/modules/boost.predef/1.83.0.bcr.1/source.json": "deb64fec0911e7a5d38d8517ecdd470b0b3983506662e01f63f8925688c05c01", - "https://bcr.bazel.build/modules/boost.predef/1.83.0/MODULE.bazel": "7bfd5416496b14c32e0684e57e47b8452fc5fe0e4afad294b31c2140b1d73dc8", - "https://bcr.bazel.build/modules/boost.preprocessor/1.83.0.bcr.1/MODULE.bazel": "edb9fb7900ea7002cbefffd97302b071d7cbd8f948b51c7b1a75043bd2985eba", - "https://bcr.bazel.build/modules/boost.preprocessor/1.83.0.bcr.1/source.json": "69dc4f6fc76305c21c4a651c94ccfdc8a76d8fbae1151e7c1d1a4599dffc0f03", - "https://bcr.bazel.build/modules/boost.preprocessor/1.83.0/MODULE.bazel": "5d1096729ebd16d2679c798110c0896720be23959c59afa36547d04815e255c8", - "https://bcr.bazel.build/modules/boost.random/1.83.0.bcr.1/MODULE.bazel": "cabe3ba820c9588a9ca22548b88ad8c4b307be085691b286ff0dae8a46b25fa0", - "https://bcr.bazel.build/modules/boost.random/1.83.0.bcr.1/source.json": "5cc5c8c13e525c5b1c12ea5b31359a16bfaf25b750c0601c0624b9342872e05d", - "https://bcr.bazel.build/modules/boost.range/1.83.0.bcr.1/MODULE.bazel": "136d623462d1d5c7cf79df83b5ce17a8582a92abb116da9d88c5e5594e5a7d92", - "https://bcr.bazel.build/modules/boost.range/1.83.0.bcr.1/source.json": "f99062101034f19d9bf2bef3e07cc99bf192640b72560663227e5375efd1e144", - "https://bcr.bazel.build/modules/boost.range/1.83.0/MODULE.bazel": "ceba0feb376949eecb77944bc37ac434261dede457fc449958164fca5d1430db", - "https://bcr.bazel.build/modules/boost.regex/1.83.0.bcr.1/MODULE.bazel": "5e2c235ea0bdbd8fa942f429ed8aefa1ad3e1471993a17a7b82c1f127dbcb3a2", - "https://bcr.bazel.build/modules/boost.regex/1.83.0.bcr.1/source.json": "1d5fda14ae73d5f12ced3c8731006f28b54bdaf75da35f6cc06c2b57b724d011", - "https://bcr.bazel.build/modules/boost.regex/1.83.0/MODULE.bazel": "75a260515a22080dc48aa02fd6dc1aa6bc42695dd7c6a449f37343cd0ea9b018", - "https://bcr.bazel.build/modules/boost.smart_ptr/1.83.0.bcr.1/MODULE.bazel": "5652293a4b393e5a56ef4113e0f41b6121d22a0d8f7ccd3e27c4042b947683e3", - "https://bcr.bazel.build/modules/boost.smart_ptr/1.83.0.bcr.1/source.json": "1b1b6b549073ef6b1bdbd9898490f73ddb6324774d441cd24d6d1181d4bc75f4", - "https://bcr.bazel.build/modules/boost.smart_ptr/1.83.0/MODULE.bazel": "67537f1bcdf50ab51ad79d9ac4e089018a7ebad1a2d362efcc54f5fc5ba45bd4", - "https://bcr.bazel.build/modules/boost.static_assert/1.83.0.bcr.1/MODULE.bazel": "2b605adc483c6241865f1e862437331bc6f56c0d376769908b70ba18d3da1f07", - "https://bcr.bazel.build/modules/boost.static_assert/1.83.0.bcr.1/source.json": "a0eac8de976fff7efdf498933d7494df30eff471c51c1edfc822007069697ed7", - "https://bcr.bazel.build/modules/boost.static_assert/1.83.0/MODULE.bazel": "680325e3252ae8306555bcf0539d16dcf9ccf9656d8781dfa3449a554d8da016", - "https://bcr.bazel.build/modules/boost.system/1.83.0.bcr.1/MODULE.bazel": "5f905d0fbb1ce99231f3fa278b2e5999aa7395c6393ac42d479ae21824adf03f", - "https://bcr.bazel.build/modules/boost.system/1.83.0.bcr.1/source.json": "0676ab63c01c5ddf731a5cf54667ffc6560e9fb52401a2a9ac6a10c5a9909019", - "https://bcr.bazel.build/modules/boost.throw_exception/1.83.0.bcr.1/MODULE.bazel": "b757c832f5f5f818d87c9eaa993d3eb211554197321c3edf641e2c8821cf19c2", - "https://bcr.bazel.build/modules/boost.throw_exception/1.83.0.bcr.1/source.json": "c752d584840e9183141f9d53f07f2051016c16771a973cdd1487f9585980c2e5", - "https://bcr.bazel.build/modules/boost.throw_exception/1.83.0/MODULE.bazel": "5df92502378293277ca48837e41f33805ede9e6165acefbf83d96b861919e56e", - "https://bcr.bazel.build/modules/boost.tokenizer/1.83.0/MODULE.bazel": "6501c3df8303a842740307b7c4443bb5d01d1ce10abf88d497d9607790bf1d23", - "https://bcr.bazel.build/modules/boost.tokenizer/1.83.0/source.json": "57aa54da8b6ca096ed45a05c752cf68a343567b03e3c069a4f77f485e9389cb9", - "https://bcr.bazel.build/modules/boost.tti/1.83.0.bcr.1/MODULE.bazel": "86dd0d443379e67bb41e9b8c9097d652699ddfc0986bd2fb0462f6f5294ee84d", - "https://bcr.bazel.build/modules/boost.tti/1.83.0.bcr.1/source.json": "660900b6e3615af5b222cf699ff220787d0550e380e28408e475a6a0d354d794", - "https://bcr.bazel.build/modules/boost.tuple/1.83.0.bcr.1/MODULE.bazel": "1d540b5efd3b65eeabd3621e5187a799e21bfa9ffc6afd7d4ad307cc4a27a6d4", - "https://bcr.bazel.build/modules/boost.tuple/1.83.0.bcr.1/source.json": "7aa33ec2aaae45605049ea0ec1c1de9517a1e1278a22d0a521521d4023f9ad87", - "https://bcr.bazel.build/modules/boost.tuple/1.83.0/MODULE.bazel": "96640d5e7abec507a5dd63a9f8a6f90e45bc02d7fed6a0fb51e5e869bba9fecd", - "https://bcr.bazel.build/modules/boost.type_traits/1.83.0.bcr.1/MODULE.bazel": "89bbca2de42dc79ffcabb3625103771e4b04566eee6b8cfe7fa807d133d053f7", - "https://bcr.bazel.build/modules/boost.type_traits/1.83.0.bcr.1/source.json": "051e4969558e4a356c5059f4a4f41335b3968b91fa8ae1772898676ec95ded56", - "https://bcr.bazel.build/modules/boost.type_traits/1.83.0/MODULE.bazel": "4a094b5ecac0d41b0c071455ec0b7384e74b54118b4861bd054d5d9ff521d748", - "https://bcr.bazel.build/modules/boost.typeof/1.83.0.bcr.1/MODULE.bazel": "6b3dd6cfe993a8ccf98bce79506be41ec893c65013d7a5be9177c8a185e86d87", - "https://bcr.bazel.build/modules/boost.typeof/1.83.0.bcr.1/source.json": "0ec40beca58de1b17c1ddb18c357ef525977e4312540122b3b91cbe485556f17", - "https://bcr.bazel.build/modules/boost.typeof/1.83.0/MODULE.bazel": "da614fe02971fa4d77c33e94274d1d72ca7fae6ef3e94288f90064b1c8bb6278", - "https://bcr.bazel.build/modules/boost.unordered/1.83.0/MODULE.bazel": "2fffc6c8bbb119a59561d40dc8add727636b778adb4b1df2a523cd1b1b2393ac", - "https://bcr.bazel.build/modules/boost.unordered/1.83.0/source.json": "c1488a104633d662cd782a7afcd7278f55384eada175a1154c237fd70dbb175c", - "https://bcr.bazel.build/modules/boost.utility/1.83.0.bcr.1/MODULE.bazel": "1346dc27d6c8b7ced10896224ed3e406adac3fd79c8450d78c291228f1b9075d", - "https://bcr.bazel.build/modules/boost.utility/1.83.0.bcr.1/source.json": "15636369b5452784e7bd04f7ae52c751e591dd4ebb852688fccc66234d452929", - "https://bcr.bazel.build/modules/boost.utility/1.83.0/MODULE.bazel": "e122ee2a63d4e76dec8d2f81b13f95b7638fcbcd15f752610a3343f13bdb97fd", - "https://bcr.bazel.build/modules/boost.uuid/1.83.0.bcr.1/MODULE.bazel": "0ec51572f062cfb4795cf57c84a16b1b61890954dead42dd55e8616e09159c37", - "https://bcr.bazel.build/modules/boost.uuid/1.83.0.bcr.1/source.json": "d53f1653d8c2062f42884311de23220573e287b1eb224c465845357ded8c5a89", - "https://bcr.bazel.build/modules/boost.variant2/1.83.0.bcr.1/MODULE.bazel": "c60baa3b8923712a156197ffaf5cf9972bf35e44d00a90f7019a06761f391d3e", - "https://bcr.bazel.build/modules/boost.variant2/1.83.0.bcr.1/source.json": "041f94707bd509bc1f93782ccb6c38d491ac0dca25d26011f2047639d3a2bcd1", - "https://bcr.bazel.build/modules/boost.winapi/1.83.0.bcr.1/MODULE.bazel": "faf78b50dae672a38b77db545a460428cfe47a8d79466455ef397d76037e9e40", - "https://bcr.bazel.build/modules/boost.winapi/1.83.0.bcr.1/source.json": "0957b4dabe425e7f9d8d02db63969b808a280c11388ad7814e50b0779ae592cc", - "https://bcr.bazel.build/modules/boringssl/0.0.0-20211025-d4f1ab9/MODULE.bazel": "6ee6353f8b1a701fe2178e1d925034294971350b6d3ac37e67e5a7d463267834", - "https://bcr.bazel.build/modules/boringssl/0.0.0-20211025-d4f1ab9/source.json": "323bafff99739f6aba35b69a84f0bc04ddb4540a46c1694355f60f073dff3001", - "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", - "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", - "https://bcr.bazel.build/modules/gflags/2.2.2/MODULE.bazel": "ba6502c3fee189734f359454db8a49b7c08afd7271b32e7c6fc38c2d2e1edbeb", - "https://bcr.bazel.build/modules/gflags/2.2.2/source.json": "b06d93702e18b5d75a69d53464c37ef5c2a9b4e237a8d4f2bf0217b3b0af2bee", - "https://bcr.bazel.build/modules/glog/0.5.0/MODULE.bazel": "f5e4f5ae1c0642c84a06a68c5428a576b28bac2cd1dfa3faf5b6d683c69007a4", - "https://bcr.bazel.build/modules/glog/0.5.0/source.json": "241565699f6a5428189e090297f6f9742259e791908cbace6e9f0eeb8104bd9e", - "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", - "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", - "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", - "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/source.json": "41e9e129f80d8c8bf103a7acc337b76e54fad1214ac0a7084bf24f4cd924b8b4", - "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", - "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", - "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d", - "https://bcr.bazel.build/modules/libevent/2.1.12/MODULE.bazel": "not found", - "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", - "https://bcr.bazel.build/modules/libunwind/1.8.1/MODULE.bazel": "d9b947b15135786aed51671c22872a93e2637c74caa20e94f2061fd3b1efd6c3", - "https://bcr.bazel.build/modules/libunwind/1.8.1/source.json": "2b4e58e2fa6ed7204862ab1a4e5978e4107340884c3217596ff7910b8dfae919", - "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", - "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", - "https://bcr.bazel.build/modules/platforms/0.0.11/source.json": "f7e188b79ebedebfe75e9e1d098b8845226c7992b307e28e1496f23112e8fc29", - "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", - "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", - "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", - "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", - "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", - "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", - "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", - "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", - "https://bcr.bazel.build/modules/protobuf/27.3/MODULE.bazel": "d94898cbf9d6d25c0edca2521211413506b68a109a6b01776832ed25154d23d7", - "https://bcr.bazel.build/modules/protobuf/27.3/source.json": "d6fdc641a99c600df6eb0fa5b99879ca497dbcf6fd1287372576a83f82dd93b6", - "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", - "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", - "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/source.json": "be4789e951dd5301282729fe3d4938995dc4c1a81c2ff150afc9f1b0504c6022", - "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", - "https://bcr.bazel.build/modules/re2/2023-09-01/source.json": "e044ce89c2883cd957a2969a43e79f7752f9656f6b20050b62f90ede21ec6eb4", - "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", - "https://bcr.bazel.build/modules/rules_cc/0.0.12/MODULE.bazel": "28259f5cb8dd72d7503eb4fd1f9c89ab13759a438b1b78cb9655568d8566d7f9", - "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", - "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", - "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", - "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", - "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", - "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", - "https://bcr.bazel.build/modules/rules_cc/0.1.1/source.json": "d61627377bd7dd1da4652063e368d9366fc9a73920bfa396798ad92172cf645c", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.12.0/MODULE.bazel": "d850fab025ce79a845077035861034393f1cc1efc1d9d58d766272a26ba67def", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.12.0/source.json": "c97ddc022179fe30d1a9b94425d1e56d0a633f72332c55463e584a52ce1b38ac", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.8.0/MODULE.bazel": "e9b4fe0b66c8a7e57bd385a304ea812dcaac3b69b762e2346c808cde26b8cb8d", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", - "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", - "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", - "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", - "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", - "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", - "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", - "https://bcr.bazel.build/modules/rules_jvm_external/5.1/source.json": "5abb45cc9beb27b77aec6a65a11855ef2b55d95dfdc358e9f312b78ae0ba32d5", - "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", - "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", - "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", - "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", - "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", - "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c", - "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", - "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", - "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/source.json": "d57902c052424dfda0e71646cb12668d39c4620ee0544294d9d941e7d12bc3a9", - "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", - "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", - "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", - "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", - "https://bcr.bazel.build/modules/rules_python/0.25.0/source.json": "c45006984eeaa18ad14c006091b264bff620c41952e9184edfe225ea95c3f986", - "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", - "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", - "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", - "https://bcr.bazel.build/modules/stardoc/0.5.3/source.json": "cd53fe968dc8cd98197c052db3db6d82562960c87b61e7a90ee96f8e4e0dda97", - "https://bcr.bazel.build/modules/thrift/0.21.0/MODULE.bazel": "not found", - "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", - "https://bcr.bazel.build/modules/xz/5.4.5.bcr.5/MODULE.bazel": "b93d7035ac14c900dfdf7624e42cfcbfc34415aa8d3c83a6f225867c48d28dad", - "https://bcr.bazel.build/modules/xz/5.4.5.bcr.5/source.json": "30c4e5c856087a60d92e2522eafd316c0661671f4478ca94c6b7bd877010210a", - "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", - "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", - "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", - "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", - "https://raw.githubusercontent.com/secretflow/bazel-registry/main/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", - "https://raw.githubusercontent.com/secretflow/bazel-registry/main/modules/leveldb/1.23/MODULE.bazel": "450b5dafff03fd68d2c57544e0cdca1411e9cf42ae599e31fbbbfdf1ddea92b3", - "https://raw.githubusercontent.com/secretflow/bazel-registry/main/modules/leveldb/1.23/source.json": "82a078a44ec4a6c299fe108e87b6d7a0ce56dd46206bfc69495001087d7e2dfb", - "https://raw.githubusercontent.com/secretflow/bazel-registry/main/modules/openssl/3.3.2.bcr.1/MODULE.bazel": "e26204291f98123dd57567b1b558518777787b488d946fbabe181249e1336029", - "https://raw.githubusercontent.com/secretflow/bazel-registry/main/modules/openssl/3.3.2.bcr.1/source.json": "ae3d5c57f40316cdd5d017a5fdc280d5980119556ec61000c0d7d9276b1d8610" - }, - "selectedYankedVersions": {}, - "moduleExtensions": { - "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { - "general": { - "bzlTransitiveDigest": "7ii+gFxWSxHhQPrBxfMEHhtrGvHmBTvsh+KOyGunP/s=", - "usagesDigest": "lTon/N84OKzwOBzHgfdF8vJqtH0gCUX9FqtuRj2zj44=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_apple_cc": { - "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf", - "attributes": {} - }, - "local_config_apple_cc_toolchains": { - "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf_toolchains", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support~", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_foreign_cc~//foreign_cc:extensions.bzl%tools": { - "general": { - "bzlTransitiveDigest": "f6RBwbUKS3C3FNQJAYhIWLiHaTLQDr1nKJidK7iQTBg=", - "usagesDigest": "rcono0M8vAWDAJxsr93rRa5vX8s73ZexMVAKXNZRFTg=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "cmake-3.23.2-linux-aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-aarch64.tar.gz" - ], - "sha256": "f2654bf780b53f170bbbec44d8ac67d401d24788e590faa53036a89476efa91e", - "strip_prefix": "cmake-3.23.2-linux-aarch64", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" - } - }, - "rules_foreign_cc_framework_toolchain_macos": { - "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", - "ruleClassName": "framework_toolchain_repository", - "attributes": { - "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:macos_commands.bzl", - "exec_compatible_with": [ - "@platforms//os:macos" - ] - } - }, - "ninja_1.12.1_win": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip" - ], - "sha256": "f550fec705b6d6ff58f2db3c374c2277a37691678d6aba463adcbb129108467a", - "strip_prefix": "", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja.exe\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" - } - }, - "gnumake_src": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "sha256": "dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3", - "strip_prefix": "make-4.4.1", - "urls": [ - "https://mirror.bazel.build/ftpmirror.gnu.org/gnu/make/make-4.4.1.tar.gz", - "http://ftpmirror.gnu.org/gnu/make/make-4.4.1.tar.gz" - ] - } - }, - "gettext_runtime": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "\ncc_import(\n name = \"gettext_runtime\",\n shared_library = \"bin/libintl-8.dll\",\n visibility = [\"//visibility:public\"],\n)\n ", - "sha256": "1f4269c0e021076d60a54e98da6f978a3195013f6de21674ba0edbc339c5b079", - "urls": [ - "https://mirror.bazel.build/download.gnome.org/binaries/win64/dependencies/gettext-runtime_0.18.1.1-2_win64.zip", - "https://download.gnome.org/binaries/win64/dependencies/gettext-runtime_0.18.1.1-2_win64.zip" - ] - } - }, - "cmake_src": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "sha256": "f316b40053466f9a416adf981efda41b160ca859e97f6a484b447ea299ff26aa", - "strip_prefix": "cmake-3.23.2", - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2.tar.gz" - ], - "patches": [ - "@@rules_foreign_cc~//toolchains:cmake-c++11.patch" - ] - } - }, - "bazel_skylib": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz" - ], - "sha256": "f7be3474d42aae265405a592bb7da8e171919d74c16f082a5457840f06054728" - } - }, - "cmake-3.23.2-macos-universal": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-macos-universal.tar.gz" - ], - "sha256": "853a0f9af148c5ef47282ffffee06c4c9f257be2635936755f39ca13c3286c88", - "strip_prefix": "cmake-3.23.2-macos-universal/CMake.app/Contents", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" - } - }, - "ninja_1.12.1_linux-aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux-aarch64.zip" - ], - "sha256": "5c25c6570b0155e95fce5918cb95f1ad9870df5768653afe128db822301a05a1", - "strip_prefix": "", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" - } - }, - "meson_src": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "exports_files([\"meson.py\"])\n\nfilegroup(\n name = \"runtime\",\n srcs = glob([\"mesonbuild/**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "sha256": "567e533adf255de73a2de35049b99923caf872a455af9ce03e01077e0d384bed", - "strip_prefix": "meson-1.5.1", - "urls": [ - "https://mirror.bazel.build/github.com/mesonbuild/meson/releases/download/1.5.1/meson-1.5.1.tar.gz", - "https://github.com/mesonbuild/meson/releases/download/1.5.1/meson-1.5.1.tar.gz" - ] - } - }, - "rules_foreign_cc_framework_toolchain_freebsd": { - "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", - "ruleClassName": "framework_toolchain_repository", - "attributes": { - "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:freebsd_commands.bzl", - "exec_compatible_with": [ - "@platforms//os:freebsd" - ] - } - }, - "rules_foreign_cc_framework_toolchain_linux": { - "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", - "ruleClassName": "framework_toolchain_repository", - "attributes": { - "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:linux_commands.bzl", - "exec_compatible_with": [ - "@platforms//os:linux" - ] - } - }, - "rules_python": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", - "strip_prefix": "rules_python-0.23.1", - "url": "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.23.1.tar.gz" - } - }, - "ninja_1.12.1_mac": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-mac.zip" - ], - "sha256": "89a287444b5b3e98f88a945afa50ce937b8ffd1dcc59c555ad9b1baf855298c9", - "strip_prefix": "", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" - } - }, - "pkgconfig_src": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "sha256": "6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591", - "strip_prefix": "pkg-config-0.29.2", - "patches": [ - "@@rules_foreign_cc~//toolchains:pkgconfig-detectenv.patch", - "@@rules_foreign_cc~//toolchains:pkgconfig-makefile-vc.patch", - "@@rules_foreign_cc~//toolchains:pkgconfig-builtin-glib-int-conversion.patch" - ], - "urls": [ - "https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz", - "https://mirror.bazel.build/pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" - ] - } - }, - "ninja_build_src": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "integrity": "sha256-ghvf9Io/aDvEuztvC1/nstZHz2XVKutjMoyRpsbfKFo=", - "strip_prefix": "ninja-1.12.1", - "urls": [ - "https://mirror.bazel.build/github.com/ninja-build/ninja/archive/v1.12.1.tar.gz", - "https://github.com/ninja-build/ninja/archive/v1.12.1.tar.gz" - ] - } - }, - "glib_src": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "\ncc_import(\n name = \"msvc_hdr\",\n hdrs = [\"msvc_recommended_pragmas.h\"],\n visibility = [\"//visibility:public\"],\n)\n ", - "sha256": "bc96f63112823b7d6c9f06572d2ad626ddac7eb452c04d762592197f6e07898e", - "strip_prefix": "glib-2.26.1", - "urls": [ - "https://mirror.bazel.build/download.gnome.org/sources/glib/2.26/glib-2.26.1.tar.gz", - "https://download.gnome.org/sources/glib/2.26/glib-2.26.1.tar.gz" - ] - } - }, - "ninja_1.12.1_linux": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip" - ], - "sha256": "6f98805688d19672bd699fbbfa2c2cf0fc054ac3df1f0e6a47664d963d530255", - "strip_prefix": "", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" - } - }, - "cmake-3.23.2-windows-x86_64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-windows-x86_64.zip" - ], - "sha256": "2329387f3166b84c25091c86389fb891193967740c9bcf01e7f6d3306f7ffda0", - "strip_prefix": "cmake-3.23.2-windows-x86_64", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake.exe\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake.exe\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" - } - }, - "glib_runtime": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "\nexports_files(\n [\n \"bin/libgio-2.0-0.dll\",\n \"bin/libglib-2.0-0.dll\",\n \"bin/libgmodule-2.0-0.dll\",\n \"bin/libgobject-2.0-0.dll\",\n \"bin/libgthread-2.0-0.dll\",\n ],\n visibility = [\"//visibility:public\"],\n)\n ", - "sha256": "88d857087e86f16a9be651ee7021880b3f7ba050d34a1ed9f06113b8799cb973", - "urls": [ - "https://mirror.bazel.build/download.gnome.org/binaries/win64/glib/2.26/glib_2.26.1-1_win64.zip", - "https://download.gnome.org/binaries/win64/glib/2.26/glib_2.26.1-1_win64.zip" - ] - } - }, - "rules_foreign_cc_framework_toolchains": { - "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", - "ruleClassName": "framework_toolchain_repository_hub", - "attributes": {} - }, - "bazel_features": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "ba1282c1aa1d1fffdcf994ab32131d7c7551a9bc960fbf05f42d55a1b930cbfb", - "strip_prefix": "bazel_features-1.15.0", - "url": "https://github.com/bazel-contrib/bazel_features/releases/download/v1.15.0/bazel_features-v1.15.0.tar.gz" - } - }, - "ninja_1.12.1_toolchains": { - "bzlFile": "@@rules_foreign_cc~//toolchains:prebuilt_toolchains_repository.bzl", - "ruleClassName": "prebuilt_toolchains_repository", - "attributes": { - "repos": { - "ninja_1.12.1_linux": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux" - ], - "ninja_1.12.1_linux-aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:linux" - ], - "ninja_1.12.1_mac": [ - "@platforms//cpu:x86_64", - "@platforms//os:macos" - ], - "ninja_1.12.1_mac_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:macos" - ], - "ninja_1.12.1_win": [ - "@platforms//cpu:x86_64", - "@platforms//os:windows" - ] - }, - "tool": "ninja" - } - }, - "glib_dev": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file_content": "\nload(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\ncc_import(\n name = \"glib_dev\",\n hdrs = glob([\"include/**\"]),\n shared_library = \"@glib_runtime//:bin/libglib-2.0-0.dll\",\n visibility = [\"//visibility:public\"],\n)\n ", - "sha256": "bdf18506df304d38be98a4b3f18055b8b8cca81beabecad0eece6ce95319c369", - "urls": [ - "https://mirror.bazel.build/download.gnome.org/binaries/win64/glib/2.26/glib-dev_2.26.1-1_win64.zip", - "https://download.gnome.org/binaries/win64/glib/2.26/glib-dev_2.26.1-1_win64.zip" - ] - } - }, - "cmake_3.23.2_toolchains": { - "bzlFile": "@@rules_foreign_cc~//toolchains:prebuilt_toolchains_repository.bzl", - "ruleClassName": "prebuilt_toolchains_repository", - "attributes": { - "repos": { - "cmake-3.23.2-linux-aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:linux" - ], - "cmake-3.23.2-linux-x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux" - ], - "cmake-3.23.2-macos-universal": [ - "@platforms//os:macos" - ], - "cmake-3.23.2-windows-i386": [ - "@platforms//cpu:x86_32", - "@platforms//os:windows" - ], - "cmake-3.23.2-windows-x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:windows" - ] - }, - "tool": "cmake" - } - }, - "cmake-3.23.2-windows-i386": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-windows-i386.zip" - ], - "sha256": "6a4fcd6a2315b93cb23c93507efccacc30c449c2bf98f14d6032bb226c582e07", - "strip_prefix": "cmake-3.23.2-windows-i386", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake.exe\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake.exe\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" - } - }, - "cmake-3.23.2-linux-x86_64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-x86_64.tar.gz" - ], - "sha256": "aaced6f745b86ce853661a595bdac6c5314a60f8181b6912a0a4920acfa32708", - "strip_prefix": "cmake-3.23.2-linux-x86_64", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_bin\",\n srcs = [\"bin/cmake\"],\n)\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"**/* *\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n env = {\"CMAKE\": \"$(execpath :cmake_bin)\"},\n tools = [\":cmake_bin\"],\n)\n" - } - }, - "ninja_1.12.1_mac_aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-mac.zip" - ], - "sha256": "89a287444b5b3e98f88a945afa50ce937b8ffd1dcc59c555ad9b1baf855298c9", - "strip_prefix": "", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" - } - }, - "rules_foreign_cc_framework_toolchain_windows": { - "bzlFile": "@@rules_foreign_cc~//foreign_cc/private/framework:toolchain.bzl", - "ruleClassName": "framework_toolchain_repository", - "attributes": { - "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:windows_commands.bzl", - "exec_compatible_with": [ - "@platforms//os:windows" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_foreign_cc~", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_foreign_cc~", - "rules_foreign_cc", - "rules_foreign_cc~" - ] - ] - } - }, - "@@rules_python~//python/extensions:python.bzl%python": { - "general": { - "bzlTransitiveDigest": "Kf/7zZzswWR1HLTsIkowkR+hnYBf2occRd9uW0D1KME=", - "usagesDigest": "M8/NqQuC9d+paxu5fLyiB29Pv9UCmhBJfvdJsaVw824=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "python_3_11_s390x-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "e477f0749161f9aa7887964f089d9460a539f6b4a8fdab5166f898210e1a87a4", - "patches": [], - "platform": "s390x-unknown-linux-gnu", - "python_version": "3.11.4", - "release_filename": "20230726/cpython-3.11.4+20230726-s390x-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-s390x-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_aarch64-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "2e84fc53f4e90e11963281c5c871f593abcb24fc796a50337fa516be99af02fb", - "patches": [], - "platform": "aarch64-unknown-linux-gnu", - "python_version": "3.11.4", - "release_filename": "20230726/cpython-3.11.4+20230726-aarch64-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-aarch64-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_aarch64-apple-darwin": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "cb6d2948384a857321f2aa40fa67744cd9676a330f08b6dad7070bda0b6120a4", - "patches": [], - "platform": "aarch64-apple-darwin", - "python_version": "3.11.4", - "release_filename": "20230726/cpython-3.11.4+20230726-aarch64-apple-darwin-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-aarch64-apple-darwin-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_9": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "toolchain_aliases", - "attributes": { - "python_version": "3.9.17", - "user_repository_name": "python_3_9" - } - }, - "pythons_hub": { - "bzlFile": "@@rules_python~//python/extensions/private:pythons_hub.bzl", - "ruleClassName": "hub_repo", - "attributes": { - "default_python_version": "3.11", - "toolchain_prefixes": [ - "_0000_python_3_9_", - "_0001_python_3_11_" - ], - "toolchain_python_versions": [ - "3.9", - "3.11" - ], - "toolchain_set_python_version_constraints": [ - "True", - "False" - ], - "toolchain_user_repository_names": [ - "python_3_9", - "python_3_11" - ] - } - }, - "python_3_11_x86_64-pc-windows-msvc": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "878614c03ea38538ae2f758e36c85d2c0eb1eaaca86cd400ff8c76693ee0b3e1", - "patches": [], - "platform": "x86_64-pc-windows-msvc", - "python_version": "3.11.4", - "release_filename": "20230726/cpython-3.11.4+20230726-x86_64-pc-windows-msvc-shared-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-x86_64-pc-windows-msvc-shared-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_9_aarch64-apple-darwin": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "73dbe2d702210b566221da9265acc274ba15275c5d0d1fa327f44ad86cde9aa1", - "patches": [], - "platform": "aarch64-apple-darwin", - "python_version": "3.9.17", - "release_filename": "20230726/cpython-3.9.17+20230726-aarch64-apple-darwin-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-aarch64-apple-darwin-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_9_x86_64-pc-windows-msvc": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "9b9a1e21eff29dcf043cea38180cf8ca3604b90117d00062a7b31605d4157714", - "patches": [], - "platform": "x86_64-pc-windows-msvc", - "python_version": "3.9.17", - "release_filename": "20230726/cpython-3.9.17+20230726-x86_64-pc-windows-msvc-shared-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-x86_64-pc-windows-msvc-shared-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_9_ppc64le-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "c591a28d943dce5cf9833e916125fdfbeb3120270c4866ee214493ccb5b83c3c", - "patches": [], - "platform": "ppc64le-unknown-linux-gnu", - "python_version": "3.9.17", - "release_filename": "20230726/cpython-3.9.17+20230726-ppc64le-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-ppc64le-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_9_aarch64-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "b77012ddaf7e0673e4aa4b1c5085275a06eee2d66f33442b5c54a12b62b96cbe", - "patches": [], - "platform": "aarch64-unknown-linux-gnu", - "python_version": "3.9.17", - "release_filename": "20230726/cpython-3.9.17+20230726-aarch64-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-aarch64-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "toolchain_aliases", - "attributes": { - "python_version": "3.11.4", - "user_repository_name": "python_3_11" - } - }, - "python_3_11_ppc64le-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "df7b92ed9cec96b3bb658fb586be947722ecd8e420fb23cee13d2e90abcfcf25", - "patches": [], - "platform": "ppc64le-unknown-linux-gnu", - "python_version": "3.11.4", - "release_filename": "20230726/cpython-3.11.4+20230726-ppc64le-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-ppc64le-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_x86_64-apple-darwin": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "47e1557d93a42585972772e82661047ca5f608293158acb2778dccf120eabb00", - "patches": [], - "platform": "x86_64-apple-darwin", - "python_version": "3.11.4", - "release_filename": "20230726/cpython-3.11.4+20230726-x86_64-apple-darwin-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-x86_64-apple-darwin-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_versions": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "multi_toolchain_aliases", - "attributes": { - "python_versions": { - "3.11": "python_3_11", - "3.9": "python_3_9" - } - } - }, - "python_3_9_x86_64-apple-darwin": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "dfe1bea92c94b9cb779288b0b06e39157c5ff7e465cdd24032ac147c2af485c0", - "patches": [], - "platform": "x86_64-apple-darwin", - "python_version": "3.9.17", - "release_filename": "20230726/cpython-3.9.17+20230726-x86_64-apple-darwin-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-x86_64-apple-darwin-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_9_x86_64-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "26c4a712b4b8e11ed5c027db5654eb12927c02da4857b777afb98f7a930ce637", - "patches": [], - "platform": "x86_64-unknown-linux-gnu", - "python_version": "3.9.17", - "release_filename": "20230726/cpython-3.9.17+20230726-x86_64-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-x86_64-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_9_s390x-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "01454d7cc7c9c2fccde42ba868c4f372eaaafa48049d49dd94c9cf2875f497e6", - "patches": [], - "platform": "s390x-unknown-linux-gnu", - "python_version": "3.9.17", - "release_filename": "20230726/cpython-3.9.17+20230726-s390x-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.9.17+20230726-s390x-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_x86_64-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "e26247302bc8e9083a43ce9e8dd94905b40d464745b1603041f7bc9a93c65d05", - "patches": [], - "platform": "x86_64-unknown-linux-gnu", - "python_version": "3.11.4", - "release_filename": "20230726/cpython-3.11.4+20230726-x86_64-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20230726/cpython-3.11.4+20230726-x86_64-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python~", - "bazel_tools", - "bazel_tools" - ] - ] - } - } - } -} From 2c21f494adc3dfa3f431a5c0514c78e85cfde25e Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Sun, 5 Oct 2025 11:36:47 +0530 Subject: [PATCH 26/49] Fixed bugs in local collection cache and collection refresh logic --- .../multithreaded_couchbase_client.cpp | 314 ++++++++---- src/brpc/couchbase.cpp | 482 ++++++++++-------- src/brpc/couchbase.h | 13 +- 3 files changed, 483 insertions(+), 326 deletions(-) diff --git a/example/couchbase_c++/multithreaded_couchbase_client.cpp b/example/couchbase_c++/multithreaded_couchbase_client.cpp index 1065543af0..978c0b5137 100644 --- a/example/couchbase_c++/multithreaded_couchbase_client.cpp +++ b/example/couchbase_c++/multithreaded_couchbase_client.cpp @@ -19,12 +19,10 @@ #include #include #include -#include -#include - #include #include #include +#include // ANSI color codes #define GREEN "\033[32m" @@ -34,166 +32,270 @@ #define CYAN "\033[36m" #define RESET "\033[0m" -DEFINE_string(server, "127.0.0.1:11210", "IP Address of server"); -DEFINE_string(connection_type, "single", "Connection type"); -DEFINE_int32(operations_per_thread, 50, - "Number of operations each thread should perform"); -DEFINE_int32(sleep_ms, 100, "Sleep time between operations in milliseconds"); - -const int NUM_THREADS = 16; -const int THREADS_PER_BUCKET = 4; +const int NUM_THREADS = 20; +const int THREADS_PER_BUCKET = 5; // Simple global config struct { - std::string username; - std::string password; - std::vector bucket_names; - std::vector collection_names; + std::string username = "Administrator"; + std::string password = "password"; + std::vector bucket_names = {"testing0", "testing1", "testing2", "testing3"}; } g_config; -// Thread arguments +// Simple thread statistics +struct ThreadStats { + std::atomic operations_attempted{0}; + std::atomic operations_successful{0}; + std::atomic operations_failed{0}; + + void reset() { + operations_attempted = 0; + operations_successful = 0; + operations_failed = 0; + } +}; + +// Global statistics +struct GlobalStats { + ThreadStats total; + std::vector per_thread_stats; + + GlobalStats() : per_thread_stats(NUM_THREADS) {} + + void aggregate_stats() { + total.reset(); + for (const auto& stats : per_thread_stats) { + total.operations_attempted += stats.operations_attempted.load(); + total.operations_successful += stats.operations_successful.load(); + total.operations_failed += stats.operations_failed.load(); + } + } +} g_stats; + +// Simple thread arguments struct ThreadArgs { int thread_id; int bucket_id; std::string bucket_name; + ThreadStats* stats; }; -// Simple operation function using high-level API -bool perform_operation(brpc::CouchbaseOperations& couchbase_ops, - const std::string& key, - const std::string& collection = "_default") { - // Simple ADD operation - std::string value = butil::string_printf( - R"({"thread_id": %llu, "timestamp": %lld})", - (unsigned long long)bthread_self(), butil::gettimeofday_us()); - - brpc::CouchbaseOperations::Result result = - couchbase_ops.Add(key, value, collection); - return result.success; +// Simple CRUD operations on default collection +void perform_crud_operations_default(brpc::CouchbaseOperations& couchbase_ops, + const std::string& base_key, + ThreadStats* stats) { + std::string key = base_key + "_default"; + std::string value = butil::string_printf(R"({"thread_id": %d, "collection": "default"})", + (int)bthread_self()); + + stats->operations_attempted++; + + // UPSERT + brpc::CouchbaseOperations::Result result = couchbase_ops.Upsert(key, value); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + return; + } + + stats->operations_attempted++; + + // GET + result = couchbase_ops.Get(key); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + return; + } + + stats->operations_attempted++; + + // DELETE + result = couchbase_ops.Delete(key); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + } +} + +// Simple CRUD operations on col1 collection +void perform_crud_operations_col1(brpc::CouchbaseOperations& couchbase_ops, + const std::string& base_key, + ThreadStats* stats) { + std::string key = base_key + "_col1"; + std::string value = butil::string_printf(R"({"thread_id": %d, "collection": "col1"})", + (int)bthread_self()); + + stats->operations_attempted++; + + // UPSERT + brpc::CouchbaseOperations::Result result = couchbase_ops.Upsert(key, value, "col1"); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + return; + } + + stats->operations_attempted++; + + // GET + result = couchbase_ops.Get(key, "col1"); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + return; + } + + stats->operations_attempted++; + + // DELETE + result = couchbase_ops.Delete(key, "col1"); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + } } -// Thread worker function using high-level API +// Simple thread worker function void* thread_worker(void* arg) { ThreadArgs* args = static_cast(arg); - LOG(INFO) << "Thread " << args->thread_id << " starting on bucket " - << args->bucket_name; + std::cout << CYAN << "Thread " << args->thread_id << " starting on bucket " + << args->bucket_name << RESET << std::endl; // Create CouchbaseOperations instance for this thread brpc::CouchbaseOperations couchbase_ops; - // Authentication using high-level method + // Authentication brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( - g_config.username, g_config.password, FLAGS_server, false, ""); + g_config.username, g_config.password, "127.0.0.1:11210", false, ""); + if (!auth_result.success) { - LOG(ERROR) << "Thread " << args->thread_id << ": Auth failed - " - << auth_result.error_message; + std::cout << RED << "Thread " << args->thread_id << ": Auth failed - " + << auth_result.error_message << RESET << std::endl; return NULL; } - // Select bucket using high-level method - brpc::CouchbaseOperations::Result bucket_result = - couchbase_ops.SelectBucket(args->bucket_name); + // Select bucket + brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.SelectBucket(args->bucket_name); + if (!bucket_result.success) { - LOG(ERROR) << "Thread " << args->thread_id << ": Bucket selection failed - " - << bucket_result.error_message; + std::cout << RED << "Thread " << args->thread_id << ": Bucket selection failed - " + << bucket_result.error_message << RESET << std::endl; return NULL; } - LOG(INFO) << "Thread " << args->thread_id << " connected to bucket " - << args->bucket_name; - - // Perform operations - int success_count = 0; - for (int i = 0; i < FLAGS_operations_per_thread; ++i) { - std::string key = - butil::string_printf("thread_%d_op_%d", args->thread_id, i); - - // Choose collection if available, otherwise use default - std::string collection = "_default"; - if (!g_config.collection_names.empty()) { - collection = - g_config.collection_names[i % g_config.collection_names.size()]; - } - - if (perform_operation(couchbase_ops, key, collection)) { - success_count++; - } - - if (FLAGS_sleep_ms > 0) { - bthread_usleep(FLAGS_sleep_ms * 1000); - } + std::cout << GREEN << "Thread " << args->thread_id << " connected to bucket " + << args->bucket_name << RESET << std::endl; + + // Perform operations - 10 times on default collection, 10 times on col1 collection + for (int i = 0; i < 10; ++i) { + std::string base_key = butil::string_printf("thread_%d_op_%d", args->thread_id, i); + + // CRUD operations on default collection + perform_crud_operations_default(couchbase_ops, base_key, args->stats); + + // CRUD operations on col1 collection + perform_crud_operations_col1(couchbase_ops, base_key, args->stats); + + // Small delay between operations + bthread_usleep(10000); // 10ms } - LOG(INFO) << "Thread " << args->thread_id << " completed: " << success_count - << "/" << FLAGS_operations_per_thread << " operations successful"; + int successful = args->stats->operations_successful.load(); + int attempted = args->stats->operations_attempted.load(); + int failed = args->stats->operations_failed.load(); + + std::cout << GREEN << "Thread " << args->thread_id << " completed: " + << successful << "/" << attempted << " operations successful, " + << failed << " failed" << RESET << std::endl; return NULL; } -// Simple config function -void get_config() { - std::cout << CYAN << "=== Simple Multithreaded Couchbase Client ===" << RESET - << std::endl; - - std::cout << "Username: "; - std::cin >> g_config.username; - std::cout << "Password: "; - std::cin >> g_config.password; - - std::cout << "Enter 4 bucket names:" << std::endl; - for (int i = 0; i < 4; ++i) { - std::string bucket; - std::cout << "Bucket " << (i + 1) << ": "; - std::cin >> bucket; - g_config.bucket_names.push_back(bucket); - } - - int num_collections; - std::cout << "Number of collections (0 for none): "; - std::cin >> num_collections; - - for (int i = 0; i < num_collections; ++i) { - std::string collection; - std::cout << "Collection " << (i + 1) << ": "; - std::cin >> collection; - g_config.collection_names.push_back(collection); +// Print simple statistics +void print_stats() { + g_stats.aggregate_stats(); + + std::cout << std::endl; + std::cout << CYAN << "=== TEST RESULTS ===" << RESET << std::endl; + + int total_attempted = g_stats.total.operations_attempted.load(); + int total_successful = g_stats.total.operations_successful.load(); + int total_failed = g_stats.total.operations_failed.load(); + + double success_rate = total_attempted > 0 ? (double)total_successful / total_attempted * 100.0 : 0.0; + + std::cout << GREEN << "Overall Performance:" << RESET << std::endl; + std::cout << " Total Operations: " << total_attempted << std::endl; + std::cout << " Successful: " << total_successful << " (" << success_rate << "%)" << std::endl; + std::cout << " Failed: " << total_failed << std::endl; + std::cout << std::endl; + + // Per-thread breakdown + std::cout << YELLOW << "Per-Thread Performance:" << RESET << std::endl; + for (int i = 0; i < NUM_THREADS; ++i) { + const auto& stats = g_stats.per_thread_stats[i]; + int attempted = stats.operations_attempted.load(); + int successful = stats.operations_successful.load(); + int failed = stats.operations_failed.load(); + + std::cout << " Thread " << i << ": " << attempted << " ops, " + << successful << " success, " << failed << " failed" << std::endl; } + std::cout << std::endl; } int main(int argc, char* argv[]) { - gflags::ParseCommandLineFlags(&argc, &argv, true); - - std::cout << GREEN << "Starting 16 bthreads (4 per bucket)" << RESET - << std::endl; + std::cout << GREEN << "Starting Simple Multithreaded Couchbase Client" << RESET << std::endl; + std::cout << YELLOW << "20 threads: 5 per bucket (testing0, testing1, testing2, testing3)" << RESET << std::endl; + std::cout << BLUE << "Each thread performs CRUD operations on default collection and col1 collection" << RESET << std::endl; - get_config(); - - // Create bthreads + // Create threads and arguments std::vector threads(NUM_THREADS); std::vector args(NUM_THREADS); + // Assign threads to buckets for (int i = 0; i < NUM_THREADS; ++i) { args[i].thread_id = i; args[i].bucket_id = i / THREADS_PER_BUCKET; args[i].bucket_name = g_config.bucket_names[args[i].bucket_id]; + args[i].stats = &g_stats.per_thread_stats[i]; + } + + // Print thread assignments + std::cout << "Thread Assignments:" << RESET << std::endl; + for (int i = 0; i < NUM_THREADS; ++i) { + std::cout << "Thread " << i << " -> Bucket: " << args[i].bucket_name << std::endl; + } + std::cout << std::endl; - if (bthread_start_background(&threads[i], NULL, thread_worker, &args[i]) != - 0) { - LOG(ERROR) << "Failed to create thread " << i; + // Start all threads + for (int i = 0; i < NUM_THREADS; ++i) { + if (bthread_start_background(&threads[i], NULL, thread_worker, &args[i]) != 0) { + std::cout << RED << "Failed to create thread " << i << RESET << std::endl; return -1; } - - bthread_usleep(50000); // 50ms delay between thread starts } - std::cout << "All 16 threads started!" << std::endl; + std::cout << GREEN << "All 20 threads started!" << RESET << std::endl; - // Wait for all threads + // Wait for all threads to complete + std::cout << YELLOW << "Waiting for all threads to complete..." << RESET << std::endl; for (int i = 0; i < NUM_THREADS; ++i) { bthread_join(threads[i], NULL); } std::cout << GREEN << "All threads completed!" << RESET << std::endl; + + // Print statistics + print_stats(); + return 0; } diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index e60651c421..15418ed035 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -19,9 +19,16 @@ #include //for crc32 Vbucket_id +// Debug flag for enabling debug statements +static bool DBUG = true; // Set to true to enable debug logs + +// Debug print macro +#define DEBUG_PRINT(msg) do { if (DBUG) { std::cout << "[DEBUG] " << msg << std::endl; } } while(0) + #include "brpc/policy/couchbase_protocol.h" #include "brpc/proto_base.pb.h" #include "butil/logging.h" +#include #include "butil/macros.h" #include "butil/string_printf.h" #include "butil/sys_byteorder.h" @@ -78,29 +85,28 @@ bool brpc::CouchbaseMetadataTracking::get_manifest_to_collection_id( CouchbaseMetadataTracking::CollectionManifest* manifest, string scope, string collection, uint8_t* collection_id) { if (manifest == nullptr || collection_id == nullptr) { - LOG(ERROR) << "Invalid input: manifest or collection_id is null"; + DEBUG_PRINT("Invalid input: manifest or collection_id is null"); return false; } auto it1 = manifest->scope_to_collectionID_map.find(scope); if (it1 == manifest->scope_to_collectionID_map.end()) { - LOG(ERROR) << "Scope: " << scope << " not found in manifest"; + DEBUG_PRINT("Scope: " << scope << " not found in manifest"); return false; } auto it2 = it1->second.find(collection); if (it2 == it1->second.end()) { - LOG(ERROR) << "Collection: " << collection - << " not found in scope: " << scope; + DEBUG_PRINT("Collection: " << collection << " not found in scope: " << scope); return false; } *collection_id = it2->second; return true; } -bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest( +bool CouchbaseMetadataTracking::json_to_collection_manifest( const string& json, CouchbaseMetadataTracking::CollectionManifest* manifest) { if (manifest == nullptr) { - LOG(ERROR) << "Invalid input: manifest is null"; + DEBUG_PRINT("Invalid input: manifest is null"); return false; } @@ -109,7 +115,7 @@ bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest( manifest->scope_to_collectionID_map.clear(); if (json.empty()) { - LOG(ERROR) << "JSON string is empty"; + DEBUG_PRINT("JSON string is empty"); return false; } @@ -118,12 +124,12 @@ bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest( document.Parse(json.c_str()); if (document.HasParseError()) { - LOG(ERROR) << "Failed to parse JSON: " << document.GetParseError(); + DEBUG_PRINT("Failed to parse JSON: " << document.GetParseError()); return false; } if (!document.IsObject()) { - LOG(ERROR) << "JSON root is not an object"; + DEBUG_PRINT("JSON root is not an object"); return false; } @@ -131,13 +137,13 @@ bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest( if (document.HasMember("uid") && document["uid"].IsString()) { manifest->uid = document["uid"].GetString(); } else { - LOG(ERROR) << "Missing or invalid 'uid' field in JSON"; + DEBUG_PRINT("Missing or invalid 'uid' field in JSON"); return false; } // Extract scopes if (!document.HasMember("scopes") || !document["scopes"].IsArray()) { - LOG(ERROR) << "Missing or invalid 'scopes' field in JSON"; + DEBUG_PRINT("Missing or invalid 'scopes' field in JSON"); return false; } @@ -146,21 +152,20 @@ bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest( const BUTIL_RAPIDJSON_NAMESPACE::Value& scope = scopes[i]; if (!scope.IsObject()) { - LOG(ERROR) << "Scope at index " << i << " is not an object"; + DEBUG_PRINT("Scope at index " << i << " is not an object"); return false; } // Extract scope name if (!scope.HasMember("name") || !scope["name"].IsString()) { - LOG(ERROR) << "Missing or invalid 'name' field in scope at index " << i; + DEBUG_PRINT("Missing or invalid 'name' field in scope at index " << i); return false; } string scope_name = scope["name"].GetString(); // Extract collections if (!scope.HasMember("collections") || !scope["collections"].IsArray()) { - LOG(ERROR) << "Missing or invalid 'collections' field in scope '" - << scope_name << "'"; + DEBUG_PRINT("Missing or invalid 'collections' field in scope '" << scope_name << "'"); return false; } @@ -172,23 +177,20 @@ bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest( const BUTIL_RAPIDJSON_NAMESPACE::Value& collection = collections[j]; if (!collection.IsObject()) { - LOG(ERROR) << "Collection at index " << j << " in scope '" << scope_name - << "' is not an object"; + DEBUG_PRINT("Collection at index " << j << " in scope '" << scope_name << "' is not an object"); return false; } // Extract collection name if (!collection.HasMember("name") || !collection["name"].IsString()) { - LOG(ERROR) << "Missing or invalid 'name' field in collection at index " - << j << " in scope '" << scope_name << "'"; + DEBUG_PRINT("Missing or invalid 'name' field in collection at index " << j << " in scope '" << scope_name << "'"); return false; } string collection_name = collection["name"].GetString(); // Extract collection uid (hex string) if (!collection.HasMember("uid") || !collection["uid"].IsString()) { - LOG(ERROR) << "Missing or invalid 'uid' field in collection '" - << collection_name << "' in scope '" << scope_name << "'"; + DEBUG_PRINT("Missing or invalid 'uid' field in collection '" << collection_name << "' in scope '" << scope_name << "'"); return false; } string collection_uid_str = collection["uid"].GetString(); @@ -199,16 +201,12 @@ bool brpc::CouchbaseMetadataTracking::json_to_collection_manifest( // Convert hex string to integer unsigned long uid_val = std::stoul(collection_uid_str, nullptr, 16); if (uid_val > 255) { - LOG(ERROR) << "Collection uid '" << collection_uid_str - << "' exceeds uint8_t range in collection '" - << collection_name << "' in scope '" << scope_name << "'"; + DEBUG_PRINT("Collection uid '" << collection_uid_str << "' exceeds uint8_t range in collection '" << collection_name << "' in scope '" << scope_name << "'"); return false; } collection_id = static_cast(uid_val); } catch (const std::exception& e) { - LOG(ERROR) << "Failed to parse collection uid '" << collection_uid_str - << "' as hex in collection '" << collection_name - << "' in scope '" << scope_name << "': " << e.what(); + DEBUG_PRINT("Failed to parse collection uid '" << collection_uid_str << "' as hex in collection '" << collection_name << "' in scope '" << scope_name << ": " << e.what()); return false; } @@ -305,7 +303,7 @@ void CouchbaseOperations::CouchbaseRequest::Clear() { // bool CouchbaseOperations::CouchbaseRequest::GetScopeId(const // butil::StringPiece& scope_name) { // if (scope_name.empty()) { -// LOG(ERROR) << "Empty scope name"; +// DEBUG_PRINT("Empty scope name"); // return false; // } // // Opcode 0xBC for Get Scope ID (see Collections.md) @@ -333,7 +331,7 @@ void CouchbaseOperations::CouchbaseRequest::Clear() { bool CouchbaseOperations::CouchbaseRequest::SelectBucketRequest( const butil::StringPiece& bucket_name) { if (bucket_name.empty()) { - LOG(ERROR) << "Empty bucket name"; + DEBUG_PRINT("Empty bucket name"); return false; } // construct the request header @@ -348,11 +346,11 @@ bool CouchbaseOperations::CouchbaseRequest::SelectBucketRequest( 0, 0}; if (_buf.append(&header, sizeof(header))) { - LOG(ERROR) << "Failed to append header to buffer"; + DEBUG_PRINT("Failed to append header to buffer"); return false; } if (_buf.append(bucket_name.data(), bucket_name.size())) { - LOG(ERROR) << "Failed to append bucket name to buffer"; + DEBUG_PRINT("Failed to append bucket name to buffer"); return false; } ++_pipelined_count; @@ -388,7 +386,7 @@ bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { if (!urandom || fread(raw_id, 1, CONNECTION_ID_SIZE, urandom) != CONNECTION_ID_SIZE) { if (urandom) fclose(urandom); - LOG(ERROR) << "Failed to generate random connection id"; + DEBUG_PRINT("Failed to generate random connection id"); return false; } fclose(urandom); @@ -426,15 +424,15 @@ bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { }; if (_buf.append(&header, sizeof(header))) { - LOG(ERROR) << "Failed to append Hello header to buffer"; + DEBUG_PRINT("Failed to append Hello header to buffer"); return false; } if (_buf.append(key.data(), key_len)) { - LOG(ERROR) << "Failed to append Hello JSON key to buffer"; + DEBUG_PRINT("Failed to append Hello JSON key to buffer"); return false; } if (_buf.append(reinterpret_cast(features), value_len)) { - LOG(ERROR) << "Failed to append Hello features to buffer"; + DEBUG_PRINT("Failed to append Hello features to buffer"); return false; } ++_pipelined_count; @@ -444,13 +442,13 @@ bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest( const butil::StringPiece& username, const butil::StringPiece& password) { if (username.empty() || password.empty()) { - LOG(ERROR) << "Empty username or password"; + DEBUG_PRINT("Empty username or password"); return false; } // insert the features to get enabled, calling function HelloRequest() will do // this. if (!HelloRequest()) { - LOG(ERROR) << "Failed to send HelloRequest for authentication"; + DEBUG_PRINT("Failed to send HelloRequest for authentication"); return false; } // Construct the request header @@ -478,7 +476,7 @@ bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest( auth_str.append(kPadding, sizeof(kPadding)); auth_str.append(password.data(), password.size()); if (_buf.append(auth_str.data(), auth_str.size())) { - LOG(ERROR) << "Failed to append auth string to buffer"; + DEBUG_PRINT("Failed to append auth string to buffer"); return false; } ++_pipelined_count; @@ -487,7 +485,7 @@ bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest( bool CouchbaseOperations::CouchbaseRequest::MergePartialFromCodedStream( ::google::protobuf::io::CodedInputStream* input) { - LOG(WARNING) << "You're not supposed to parse a CouchbaseRequest"; + DEBUG_PRINT("You're not supposed to parse a CouchbaseRequest"); // simple approach just making it work. butil::IOBuf tmp; @@ -523,7 +521,7 @@ bool CouchbaseOperations::CouchbaseRequest::MergePartialFromCodedStream( void CouchbaseOperations::CouchbaseRequest::SerializeWithCachedSizes( ::google::protobuf::io::CodedOutputStream* output) const { - LOG(WARNING) << "You're not supposed to serialize a CouchbaseRequest"; + DEBUG_PRINT("You're not supposed to serialize a CouchbaseRequest"); // simple approach just making it work. butil::IOBufAsZeroCopyInputStream wrapper(_buf); @@ -579,7 +577,7 @@ void CouchbaseOperations::CouchbaseResponse::Clear() {} bool CouchbaseOperations::CouchbaseResponse::MergePartialFromCodedStream( ::google::protobuf::io::CodedInputStream* input) { - LOG(WARNING) << "You're not supposed to parse a CouchbaseResponse"; + DEBUG_PRINT("You're not supposed to parse a CouchbaseResponse"); // simple approach just making it work. const void* data = NULL; @@ -593,7 +591,7 @@ bool CouchbaseOperations::CouchbaseResponse::MergePartialFromCodedStream( void CouchbaseOperations::CouchbaseResponse::SerializeWithCachedSizes( ::google::protobuf::io::CodedOutputStream* output) const { - LOG(WARNING) << "You're not supposed to serialize a CouchbaseResponse"; + DEBUG_PRINT("You're not supposed to serialize a CouchbaseResponse"); // simple approach just making it work. butil::IOBufAsZeroCopyInputStream wrapper(_buf); @@ -809,76 +807,42 @@ bool CouchbaseOperations::CouchbaseRequest::get_cached_or_fetch_collection_id( string collection_name, uint8_t* coll_id, brpc::CouchbaseMetadataTracking* metadata_tracking, brpc::Channel* channel, const string& server, const string& selected_bucket) { - if (collection_name.empty()) { - LOG(ERROR) << "Empty collection name"; - return false; - } - if (channel == nullptr) { - LOG(ERROR) << "No channel found, make sure to call Authenticate() first"; - return false; - } - if (server.empty()) { - LOG(ERROR) << "Server is empty, make sure to call Authenticate() first"; - return false; - } - if (selected_bucket.empty()) { - LOG(ERROR) << "No bucket selected, make sure to call SelectBucket() first"; - return false; - } - - brpc::CouchbaseMetadataTracking::CollectionManifest manifest; - //check if the server/bucket exists in the cached collection manifest - if (!metadata_tracking->get_bucket_to_collection_manifest( - server, selected_bucket, &manifest)) { - LOG(INFO) << "No cached collection manifest found for bucket " - << selected_bucket << " on server " << server - << ", fetching from server"; - // No cached manifest found, fetch from server - CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; - CouchbaseOperations::CouchbaseResponse temp_get_manifest_response; - brpc::Controller temp_cntl; - temp_get_manifest_request.GetCollectionManifest(); - channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, - &temp_get_manifest_response, NULL); - if (temp_cntl.Failed()) { - LOG(ERROR) << "Failed to get collection manifest for bucket " - << selected_bucket << " on server " << server << ": " - << temp_cntl.ErrorText(); + if (collection_name.empty()) { + DEBUG_PRINT("Empty collection name"); return false; } - string manifest_json; - if (!temp_get_manifest_response.PopManifest(&manifest_json)) { - LOG(ERROR) - << "Failed to parse response for collection Manifest in bucket " - << selected_bucket << " on server " << server << ": " - << temp_get_manifest_response.LastError(); + if (channel == nullptr) { + DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); return false; - } else { - // convert JSON to manifest structure - if (!metadata_tracking->json_to_collection_manifest(manifest_json, - &manifest)) { - LOG(ERROR) << "Failed to parse collection manifest JSON for bucket " - << selected_bucket << " on server " << server; - return false; - } - // Cache the collection manifest in global cache - if (!metadata_tracking->set_bucket_to_collection_manifest( - server, selected_bucket, manifest)) { - LOG(ERROR) << "Failed to cache collection ID for collection " - << collection_name << " in bucket " << selected_bucket - << " on server " << server; + } + if (server.empty()) { + DEBUG_PRINT("Server is empty, make sure to call Authenticate() first"); + return false; + } + if (selected_bucket.empty()) { + DEBUG_PRINT("No bucket selected, make sure to call SelectBucket() first"); + return false; + } + + brpc::CouchbaseMetadataTracking::CollectionManifest manifest; + //check if the server/bucket exists in the cached collection manifest + if (!metadata_tracking->get_bucket_to_collection_manifest( + server, selected_bucket, &manifest)) { + DEBUG_PRINT("No cached collection manifest found for bucket " << selected_bucket << " on server " << server << ", fetching from server"); + // No cached manifest found, fetch from server + if(!RefreshCollectionManifest(channel, server, selected_bucket)) { return false; } - //copy the data to the local cache of the CouchbaseOperations instance which called this. - (*local_collection_manifest_cache)[selected_bucket] = manifest; - if( !metadata_tracking->get_manifest_to_collection_id( - &manifest, "_default", collection_name, coll_id)) { - return false; + // local cache will also be updated in RefreshCollectionManifest + // get the reference to collectionID from local cache + if( !getLocalCachedCollectionId(selected_bucket, "_default", collection_name, coll_id)) { + // collectionID not found in the latest manifest fetched from server + return false; } - // Collection does exist and the collection id is stored in coll_id + //collectionID has been found in the latest manifest fetched from server and is stored in coll_id return true; } - } else { + else { // check if collection name to id mapping exists. if (!metadata_tracking->get_manifest_to_collection_id( &manifest, "_default", collection_name, coll_id)) { @@ -887,17 +851,18 @@ bool CouchbaseOperations::CouchbaseRequest::get_cached_or_fetch_collection_id( if(!RefreshCollectionManifest(channel, server, selected_bucket)) { return false; } - // get the reference to latest manifest fetched from server - if (!metadata_tracking->get_bucket_to_collection_manifest( - server, selected_bucket, &manifest)) { - return false; - } - // re-verify the collectionID in the latest manifest. - if( !metadata_tracking->get_manifest_to_collection_id( - &manifest, "_default", collection_name, coll_id)) { + // local cache will also be updated in RefreshCollectionManifest + // get the reference to collectionID from local cache + if( !getLocalCachedCollectionId(selected_bucket, "_default", collection_name, coll_id)) { + // collectionID not found in the latest manifest fetched from server return false; } + //collectionID has been found in the latest manifest fetched from server and is stored in coll_id + return true; } + // update the local cache with the manifest in global cache + (*local_collection_manifest_cache)[selected_bucket] = manifest; + // collectionID found in the cached manifest return true; } } @@ -906,46 +871,58 @@ bool CouchbaseOperations::CouchbaseRequest::get_cached_or_fetch_collection_id( bool CouchbaseOperations::CouchbaseRequest::GetRequest( const butil::StringPiece& key, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { + DEBUG_PRINT("GetRequest called with key: " << key << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. if(local_collection_manifest_cache->empty()){ + DEBUG_PRINT("Local collection manifest cache is empty in GetRequest"); // if local cache is empty, goto global cache or fetch from server if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + DEBUG_PRINT("Failed to get collection id from global cache or server in GetRequest"); return false; } } // check if the collection id is available in the local cache else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + DEBUG_PRINT("Collection id not found in local cache in GetRequest"); // if not check in the global cache or fetch from server if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + DEBUG_PRINT("Failed to get collection id from global cache or server in GetRequest"); return false; } } } + DEBUG_PRINT("GetRequest using coll_id: " << (int)coll_id); return GetOrDelete(policy::CB_BINARY_GET, key, coll_id); } bool CouchbaseOperations::CouchbaseRequest::DeleteRequest( const butil::StringPiece& key, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { + DEBUG_PRINT("DeleteRequest called with key: " << key << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. if(local_collection_manifest_cache->empty()){ + DEBUG_PRINT("Local collection manifest cache is empty in DeleteRequest"); // if local cache is empty, goto global cache or fetch from server if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + DEBUG_PRINT("Failed to get collection id from global cache or server in DeleteRequest"); return false; } } // check if the collection id is available in the local cache else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + DEBUG_PRINT("Collection id not found in local cache in DeleteRequest"); // if not check in the global cache or fetch from server if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + DEBUG_PRINT("Failed to get collection id from global cache or server in DeleteRequest"); return false; } } } + DEBUG_PRINT("DeleteRequest using coll_id: " << (int)coll_id); return GetOrDelete(policy::CB_BINARY_DELETE, key, coll_id); } @@ -1021,9 +998,12 @@ bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, return false; } if (header.status != (uint16_t)STATUS_SUCCESS) { - LOG_IF(ERROR, header.extras_length != 0) - << "GET response must not have flags"; - LOG_IF(ERROR, header.key_length != 0) << "GET response must not have key"; + if (DBUG && header.extras_length != 0) { + DEBUG_PRINT("GET response must not have flags"); + } + if (DBUG && header.key_length != 0) { + DEBUG_PRINT("GET response must not have key"); + } const int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; _status_code = header.status; @@ -1173,9 +1153,12 @@ bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, butil::string_printf(&_err, "Not enough data"); return false; } - LOG_IF(ERROR, header.extras_length != 0) - << "STORE response must not have flags"; - LOG_IF(ERROR, header.key_length != 0) << "STORE response must not have key"; + if (DBUG && header.extras_length != 0) { + DEBUG_PRINT("STORE response must not have flags"); + } + if (DBUG && header.key_length != 0) { + DEBUG_PRINT("STORE response must not have key"); + } int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; if (header.status != (uint16_t)STATUS_SUCCESS) { @@ -1193,8 +1176,9 @@ bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, } return false; } - LOG_IF(ERROR, value_size != 0) - << "STORE response must not have value, actually=" << value_size; + if (DBUG && value_size != 0) { + DEBUG_PRINT("STORE response must not have value, actually=" << value_size); + } _buf.pop_front(sizeof(header) + header.total_body_length); if (cas_value) { *cas_value = header.cas_value; @@ -1299,23 +1283,29 @@ bool CouchbaseOperations::CouchbaseRequest::UpsertRequest( uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { + DEBUG_PRINT("UpsertRequest called with key: " << key << ", value: " << value << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. if(local_collection_manifest_cache->empty()){ + DEBUG_PRINT("Local collection manifest cache is empty in UpsertRequest"); // if local cache is empty, goto global cache or fetch from server if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + DEBUG_PRINT("Failed to get collection id from global cache or server in UpsertRequest"); return false; } } // check if the collection id is available in the local cache else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + DEBUG_PRINT("Collection id not found in local cache in UpsertRequest"); // if not check in the global cache or fetch from server if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + DEBUG_PRINT("Failed to get collection id from global cache or server in UpsertRequest"); return false; } } } + DEBUG_PRINT("UpsertRequest using coll_id: " << (int)coll_id); return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value, coll_id); } @@ -1372,15 +1362,15 @@ bool CouchbaseOperations::CouchbaseRequest::RefreshCollectionManifest(brpc::Chan // first fetch the manifest // then compare the UID with the cached one if (channel == nullptr) { - LOG(ERROR) << "No channel found, make sure to call Authenticate() first"; + DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); return false; } if (server.empty()) { - LOG(ERROR) << "Server is empty, make sure to call Authenticate() first"; + DEBUG_PRINT("Server is empty, make sure to call Authenticate() first"); return false; } if (bucket.empty()) { - LOG(ERROR) << "No bucket selected, make sure to call SelectBucket() first"; + DEBUG_PRINT("No bucket selected, make sure to call SelectBucket() first"); return false; } CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; @@ -1390,20 +1380,18 @@ bool CouchbaseOperations::CouchbaseRequest::RefreshCollectionManifest(brpc::Chan channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, &temp_get_manifest_response, NULL); if (temp_cntl.Failed()) { - LOG(ERROR) << "Failed to get collection manifest: bRPC controller error" << - temp_cntl.ErrorText(); return false; + DEBUG_PRINT("Failed to get collection manifest: bRPC controller error " << temp_cntl.ErrorText()); return false; } string manifest_json; if (!temp_get_manifest_response.PopManifest(&manifest_json)) { - LOG(ERROR) << "Failed to parse response for refreshing collection Manifest: " << - temp_get_manifest_response.LastError(); return false; + DEBUG_PRINT("Failed to parse response for refreshing collection Manifest: " << temp_get_manifest_response.LastError()); return false; } // Compare the UID with the cached one // If they are different, refresh the cache brpc::CouchbaseMetadataTracking::CollectionManifest manifest; if(!common_metadata_tracking.json_to_collection_manifest(manifest_json, &manifest)){ - LOG(ERROR) << "Failed to parse collection manifest JSON"; + DEBUG_PRINT("Failed to parse collection manifest JSON"); return false; } brpc::CouchbaseMetadataTracking::CollectionManifest cached_manifest; @@ -1412,29 +1400,53 @@ bool CouchbaseOperations::CouchbaseRequest::RefreshCollectionManifest(brpc::Chan // No cached manifest found, set the new one if(!common_metadata_tracking.set_bucket_to_collection_manifest(server, bucket, manifest)){ - LOG(ERROR) << "Failed to cache collection manifest for bucket " << - bucket << " on server " << server; return false; + DEBUG_PRINT("Failed to cache collection manifest for bucket " << bucket << " on server " << server); return false; } - LOG(INFO) << "Cached collection manifest for bucket " << - bucket << " on server " << server; return true; + DEBUG_PRINT("Cached collection manifest for bucket " << bucket << " on server " << server); + // also update the local cache + (*local_collection_manifest_cache)[bucket] = manifest; + return true; } - if(manifest.uid != cached_manifest.uid) { - LOG(INFO) << "Collection manifest has changed for bucket " << - bucket << " on server " << server; + else if(manifest.uid != cached_manifest.uid) { + DEBUG_PRINT("Collection manifest has changed for bucket " << bucket << " on server " << server); if(!common_metadata_tracking.set_bucket_to_collection_manifest(server, bucket, manifest)){ - LOG(ERROR) << "Failed to update cached collection manifest for bucket " - << bucket << " on server " << server; return false; + DEBUG_PRINT("Failed to update cached collection manifest for bucket " << bucket << " on server " << server); return false; + } + DEBUG_PRINT("Updated cached collection manifest for bucket " << bucket << " on server " << server); + // also update the local cache if needed. + if(local_collection_manifest_cache->find(bucket) != + local_collection_manifest_cache->end()){ + // if the bucket already exists in the local cache, check the UID + if( (*local_collection_manifest_cache)[bucket].uid != manifest.uid){ + // if the UID is different, update the local cache + (*local_collection_manifest_cache)[bucket] = manifest; + DEBUG_PRINT("Updated local collection manifest cache for bucket " << bucket << " on server " << server); + } + } + else{ + // if the bucket does not exist in the local cache, add it + (*local_collection_manifest_cache)[bucket] = manifest; + DEBUG_PRINT("Added to local collection manifest cache for bucket " << bucket << " on server " << server); } - LOG(INFO) << "Updated cached collection manifest for bucket " << - bucket << " on server " << server; - // also update the local cache - (*local_collection_manifest_cache)[bucket] = manifest; return true; } else{ - LOG(INFO) << "Collection manifest is already up-to-date for bucket " << - bucket << " on server " << server; + DEBUG_PRINT("Collection manifest is already up-to-date for bucket " << bucket << " on server " << server); + if(local_collection_manifest_cache->find(bucket) != + local_collection_manifest_cache->end()){ + // if the bucket already exists in the local cache, check the UID + if( (*local_collection_manifest_cache)[bucket].uid != manifest.uid){ + // if the UID is different, update the local cache + (*local_collection_manifest_cache)[bucket] = manifest; + DEBUG_PRINT("Updated local collection manifest cache for bucket " << bucket << " on server " << server); + } + } + else{ + // if the bucket does not exist in the local cache, add it + (*local_collection_manifest_cache)[bucket] = manifest; + DEBUG_PRINT("Added to local collection manifest cache for bucket " << bucket << " on server " << server); + } return false; } } @@ -1444,23 +1456,29 @@ bool CouchbaseOperations::CouchbaseRequest::AddRequest( uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { + DEBUG_PRINT("AddRequest called with key: " << key << ", value: " << value << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. if(local_collection_manifest_cache->empty()){ + DEBUG_PRINT("Local collection manifest cache is empty in AddRequest"); // if local cache is empty, goto global cache or fetch from server if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + DEBUG_PRINT("Failed to get collection id from global cache or server in AddRequest"); return false; } } // check if the collection id is available in the local cache else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + DEBUG_PRINT("Collection id not found in local cache in AddRequest"); // if not check in the global cache or fetch from server if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + DEBUG_PRINT("Failed to get collection id from global cache or server in AddRequest"); return false; } } } + DEBUG_PRINT("AddRequest using coll_id: " << (int)coll_id); return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, coll_id); } @@ -1491,7 +1509,7 @@ bool CouchbaseOperations::CouchbaseRequest::AppendRequest( string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { if (value.empty()) { - LOG(ERROR) << "value to append must be non-empty"; + DEBUG_PRINT("value to append must be non-empty"); return false; } uint8_t coll_id = 0; // default collection ID @@ -1521,7 +1539,7 @@ bool CouchbaseOperations::CouchbaseRequest::PrependRequest( string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { if (value.empty()) { - LOG(ERROR) << "value to prepend must be non-empty"; + DEBUG_PRINT("value to prepend must be non-empty"); return false; } uint8_t coll_id = 0; // default collection ID @@ -1565,7 +1583,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopPrepend(uint64_t* cas_value) { bool CouchbaseOperations::CouchbaseResponse::PopSelectBucket( uint64_t* cas_value, std::string bucket_name) { if (PopStore(policy::CB_SELECT_BUCKET, cas_value) == false) { - LOG(ERROR) << "Failed to select bucket: " << _err; + DEBUG_PRINT("Failed to select bucket: " << _err); return false; } // Note: Bucket tracking is now handled at CouchbaseOperations level, not @@ -1629,7 +1647,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopCollectionId( sizeof(manifest_id_net)); // You may convert this if needed: uint64_t manifest_id = butil::NetToHost64(manifest_id_net); - LOG(INFO) << "Manifest ID: " << manifest_id; + DEBUG_PRINT("Manifest ID: " << manifest_id); _buf.pop_front(sizeof(manifest_id_net)); // Next 1 bytes → collection ID (u8) @@ -1668,10 +1686,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopManifest( if (header.status != 0) { // handle error case if (header.extras_length != 0) { - LOG(ERROR) << "Get Collections Manifest response must not have extras"; + DEBUG_PRINT("Get Collections Manifest response must not have extras"); } if (header.key_length != 0) { - LOG(ERROR) << "Get Collections Manifest response must not have key"; + DEBUG_PRINT("Get Collections Manifest response must not have key"); } _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); // Possibly read error message from value if present @@ -1811,10 +1829,12 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, header.total_body_length); return false; } - LOG_IF(ERROR, header.extras_length != 0) - << "INCR/DECR response must not have flags"; - LOG_IF(ERROR, header.key_length != 0) - << "INCR/DECR response must not have key"; + if (DBUG && header.extras_length != 0) { + DEBUG_PRINT("INCR/DECR response must not have flags"); + } + if (DBUG && header.key_length != 0) { + DEBUG_PRINT("INCR/DECR response must not have key"); + } const int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); @@ -1948,9 +1968,12 @@ bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { header.total_body_length); return false; } - LOG_IF(ERROR, header.extras_length != 0) - << "VERSION response must not have flags"; - LOG_IF(ERROR, header.key_length != 0) << "VERSION response must not have key"; + if (DBUG && header.extras_length != 0) { + DEBUG_PRINT("VERSION response must not have flags"); + } + if (DBUG && header.key_length != 0) { + DEBUG_PRINT("VERSION response must not have key"); + } const int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; _buf.pop_front(sizeof(header) + header.extras_length + header.key_length); @@ -1983,51 +2006,59 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, CouchbaseOperations::CouchbaseRequest *request, CouchbaseOperations::CouchbaseResponse *response) { if(channel == nullptr){ - LOG(ERROR)<< "No channel found, make sure to call Authenticate() first"; + DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); result->error_message = "No channel found, make sure to call Authenticate() first"; return false; } if(server.empty()){ - LOG(ERROR)<< "Server is empty, make sure to call Authenticate() first"; + DEBUG_PRINT("Server is empty, make sure to call Authenticate() first"); result->error_message = "Server is empty, make sure to call Authenticate() first"; return false; } if(bucket.empty()){ - LOG(ERROR)<< "No bucket selected, make sure to call SelectBucket() first"; + DEBUG_PRINT("No bucket selected, make sure to call SelectBucket() first"); result->error_message = "No bucket selected, make sure to call SelectBucket() first"; return false; } brpc::Controller cntl; + bool request_created = false; switch(op_type){ case CouchbaseOperations::GET: - request->GetRequest(key, collection_name, channel, server, bucket); + request_created = request->GetRequest(key, collection_name, channel, server, bucket); break; case CouchbaseOperations::UPSERT: - request->UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request_created = request->UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); break; case CouchbaseOperations::ADD: - request->AddRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request_created = request->AddRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); break; case CouchbaseOperations::APPEND: - request->AppendRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request_created = request->AppendRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); break; case CouchbaseOperations::PREPEND: - request->PrependRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request_created = request->PrependRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); break; case CouchbaseOperations::DELETE: - request->DeleteRequest(key, collection_name, channel, server, bucket); + request_created = request->DeleteRequest(key, collection_name, channel, server, bucket); break; default: - LOG(ERROR) << "Unsupported operation type"; + DEBUG_PRINT("Unsupported operation type"); result->success = false; result->value = ""; result->error_message = "Unsupported operation type"; return false; } + if(!request_created){ + DEBUG_PRINT("Failed to create request for operation type: " << op_type); + result->success = false; + result->value = ""; + result->error_message = "Failed to create request for operation type: " + std::to_string(op_type); + result->status_code = 0x88; // using 0x88 as the only possible failure code that indicates either the collectionID is not found or the there is some issue with the collection manifest either locally or on server + return false; + } channel->CallMethod(NULL, &cntl, request, response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to perform operation on key: " << key - << " to Couchbase: " << cntl.ErrorText(); + DEBUG_PRINT("Failed to perform operation on key: " << key << " to Couchbase: " << cntl.ErrorText()); result->success = false; result->value = ""; result->error_message = cntl.ErrorText(); @@ -2080,16 +2111,16 @@ CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, // and the client have a stale copy of collection manifest. // In this case, we need to refresh the collection manifest and retry the operation. if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - LOG(ERROR) << "Failed to refresh collection manifest"; + DEBUG_PRINT("Either the collection name " << collection_name << " is invalid and doesn't exist or the collection manifest is invalid."); result.success = false; result.value = ""; - result.error_message = "Failed to refresh collection manifest"; + result.error_message = "Either the collection name " + collection_name + " is invalid and doesn't exist or the collection manifest is invalid."; return result; } - // check if the colelction id is available in the local cache now. + // check if the collection id is available in the local cache now. uint8_t coll_id = 0; // default collection ID if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); result.success = false; result.value = ""; result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; @@ -2108,7 +2139,7 @@ bool CouchbaseOperations::CouchbaseRequest::getLocalCachedCollectionId(const str const string& collection, uint8_t* collection_id) { if (bucket.empty() || scope.empty() || collection.empty()) { - LOG(ERROR) << "Bucket, scope, and collection names must be non-empty"; + DEBUG_PRINT("Bucket, scope, and collection names must be non-empty"); return false; } auto it = local_collection_manifest_cache->find(bucket); @@ -2121,17 +2152,17 @@ bool CouchbaseOperations::CouchbaseRequest::getLocalCachedCollectionId(const str return true; } else{ - LOG(ERROR) << "Collection name not found in local cache: " << collection; + DEBUG_PRINT("Collection name not found in local cache: " << collection); return false; } } else{ - LOG(ERROR) << "Scope name not found in local cache: " << scope; + DEBUG_PRINT("Scope name not found in local cache: " << scope); return false; } } else{ - LOG(ERROR) << "Bucket name not found in local cache: " << bucket; + DEBUG_PRINT("Bucket name not found in local cache: " << bucket); return false; } } @@ -2148,7 +2179,7 @@ CouchbaseOperations::Result CouchbaseOperations::Upsert( // and the client have a stale copy of collection manifest. // In this case, we need to refresh the collection manifest and retry the operation. if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - LOG(ERROR) << "Failed to refresh collection manifest"; + DEBUG_PRINT("Failed to refresh collection manifest"); result.success = false; result.value = ""; result.error_message = "Failed to refresh collection manifest"; @@ -2157,7 +2188,7 @@ CouchbaseOperations::Result CouchbaseOperations::Upsert( // check if the colelction id is available in the local cache now. uint8_t coll_id = 0; // default collection ID if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); result.success = false; result.value = ""; result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; @@ -2184,7 +2215,7 @@ CouchbaseOperations::Result CouchbaseOperations::Delete( // and the client have a stale copy of collection manifest. // In this case, we need to refresh the collection manifest and retry the operation. if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - LOG(ERROR) << "Failed to refresh collection manifest"; + DEBUG_PRINT("Failed to refresh collection manifest"); result.success = false; result.value = ""; result.error_message = "Failed to refresh collection manifest"; @@ -2193,7 +2224,7 @@ CouchbaseOperations::Result CouchbaseOperations::Delete( // check if the colelction id is available in the local cache now. uint8_t coll_id = 0; // default collection ID if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); result.success = false; result.value = ""; result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; @@ -2221,7 +2252,7 @@ CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, // and the client have a stale copy of collection manifest. // In this case, we need to refresh the collection manifest and retry the operation. if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - LOG(ERROR) << "Failed to refresh collection manifest"; + DEBUG_PRINT("Failed to refresh collection manifest"); result.success = false; result.value = ""; result.error_message = "Failed to refresh collection manifest"; @@ -2230,7 +2261,7 @@ CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, // check if the colelction id is available in the local cache now. uint8_t coll_id = 0; // default collection ID if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); result.success = false; result.value = ""; result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; @@ -2267,8 +2298,7 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate( CouchbaseOperations::Result result; brpc::Channel* new_channel = new brpc::Channel(); if (new_channel->Init(server_address.c_str(), &options) != 0) { - LOG(ERROR) << "Failed to initialize Couchbase channel to " - << server_address; + DEBUG_PRINT("Failed to initialize Couchbase channel to " << server_address); delete new_channel; result.success = false; result.value = ""; @@ -2281,15 +2311,14 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate( brpc::Controller cntl; if (request.AuthenticateRequest(username.c_str(), password.c_str()) == false) { - LOG(ERROR) << "Failed to create Authenticate request for user: " - << username; + DEBUG_PRINT("Failed to create Authenticate request for user: " << username); delete new_channel; result.success = false; return result; } new_channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to access Couchbase: " << cntl.ErrorText(); + DEBUG_PRINT("Failed to access Couchbase: " << cntl.ErrorText()); delete new_channel; result.success = false; result.value = ""; @@ -2306,21 +2335,19 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate( CouchbaseOperations::Result CouchbaseOperations::SelectBucket( const string& bucket_name) { - CouchbaseRequest request; + CouchbaseRequest request(&local_bucket_to_collection_manifest); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; if (request.SelectBucketRequest(bucket_name.c_str()) == false) { - LOG(ERROR) << "Failed to create Select Bucket request for bucket: " - << bucket_name; + DEBUG_PRINT("Failed to create Select Bucket request for bucket: " << bucket_name); result.success = false; result.value = ""; return result; } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to select bucket: " << bucket_name - << " from Couchbase: " << cntl.ErrorText(); + DEBUG_PRINT("Failed to select bucket: " << bucket_name << " from Couchbase: " << cntl.ErrorText()); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); @@ -2338,6 +2365,30 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket( result.success = true; result.value = ""; result.status_code = 0; + + // fetch the collection manifest for this bucket and store it in local cache + if(request.local_collection_manifest_cache->find(bucket_name) == request.local_collection_manifest_cache->end()){ + // only fetch if not already present in the local cache + CouchbaseMetadataTracking::CollectionManifest manifest; + if(!common_metadata_tracking.get_bucket_to_collection_manifest(server_address, bucket_name, &manifest)){ + DEBUG_PRINT("Collection manifest for bucket: " << bucket_name << " not found in global cache, the local cache"); + + // manifest for this bucket/server is not cached yet, will fetch it from server now. + // refresh will also update the local cache with the fetched manifest + request.RefreshCollectionManifest(channel, server_address, bucket_name); + // We simply try once to prefetch the manifest, before any collection operation. + // If it fails, it will be lazily updated when a collection operation is performed. + } + else{ + // update the local cache with the cache manifest from global cache(common_metadata_tracking) + DEBUG_PRINT("Updated local cache collection manifest for bucket: " << bucket_name); + (*request.local_collection_manifest_cache)[bucket_name] = manifest; + } + } + else{ + DEBUG_PRINT("Collection manifest for bucket: " << bucket_name << " already present in local cache"); + } + DEBUG_PRINT("Bucket selected successfully " << bucket_name); return result; } @@ -2387,7 +2438,7 @@ CouchbaseOperations::Result CouchbaseOperations::Append( // and the client have a stale copy of collection manifest. // In this case, we need to refresh the collection manifest and retry the operation. if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - LOG(ERROR) << "Failed to refresh collection manifest"; + DEBUG_PRINT("Failed to refresh collection manifest"); result.success = false; result.value = ""; result.error_message = "Failed to refresh collection manifest"; @@ -2396,7 +2447,7 @@ CouchbaseOperations::Result CouchbaseOperations::Append( // check if the colelction id is available in the local cache now. uint8_t coll_id = 0; // default collection ID if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); result.success = false; result.value = ""; result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; @@ -2423,7 +2474,7 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend( // and the client have a stale copy of collection manifest. // In this case, we need to refresh the collection manifest and retry the operation. if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - LOG(ERROR) << "Failed to refresh collection manifest"; + DEBUG_PRINT("Failed to refresh collection manifest"); result.success = false; result.value = ""; result.error_message = "Failed to refresh collection manifest"; @@ -2432,7 +2483,7 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend( // check if the colelction id is available in the local cache now. uint8_t coll_id = 0; // default collection ID if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - LOG(ERROR) << "Collection name not found in local cache after manifest refresh: " << collection_name; + DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); result.success = false; result.value = ""; result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; @@ -2588,14 +2639,14 @@ CouchbaseOperations::Result CouchbaseOperations::Version() { brpc::Controller cntl; CouchbaseOperations::Result result; if (request.VersionRequest() == false) { - LOG(ERROR) << "Failed to create Version request"; + DEBUG_PRINT("Failed to create Version request"); result.success = false; result.value = ""; return result; } channel->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Failed to get version from Couchbase: " << cntl.ErrorText(); + DEBUG_PRINT("Failed to get version from Couchbase: " << cntl.ErrorText()); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); @@ -2618,7 +2669,7 @@ CouchbaseOperations::Result CouchbaseOperations::Version() { bool CouchbaseOperations::BeginPipeline() { if (pipeline_active) { - LOG(WARNING) << "Pipeline already active. Call ClearPipeline() first."; + DEBUG_PRINT("Pipeline already active. Call ClearPipeline() first."); return false; } @@ -2626,7 +2677,7 @@ bool CouchbaseOperations::BeginPipeline() { while (!pipeline_operations_queue.empty()) { pipeline_operations_queue.pop(); } - pipeline_request.Clear(); + pipeline_request_couchbase_req.Clear(); pipeline_active = true; return true; @@ -2636,14 +2687,15 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, const string& key, const string& value, string collection_name) { + if (!pipeline_active) { - LOG(ERROR) << "Pipeline not active. Call BeginPipeline() first."; + DEBUG_PRINT("Pipeline not active. Call BeginPipeline() first."); return false; } switch (op_type) { case GET: - if (pipeline_request.GetRequest(key, collection_name, channel, + if (pipeline_request_couchbase_req.GetRequest(key, collection_name, channel, server_address, selected_bucket) == false) { return false; @@ -2651,7 +2703,7 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, pipeline_operations_queue.push(GET); break; case UPSERT: - if (pipeline_request.UpsertRequest(key, value, 0, 0, 0, collection_name, + if (pipeline_request_couchbase_req.UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false) { return false; @@ -2659,7 +2711,7 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, pipeline_operations_queue.push(UPSERT); break; case ADD: - if (pipeline_request.AddRequest(key, value, 0, 0, 0, collection_name, + if (pipeline_request_couchbase_req.AddRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false) { return false; @@ -2667,7 +2719,7 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, pipeline_operations_queue.push(ADD); break; case APPEND: - if (pipeline_request.AppendRequest(key, value, 0, 0, 0, collection_name, + if (pipeline_request_couchbase_req.AppendRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false) { return false; @@ -2675,7 +2727,7 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, pipeline_operations_queue.push(APPEND); break; case PREPEND: - if (pipeline_request.PrependRequest(key, value, 0, 0, 0, collection_name, + if (pipeline_request_couchbase_req.PrependRequest(key, value, 0, 0, 0, collection_name, channel, server_address, selected_bucket) == false) { return false; @@ -2683,7 +2735,7 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, pipeline_operations_queue.push(PREPEND); break; case DELETE: - if (pipeline_request.DeleteRequest(key, collection_name, channel, + if (pipeline_request_couchbase_req.DeleteRequest(key, collection_name, channel, server_address, selected_bucket) == false) { return false; @@ -2691,7 +2743,7 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, pipeline_operations_queue.push(DELETE); break; default: - LOG(ERROR) << "Invalid operation type for pipelining"; + DEBUG_PRINT("Invalid operation type for pipelining"); return false; } return true; @@ -2700,15 +2752,15 @@ vector CouchbaseOperations::ExecutePipeline() { vector results; if (!pipeline_active || pipeline_operations_queue.empty()) { - LOG(ERROR) << "No pipeline active or no operations queued"; + DEBUG_PRINT("No pipeline active or no operations queued"); return results; } brpc::Controller cntl; - channel->CallMethod(NULL, &cntl, &pipeline_request, &pipeline_response, NULL); + channel->CallMethod(NULL, &cntl, &pipeline_request_couchbase_req, &pipeline_response_couchbase_resp, NULL); if (cntl.Failed()) { - LOG(ERROR) << "Pipeline execution failed: " << cntl.ErrorText(); + DEBUG_PRINT("Pipeline execution failed: " << cntl.ErrorText()); // Create failure results for all operations size_t op_count = pipeline_operations_queue.size(); results.reserve(op_count); @@ -2726,7 +2778,7 @@ vector CouchbaseOperations::ExecutePipeline() { } // Process each operation in the order they were added - CouchbaseOperations::CouchbaseResponse* response = &pipeline_response; + CouchbaseOperations::CouchbaseResponse* response = &pipeline_response_couchbase_resp; while (!pipeline_operations_queue.empty()) { CouchbaseOperations::Result result; operation_type op_type = pipeline_operations_queue.front(); @@ -2816,7 +2868,7 @@ vector CouchbaseOperations::ExecutePipeline() { break; } default: - LOG(ERROR) << "Invalid operation type in pipeline response processing"; + DEBUG_PRINT("Invalid operation type in pipeline response processing"); result.success = false; result.value = ""; result.error_message = "Invalid operation type"; @@ -2826,7 +2878,7 @@ vector CouchbaseOperations::ExecutePipeline() { } pipeline_active = false; - pipeline_request.Clear(); + pipeline_request_couchbase_req.Clear(); return results; } @@ -2835,7 +2887,7 @@ bool CouchbaseOperations::ClearPipeline() { while (!pipeline_operations_queue.empty()) { pipeline_operations_queue.pop(); } - pipeline_request.Clear(); + pipeline_request_couchbase_req.Clear(); pipeline_active = false; return true; } diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index a0fbc17c1b..aa33df92ab 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -199,7 +199,7 @@ class CouchbaseOperations { bool IsPipelineActive() const { return pipeline_active; } size_t GetPipelineSize() const { return pipeline_operations_queue.size(); } - CouchbaseOperations() : pipeline_active(false) {} + CouchbaseOperations() : pipeline_request_couchbase_req(&local_bucket_to_collection_manifest), pipeline_active(false) {} ~CouchbaseOperations() {} bool get_local_cached_collection_id(const string& bucket, const string& scope, const string& collection, uint8_t* coll_id); @@ -233,9 +233,12 @@ class CouchbaseOperations { const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, uint8_t coll_id = 0); uint32_t hash_crc32(const char* key, size_t key_length); - unordered_map *local_collection_manifest_cache; + public: - CouchbaseRequest(unordered_map *local_cache_reference) : NonreflectableMessage() { + + unordered_map *local_collection_manifest_cache; + + CouchbaseRequest(unordered_map *local_cache_reference) : NonreflectableMessage() { metadata_tracking = &common_metadata_tracking; local_collection_manifest_cache = local_cache_reference; SharedCtor(); @@ -519,8 +522,8 @@ class CouchbaseOperations { // Pipeline management - per instance queue pipeline_operations_queue; - CouchbaseRequest pipeline_request; - CouchbaseResponse pipeline_response; + CouchbaseRequest pipeline_request_couchbase_req; + CouchbaseResponse pipeline_response_couchbase_resp; bool pipeline_active; }; From 510f5d88f068d7755c18ea33a32817b0c6e019e6 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Sun, 5 Oct 2025 12:35:14 +0530 Subject: [PATCH 27/49] remove recurring statements --- src/brpc/couchbase.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 15418ed035..c5fbdf2b63 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -49,9 +49,6 @@ constexpr size_t RANDOM_ID_HEX_SIZE = 67; // 33 bytes * 2 + null terminator CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseRequest::metadata_tracking = &common_metadata_tracking; -CouchbaseMetadataTracking* - CouchbaseOperations::CouchbaseResponse::metadata_tracking = - &common_metadata_tracking; bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest( string server, string bucket, From 844d536884828a4dc70b5fdf7c836e941fa435c4 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Sun, 5 Oct 2025 22:02:24 +0530 Subject: [PATCH 28/49] Fixed bugs/repetitive calls to refreshing manifest on server --- src/brpc/couchbase.cpp | 343 +++++++++++++++++++++-------------------- 1 file changed, 174 insertions(+), 169 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index c5fbdf2b63..c08cf9855b 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -2046,11 +2046,11 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, return false; } if(!request_created){ - DEBUG_PRINT("Failed to create request for operation type: " << op_type); + DEBUG_PRINT("CollectionID does not exist." << op_type); result->success = false; result->value = ""; - result->error_message = "Failed to create request for operation type: " + std::to_string(op_type); - result->status_code = 0x88; // using 0x88 as the only possible failure code that indicates either the collectionID is not found or the there is some issue with the collection manifest either locally or on server + result->error_message = "CollectionID does not exist." + std::to_string(op_type); + result->status_code = 0x88; // using 0x88 as the only possible failure code that indicates the collectionID is not found return false; } channel->CallMethod(NULL, &cntl, request, response, NULL); @@ -2070,6 +2070,54 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, result->value = ""; result->error_message = response->LastError(); result->status_code = response->_status_code; + if(result->status_code == 0x88) { + DEBUG_PRINT("CollectionID does not exist on server, need to refresh collection manifest from server"); + // could have called sendRequest recursively, + // but if somehow the collectionID keeps on chaning, it would lead to infinite recursion and stack overflow in the end. + // so we retry once here instead and return failure if it still fails. + + // (0x88) unknown collection, this means that the collection_manifest has been updated on the server side. + // The collectionID present in the local cache/global cache is no longer valid. + // This can happen if a collection is deleted and recreated with the same name. + if (!request->RefreshCollectionManifest(channel, server, bucket)) { + DEBUG_PRINT("Failed to refresh collection manifest"); + result->error_message = "Failed to refresh collection manifest"; + } else { + DEBUG_PRINT("Successfully refreshed collection manifest"); + //retry the request; + request->Clear(); + response->Clear(); + cntl.Reset(); + if(!request->GetRequest(key, collection_name, channel, server, bucket)){ + DEBUG_PRINT("CollectionID does not exist."); + result->success = false; + result->value = ""; + result->error_message = "CollectionID does not exist."; + result->status_code = 0x88; // using 0x88 as the only possible failure code that indicates the collectionID is not found + return false; + } + channel->CallMethod(NULL, &cntl, request, response, NULL); + if (cntl.Failed()) { + DEBUG_PRINT("Failed to perform operation on key: " << key << " to Couchbase: " << cntl.ErrorText()); + result->success = false; + result->value = ""; + result->error_message = cntl.ErrorText(); + return false; // return on failure + } + if (response->PopGet(&value, &flags, &cas) == false) { + result->success = false; + result->value = ""; + result->error_message = response->LastError(); + result->status_code = response->_status_code; + return false; // return on failure + } + // Successfully got the value after retry + result->success = true; + result->value = value; + result->status_code = 0; + return true; + } + } return false; } // Successfully got the value @@ -2079,19 +2127,130 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, return true; } else{ - // For other operations, just check if it was successful - if (response->LastError().empty() && response->_status_code == 0) { - result->success = true; - result->value = ""; - result->status_code = 0; - return true; - } else { + uint64_t cas_value = 0; + //pop response on the basis of operation type + bool pop_success = false; + switch(op_type){ + case CouchbaseOperations::UPSERT: + pop_success = response->PopUpsert(&cas_value); + break; + case CouchbaseOperations::ADD: + pop_success = response->PopAdd(&cas_value); + break; + case CouchbaseOperations::APPEND: + pop_success = response->PopAppend(&cas_value); + break; + case CouchbaseOperations::PREPEND: + pop_success = response->PopPrepend(&cas_value); + break; + case CouchbaseOperations::DELETE: + pop_success = response->PopDelete(); + break; + default: + DEBUG_PRINT("Unsupported operation type in response pop"); + result->success = false; + result->value = ""; + result->error_message = "Unsupported operation type in response pop"; + return false; + } + if(!pop_success){ result->success = false; result->value = ""; result->error_message = response->LastError(); result->status_code = response->_status_code; + if(result->status_code == 0x88) { + // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. + // and the client have a stale copy of collection manifest. + // In this case, we need to refresh the collection manifest and retry the operation. + if (!request->RefreshCollectionManifest(channel, server, bucket)) { + DEBUG_PRINT("Failed to refresh collection manifest"); + result->error_message = "Failed to refresh collection manifest"; + return false; + } + // could have called sendRequest recursively, + // but if somehow the collectionID keeps on chaning, it would lead to infinite recursion and stack overflow in the end. + // so we retry once here instead and return failure if it still fails. + DEBUG_PRINT("Successfully refreshed collection manifest"); + //retry the request; + request->Clear(); + response->Clear(); + switch(op_type){ + case CouchbaseOperations::UPSERT: + request->UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::ADD: + request->AddRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::APPEND: + request->AppendRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::PREPEND: + request->PrependRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + break; + case CouchbaseOperations::DELETE: + request->DeleteRequest(key, collection_name, channel, server, bucket); + break; + default: + DEBUG_PRINT("Unsupported operation type in response pop"); + result->success = false; + result->value = ""; + result->error_message = "Unsupported operation type in response pop"; + return false; + } + channel->CallMethod(NULL, &cntl, request, response, NULL); + if (cntl.Failed()) { + DEBUG_PRINT("Failed to perform operation on key: " << key << " to Couchbase: " << cntl.ErrorText()); + result->success = false; + result->value = ""; + result->error_message = cntl.ErrorText(); + return false; // return on failure + } + pop_success = false; + switch(op_type){ + case CouchbaseOperations::UPSERT: + pop_success = response->PopUpsert(&cas_value); + break; + case CouchbaseOperations::ADD: + pop_success = response->PopAdd(&cas_value); + break; + case CouchbaseOperations::APPEND: + pop_success = response->PopAppend(&cas_value); + break; + case CouchbaseOperations::PREPEND: + pop_success = response->PopPrepend(&cas_value); + break; + case CouchbaseOperations::DELETE: + pop_success = response->PopDelete(); + break; + default: + DEBUG_PRINT("Unsupported operation type in response pop"); + result->success = false; + result->value = ""; + result->error_message = "Unsupported operation type in response pop"; + return false; + } + if(!pop_success){ + result->success = false; + result->value = ""; + result->error_message = response->LastError(); + result->status_code = response->_status_code; + return false; // return on failure + } + // Successfully performed the operation after retry + result->success = true; + result->value = ""; + result->status_code = 0; + return true; + } return false; } + // Successfully performed the operation + // Note: For operations other than GET, we don't have a value to return + // so we return empty string for value. + result->success = true; + result->value = ""; + result->status_code = 0; + return true; } } CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, @@ -2102,33 +2261,7 @@ CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(!sendRequest(CouchbaseOperations::GET, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ - if(result.status_code == 0x88) { - // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. - // and the client have a stale copy of collection manifest. - // In this case, we need to refresh the collection manifest and retry the operation. - if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - DEBUG_PRINT("Either the collection name " << collection_name << " is invalid and doesn't exist or the collection manifest is invalid."); - result.success = false; - result.value = ""; - result.error_message = "Either the collection name " + collection_name + " is invalid and doesn't exist or the collection manifest is invalid."; - return result; - } - // check if the collection id is available in the local cache now. - uint8_t coll_id = 0; // default collection ID - if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); - result.success = false; - result.value = ""; - result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; - return result; - } - // only reachable if the collection manifest now has the collectionID - sendRequest(CouchbaseOperations::GET, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response); - return result; - } - return result; - } + sendRequest(CouchbaseOperations::GET, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response); return result; } @@ -2170,33 +2303,7 @@ CouchbaseOperations::Result CouchbaseOperations::Upsert( CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(!sendRequest(CouchbaseOperations::UPSERT, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ - if(result.status_code == 0x88) { - // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. - // and the client have a stale copy of collection manifest. - // In this case, we need to refresh the collection manifest and retry the operation. - if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - DEBUG_PRINT("Failed to refresh collection manifest"); - result.success = false; - result.value = ""; - result.error_message = "Failed to refresh collection manifest"; - return result; - } - // check if the colelction id is available in the local cache now. - uint8_t coll_id = 0; // default collection ID - if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); - result.success = false; - result.value = ""; - result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; - return result; - } - // only reachable if the collection manifest now has the collectionID - sendRequest(CouchbaseOperations::UPSERT, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); - return result; - } - return result; - } + sendRequest(CouchbaseOperations::UPSERT, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); return result; } @@ -2207,30 +2314,6 @@ CouchbaseOperations::Result CouchbaseOperations::Delete( brpc::Controller cntl; CouchbaseOperations::Result result; if(!sendRequest(CouchbaseOperations::DELETE, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ - if(result.status_code == 0x88) { - // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. - // and the client have a stale copy of collection manifest. - // In this case, we need to refresh the collection manifest and retry the operation. - if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - DEBUG_PRINT("Failed to refresh collection manifest"); - result.success = false; - result.value = ""; - result.error_message = "Failed to refresh collection manifest"; - return result; - } - // check if the colelction id is available in the local cache now. - uint8_t coll_id = 0; // default collection ID - if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); - result.success = false; - result.value = ""; - result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; - return result; - } - // only reachable if the collection manifest now has the collectionID - sendRequest(CouchbaseOperations::DELETE, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response); - return result; - } return result; } return result; @@ -2243,33 +2326,7 @@ CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(!sendRequest(CouchbaseOperations::ADD, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ - if(result.status_code == 0x88) { - // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. - // and the client have a stale copy of collection manifest. - // In this case, we need to refresh the collection manifest and retry the operation. - if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - DEBUG_PRINT("Failed to refresh collection manifest"); - result.success = false; - result.value = ""; - result.error_message = "Failed to refresh collection manifest"; - return result; - } - // check if the colelction id is available in the local cache now. - uint8_t coll_id = 0; // default collection ID - if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); - result.success = false; - result.value = ""; - result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; - return result; - } - // only reachable if the collection manifest now has the collectionID - sendRequest(CouchbaseOperations::ADD, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); - return result; - } - return result; - } + sendRequest(CouchbaseOperations::ADD, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); return result; } @@ -2429,33 +2486,7 @@ CouchbaseOperations::Result CouchbaseOperations::Append( CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(!sendRequest(CouchbaseOperations::APPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ - if(result.status_code == 0x88) { - // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. - // and the client have a stale copy of collection manifest. - // In this case, we need to refresh the collection manifest and retry the operation. - if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - DEBUG_PRINT("Failed to refresh collection manifest"); - result.success = false; - result.value = ""; - result.error_message = "Failed to refresh collection manifest"; - return result; - } - // check if the colelction id is available in the local cache now. - uint8_t coll_id = 0; // default collection ID - if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); - result.success = false; - result.value = ""; - result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; - return result; - } - // only reachable if the collection manifest now has the collectionID - sendRequest(CouchbaseOperations::APPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); - return result; - } - return result; - } + sendRequest(CouchbaseOperations::APPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); return result; } @@ -2465,33 +2496,7 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend( CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(!sendRequest(CouchbaseOperations::PREPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ - if(result.status_code == 0x88) { - // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. - // and the client have a stale copy of collection manifest. - // In this case, we need to refresh the collection manifest and retry the operation. - if (!request.RefreshCollectionManifest(channel, server_address, selected_bucket)) { - DEBUG_PRINT("Failed to refresh collection manifest"); - result.success = false; - result.value = ""; - result.error_message = "Failed to refresh collection manifest"; - return result; - } - // check if the colelction id is available in the local cache now. - uint8_t coll_id = 0; // default collection ID - if(!request.getLocalCachedCollectionId(selected_bucket, "_default", collection_name, &coll_id)){ - DEBUG_PRINT("Collection name not found in local cache after manifest refresh: " << collection_name); - result.success = false; - result.value = ""; - result.error_message = "Collection name not found in local cache after manifest refresh: " + collection_name; - return result; - } - // only reachable if the collection manifest now has the collectionID - sendRequest(CouchbaseOperations::PREPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); - return result; - } - return result; - } + sendRequest(CouchbaseOperations::PREPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); return result; } From fe51d104de8b6a9ec5f65d7e21aa8755e7bd4577 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Sun, 5 Oct 2025 23:15:11 +0530 Subject: [PATCH 29/49] Formatted function/variable naming scheme and formatted code in c++ google format --- example/couchbase_c++/couchbase_client.cpp | 101 +- .../multithreaded_couchbase_client.cpp | 149 ++- src/brpc/couchbase.cpp | 1036 ++++++++++------- src/brpc/couchbase.h | 245 ++-- src/brpc/policy/couchbase_protocol.cpp | 13 +- 5 files changed, 857 insertions(+), 687 deletions(-) diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index f8ec367314..949aea5d47 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -27,8 +27,7 @@ #define RED "\033[31m" #define RESET "\033[0m" -DEFINE_string(server, "localhost:11210", - "IP Address of server"); +DEFINE_string(server, "localhost:11210", "IP Address of server"); int main() { // Create CouchbaseOperations instance for high-level operations @@ -38,8 +37,8 @@ int main() { << RESET << std::endl; // Ask username and password for authentication - std::string username="Administrator"; - std::string password="password"; + std::string username = "Administrator"; + std::string password = "password"; while (username.empty() || password.empty()) { std::cout << "Enter Couchbase username: "; std::cin >> username; @@ -56,9 +55,10 @@ int main() { } // Use high-level authentication method - // when connecting to capella use couchbase_ops.Authenticate(username, password, FLAGS_server, true, "path/to/cert.txt"); - brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( - username, password, FLAGS_server); + // when connecting to capella use couchbase_ops.authenticate(username, + // password, FLAGS_server, true, "path/to/cert.txt"); + brpc::CouchbaseOperations::Result auth_result = + couchbase_ops.authenticate(username, password, FLAGS_server); if (!auth_result.success) { LOG(ERROR) << "Authentication failed: " << auth_result.error_message; return -1; @@ -70,7 +70,7 @@ int main() { << RESET << std::endl; // Select bucket - std::string bucket_name="testing"; + std::string bucket_name = "testing"; while (bucket_name.empty()) { std::cout << "Enter Couchbase bucket name: "; std::cin >> bucket_name; @@ -83,7 +83,7 @@ int main() { // Use high-level bucket selection method brpc::CouchbaseOperations::Result bucket_result = - couchbase_ops.SelectBucket(bucket_name); + couchbase_ops.selectBucket(bucket_name); if (!bucket_result.success) { LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; return -1; @@ -97,7 +97,7 @@ int main() { R"({"name": "John Doe", "age": 30, "email": "john@example.com"})"; brpc::CouchbaseOperations::Result add_result = - couchbase_ops.Add(add_key, add_value); + couchbase_ops.add(add_key, add_value); if (add_result.success) { std::cout << GREEN << "ADD operation successful" << RESET << std::endl; } else { @@ -107,7 +107,7 @@ int main() { // Try to ADD the same key again (should fail with key exists) brpc::CouchbaseOperations::Result add_result2 = - couchbase_ops.Add(add_key, add_value); + couchbase_ops.add(add_key, add_value); if (add_result2.success) { std::cout << GREEN << "Second ADD operation unexpectedly successful" << RESET << std::endl; @@ -116,7 +116,7 @@ int main() { << add_result2.error_message << RESET << std::endl; } // Get operation using high-level method - brpc::CouchbaseOperations::Result get_result = couchbase_ops.Get(add_key); + brpc::CouchbaseOperations::Result get_result = couchbase_ops.get(add_key); if (get_result.success) { std::cout << GREEN << "GET operation successful" << RESET << std::endl; std::cout << "GET value: " << get_result.value << std::endl; @@ -128,7 +128,7 @@ int main() { // Add binprot item1 using high-level method std::string item1_key = "binprot_item1"; brpc::CouchbaseOperations::Result item1_result = - couchbase_ops.Add(item1_key, add_value); + couchbase_ops.add(item1_key, add_value); if (item1_result.success) { std::cout << GREEN << "ADD binprot item1 successful" << RESET << std::endl; } else { @@ -140,7 +140,7 @@ int main() { // Add binprot item2 using high-level method std::string item2_key = "binprot_item2"; brpc::CouchbaseOperations::Result item2_result = - couchbase_ops.Add(item2_key, add_value); + couchbase_ops.add(item2_key, add_value); if (item2_result.success) { std::cout << GREEN << "ADD binprot item2 successful" << RESET << std::endl; } else { @@ -152,7 +152,7 @@ int main() { // Add binprot item3 using high-level method std::string item3_key = "binprot_item3"; brpc::CouchbaseOperations::Result item3_result = - couchbase_ops.Add(item3_key, add_value); + couchbase_ops.add(item3_key, add_value); if (item3_result.success) { std::cout << GREEN << "ADD binprot item3 successful" << RESET << std::endl; } else { @@ -166,7 +166,7 @@ int main() { std::string upsert_value = R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})"; brpc::CouchbaseOperations::Result upsert_result = - couchbase_ops.Upsert(upsert_key, upsert_value); + couchbase_ops.upsert(upsert_key, upsert_value); if (upsert_result.success) { std::cout << GREEN @@ -183,7 +183,7 @@ int main() { std::string new_upsert_value = R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})"; brpc::CouchbaseOperations::Result new_upsert_result = - couchbase_ops.Upsert(new_upsert_key, new_upsert_value); + couchbase_ops.upsert(new_upsert_key, new_upsert_value); if (new_upsert_result.success) { std::cout << GREEN << "UPSERT operation successful when the document doesn't exist " @@ -198,7 +198,7 @@ int main() { // Check the upserted data using high-level method std::string check_key = "user::test_brpc_new_upsert"; - brpc::CouchbaseOperations::Result check_result = couchbase_ops.Get(check_key); + brpc::CouchbaseOperations::Result check_result = couchbase_ops.get(check_key); if (check_result.success) { std::cout << GREEN << "GET after UPSERT operation successful - Value: " << check_result.value << RESET << std::endl; @@ -210,7 +210,7 @@ int main() { // Delete a non-existent key using high-level method std::string delete_key = "Nonexistent_key"; brpc::CouchbaseOperations::Result delete_result = - couchbase_ops.Delete(delete_key); + couchbase_ops.delete_(delete_key); if (delete_result.success) { std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; } else { @@ -221,7 +221,7 @@ int main() { // Delete the existing key using high-level method std::string delete_existing_key = "user::test_brpc_binprot"; brpc::CouchbaseOperations::Result delete_existing_result = - couchbase_ops.Delete(delete_existing_key); + couchbase_ops.delete_(delete_existing_key); if (delete_existing_result.success) { std::cout << GREEN << "DELETE operation successful" << RESET << std::endl; } else { @@ -231,8 +231,8 @@ int main() { // Retrieve Collection ID for scope `_default` and collection // `testing_collection` - const std::string scope_name = "_default"; // default scope - std::string collection_name = "testing_collection"; // target collection + const std::string scope_name = "_default"; // default scope + std::string collection_name = "non_existent_collection"; // target collection // enter collection name as user input // std::cout << "Enter collection name (default 'testing_collection'): "; // std::string user_input; @@ -247,7 +247,7 @@ int main() { std::string coll_key = "user::collection_doc"; std::string coll_value = R"({"type":"collection","op":"add","v":1})"; brpc::CouchbaseOperations::Result coll_add_result = - couchbase_ops.Add(coll_key, coll_value, collection_name); + couchbase_ops.add(coll_key, coll_value, collection_name); if (coll_add_result.success) { std::cout << GREEN << "Collection ADD success" << RESET << std::endl; } else { @@ -255,11 +255,12 @@ int main() { << "Collection ADD failed: " << coll_add_result.error_message << RESET << std::endl; } - // std::cout << "Delete the collection."; + // Prompt user to delete the collection to test collection operations + // std::cout << "Delete & re-create the collection."; // std::cin >> collection_name; // 2. GET from collection using high-level method brpc::CouchbaseOperations::Result coll_get_result = - couchbase_ops.Get(coll_key, collection_name); + couchbase_ops.get(coll_key, collection_name); if (coll_get_result.success) { std::cout << GREEN << "Collection GET success value=" << coll_get_result.value @@ -274,7 +275,7 @@ int main() { std::string coll_upsert_value = R"({"type":"collection","op":"upsert","v":2})"; brpc::CouchbaseOperations::Result coll_upsert_result = - couchbase_ops.Upsert(coll_key, coll_upsert_value, collection_name); + couchbase_ops.upsert(coll_key, coll_upsert_value, collection_name); if (coll_upsert_result.success) { std::cout << GREEN << "Collection UPSERT success" << RESET << std::endl; } else { @@ -284,7 +285,7 @@ int main() { // 4. GET again to verify upsert using high-level method brpc::CouchbaseOperations::Result coll_get2_result = - couchbase_ops.Get(coll_key, collection_name); + couchbase_ops.get(coll_key, collection_name); if (coll_get2_result.success) { std::cout << GREEN << "Collection GET(after upsert) value=" << coll_get2_result.value @@ -293,7 +294,7 @@ int main() { // 5. DELETE from collection using high-level method brpc::CouchbaseOperations::Result coll_del_result = - couchbase_ops.Delete(coll_key, collection_name); + couchbase_ops.delete_(coll_key, collection_name); if (coll_del_result.success) { std::cout << GREEN << "Collection DELETE success" << RESET << std::endl; } else { @@ -309,7 +310,7 @@ int main() { << std::endl; // Begin a new pipeline - if (!couchbase_ops.BeginPipeline()) { + if (!couchbase_ops.beginPipeline()) { std::cout << RED << "Failed to begin pipeline" << RESET << std::endl; return -1; } @@ -326,31 +327,31 @@ int main() { // Pipeline operations - all prepared but not yet executed bool pipeline_success = true; - pipeline_success &= couchbase_ops.PipelineRequest( + pipeline_success &= couchbase_ops.pipelineRequest( brpc::CouchbaseOperations::ADD, pipeline_key1, pipeline_value1); - pipeline_success &= couchbase_ops.PipelineRequest( + pipeline_success &= couchbase_ops.pipelineRequest( brpc::CouchbaseOperations::UPSERT, pipeline_key2, pipeline_value2); - pipeline_success &= couchbase_ops.PipelineRequest( + pipeline_success &= couchbase_ops.pipelineRequest( brpc::CouchbaseOperations::ADD, pipeline_key3, pipeline_value3); - pipeline_success &= couchbase_ops.PipelineRequest( + pipeline_success &= couchbase_ops.pipelineRequest( brpc::CouchbaseOperations::GET, pipeline_key1); - pipeline_success &= couchbase_ops.PipelineRequest( + pipeline_success &= couchbase_ops.pipelineRequest( brpc::CouchbaseOperations::GET, pipeline_key2); if (!pipeline_success) { std::cout << RED << "Failed to add operations to pipeline" << RESET << std::endl; - couchbase_ops.ClearPipeline(); + couchbase_ops.clearPipeline(); return -1; } - std::cout << "Added " << couchbase_ops.GetPipelineSize() + std::cout << "Added " << couchbase_ops.getPipelineSize() << " operations to pipeline" << std::endl; // Execute all operations in a single network call std::cout << "Executing pipeline operations..." << std::endl; std::vector pipeline_results = - couchbase_ops.ExecutePipeline(); + couchbase_ops.executePipeline(); // Process results in order std::cout << GREEN << "Pipeline execution completed. Results:" << RESET @@ -375,7 +376,7 @@ int main() { std::cout << GREEN << "\n=== Pipeline with Collection Operations ===" << RESET << std::endl; - if (!couchbase_ops.BeginPipeline()) { + if (!couchbase_ops.beginPipeline()) { std::cout << RED << "Failed to begin collection pipeline" << RESET << std::endl; return -1; @@ -390,28 +391,28 @@ int main() { // Add collection-scoped operations to pipeline bool coll_pipeline_success = true; - coll_pipeline_success &= couchbase_ops.PipelineRequest( + coll_pipeline_success &= couchbase_ops.pipelineRequest( brpc::CouchbaseOperations::ADD, coll_pipeline_key1, coll_pipeline_value1, collection_name); - coll_pipeline_success &= couchbase_ops.PipelineRequest( + coll_pipeline_success &= couchbase_ops.pipelineRequest( brpc::CouchbaseOperations::UPSERT, coll_pipeline_key2, coll_pipeline_value2, collection_name); - coll_pipeline_success &= couchbase_ops.PipelineRequest( + coll_pipeline_success &= couchbase_ops.pipelineRequest( brpc::CouchbaseOperations::GET, coll_pipeline_key1, "", collection_name); coll_pipeline_success &= - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE, coll_pipeline_key1, "", collection_name); if (!coll_pipeline_success) { std::cout << RED << "Failed to add collection operations to pipeline" << RESET << std::endl; - couchbase_ops.ClearPipeline(); + couchbase_ops.clearPipeline(); return -1; } // Execute collection pipeline std::vector coll_pipeline_results = - couchbase_ops.ExecutePipeline(); + couchbase_ops.executePipeline(); std::cout << GREEN << "Collection pipeline execution completed. Results:" << RESET @@ -434,18 +435,18 @@ int main() { // Clean up remaining pipeline documents std::cout << GREEN << "\n=== Cleanup Pipeline Demo ===" << RESET << std::endl; - if (couchbase_ops.BeginPipeline()) { - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + if (couchbase_ops.beginPipeline()) { + couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key1); - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key2); - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE, pipeline_key3); - couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, + couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE, coll_pipeline_key2, "", collection_name); std::vector cleanup_results = - couchbase_ops.ExecutePipeline(); + couchbase_ops.executePipeline(); std::cout << "Cleanup completed (" << cleanup_results.size() << " operations)" << std::endl; } diff --git a/example/couchbase_c++/multithreaded_couchbase_client.cpp b/example/couchbase_c++/multithreaded_couchbase_client.cpp index 978c0b5137..df40aab48c 100644 --- a/example/couchbase_c++/multithreaded_couchbase_client.cpp +++ b/example/couchbase_c++/multithreaded_couchbase_client.cpp @@ -19,10 +19,11 @@ #include #include #include + +#include #include #include #include -#include // ANSI color codes #define GREEN "\033[32m" @@ -39,7 +40,8 @@ const int THREADS_PER_BUCKET = 5; struct { std::string username = "Administrator"; std::string password = "password"; - std::vector bucket_names = {"testing0", "testing1", "testing2", "testing3"}; + std::vector bucket_names = {"testing0", "testing1", "testing2", + "testing3"}; } g_config; // Simple thread statistics @@ -47,7 +49,7 @@ struct ThreadStats { std::atomic operations_attempted{0}; std::atomic operations_successful{0}; std::atomic operations_failed{0}; - + void reset() { operations_attempted = 0; operations_successful = 0; @@ -59,9 +61,9 @@ struct ThreadStats { struct GlobalStats { ThreadStats total; std::vector per_thread_stats; - + GlobalStats() : per_thread_stats(NUM_THREADS) {} - + void aggregate_stats() { total.reset(); for (const auto& stats : per_thread_stats) { @@ -82,38 +84,38 @@ struct ThreadArgs { // Simple CRUD operations on default collection void perform_crud_operations_default(brpc::CouchbaseOperations& couchbase_ops, - const std::string& base_key, - ThreadStats* stats) { + const std::string& base_key, + ThreadStats* stats) { std::string key = base_key + "_default"; - std::string value = butil::string_printf(R"({"thread_id": %d, "collection": "default"})", - (int)bthread_self()); - + std::string value = butil::string_printf( + R"({"thread_id": %d, "collection": "default"})", (int)bthread_self()); + stats->operations_attempted++; - + // UPSERT - brpc::CouchbaseOperations::Result result = couchbase_ops.Upsert(key, value); + brpc::CouchbaseOperations::Result result = couchbase_ops.upsert(key, value); if (result.success) { stats->operations_successful++; } else { stats->operations_failed++; return; } - + stats->operations_attempted++; - + // GET - result = couchbase_ops.Get(key); + result = couchbase_ops.get(key); if (result.success) { stats->operations_successful++; } else { stats->operations_failed++; return; } - + stats->operations_attempted++; - + // DELETE - result = couchbase_ops.Delete(key); + result = couchbase_ops.delete_(key); if (result.success) { stats->operations_successful++; } else { @@ -123,38 +125,39 @@ void perform_crud_operations_default(brpc::CouchbaseOperations& couchbase_ops, // Simple CRUD operations on col1 collection void perform_crud_operations_col1(brpc::CouchbaseOperations& couchbase_ops, - const std::string& base_key, - ThreadStats* stats) { + const std::string& base_key, + ThreadStats* stats) { std::string key = base_key + "_col1"; - std::string value = butil::string_printf(R"({"thread_id": %d, "collection": "col1"})", - (int)bthread_self()); - + std::string value = butil::string_printf( + R"({"thread_id": %d, "collection": "col1"})", (int)bthread_self()); + stats->operations_attempted++; - + // UPSERT - brpc::CouchbaseOperations::Result result = couchbase_ops.Upsert(key, value, "col1"); + brpc::CouchbaseOperations::Result result = + couchbase_ops.upsert(key, value, "col1"); if (result.success) { stats->operations_successful++; } else { stats->operations_failed++; return; } - + stats->operations_attempted++; - + // GET - result = couchbase_ops.Get(key, "col1"); + result = couchbase_ops.get(key, "col1"); if (result.success) { stats->operations_successful++; } else { stats->operations_failed++; return; } - + stats->operations_attempted++; - + // DELETE - result = couchbase_ops.Delete(key, "col1"); + result = couchbase_ops.delete_(key, "col1"); if (result.success) { stats->operations_successful++; } else { @@ -173,48 +176,53 @@ void* thread_worker(void* arg) { brpc::CouchbaseOperations couchbase_ops; // Authentication - brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( + brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticate( g_config.username, g_config.password, "127.0.0.1:11210", false, ""); - + if (!auth_result.success) { - std::cout << RED << "Thread " << args->thread_id << ": Auth failed - " + std::cout << RED << "Thread " << args->thread_id << ": Auth failed - " << auth_result.error_message << RESET << std::endl; return NULL; } // Select bucket - brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.SelectBucket(args->bucket_name); - + brpc::CouchbaseOperations::Result bucket_result = + couchbase_ops.selectBucket(args->bucket_name); + if (!bucket_result.success) { - std::cout << RED << "Thread " << args->thread_id << ": Bucket selection failed - " - << bucket_result.error_message << RESET << std::endl; + std::cout << RED << "Thread " << args->thread_id + << ": Bucket selection failed - " << bucket_result.error_message + << RESET << std::endl; return NULL; } std::cout << GREEN << "Thread " << args->thread_id << " connected to bucket " << args->bucket_name << RESET << std::endl; - // Perform operations - 10 times on default collection, 10 times on col1 collection + // Perform operations - 10 times on default collection, 10 times on col1 + // collection for (int i = 0; i < 10; ++i) { - std::string base_key = butil::string_printf("thread_%d_op_%d", args->thread_id, i); - + std::string base_key = + butil::string_printf("thread_%d_op_%d", args->thread_id, i); + // CRUD operations on default collection perform_crud_operations_default(couchbase_ops, base_key, args->stats); - + // CRUD operations on col1 collection perform_crud_operations_col1(couchbase_ops, base_key, args->stats); - + // Small delay between operations - bthread_usleep(10000); // 10ms + bthread_usleep(10000); // 10ms } int successful = args->stats->operations_successful.load(); int attempted = args->stats->operations_attempted.load(); int failed = args->stats->operations_failed.load(); - - std::cout << GREEN << "Thread " << args->thread_id << " completed: " - << successful << "/" << attempted << " operations successful, " - << failed << " failed" << RESET << std::endl; + + std::cout << GREEN << "Thread " << args->thread_id + << " completed: " << successful << "/" << attempted + << " operations successful, " << failed << " failed" << RESET + << std::endl; return NULL; } @@ -222,22 +230,25 @@ void* thread_worker(void* arg) { // Print simple statistics void print_stats() { g_stats.aggregate_stats(); - + std::cout << std::endl; std::cout << CYAN << "=== TEST RESULTS ===" << RESET << std::endl; - + int total_attempted = g_stats.total.operations_attempted.load(); int total_successful = g_stats.total.operations_successful.load(); int total_failed = g_stats.total.operations_failed.load(); - - double success_rate = total_attempted > 0 ? (double)total_successful / total_attempted * 100.0 : 0.0; - + + double success_rate = total_attempted > 0 + ? (double)total_successful / total_attempted * 100.0 + : 0.0; + std::cout << GREEN << "Overall Performance:" << RESET << std::endl; std::cout << " Total Operations: " << total_attempted << std::endl; - std::cout << " Successful: " << total_successful << " (" << success_rate << "%)" << std::endl; + std::cout << " Successful: " << total_successful << " (" << success_rate + << "%)" << std::endl; std::cout << " Failed: " << total_failed << std::endl; std::cout << std::endl; - + // Per-thread breakdown std::cout << YELLOW << "Per-Thread Performance:" << RESET << std::endl; for (int i = 0; i < NUM_THREADS; ++i) { @@ -245,17 +256,24 @@ void print_stats() { int attempted = stats.operations_attempted.load(); int successful = stats.operations_successful.load(); int failed = stats.operations_failed.load(); - - std::cout << " Thread " << i << ": " << attempted << " ops, " - << successful << " success, " << failed << " failed" << std::endl; + + std::cout << " Thread " << i << ": " << attempted << " ops, " << successful + << " success, " << failed << " failed" << std::endl; } std::cout << std::endl; } int main(int argc, char* argv[]) { - std::cout << GREEN << "Starting Simple Multithreaded Couchbase Client" << RESET << std::endl; - std::cout << YELLOW << "20 threads: 5 per bucket (testing0, testing1, testing2, testing3)" << RESET << std::endl; - std::cout << BLUE << "Each thread performs CRUD operations on default collection and col1 collection" << RESET << std::endl; + std::cout << GREEN << "Starting Simple Multithreaded Couchbase Client" + << RESET << std::endl; + std::cout + << YELLOW + << "20 threads: 5 per bucket (testing0, testing1, testing2, testing3)" + << RESET << std::endl; + std::cout << BLUE + << "Each thread performs CRUD operations on default collection and " + "col1 collection" + << RESET << std::endl; // Create threads and arguments std::vector threads(NUM_THREADS); @@ -272,13 +290,15 @@ int main(int argc, char* argv[]) { // Print thread assignments std::cout << "Thread Assignments:" << RESET << std::endl; for (int i = 0; i < NUM_THREADS; ++i) { - std::cout << "Thread " << i << " -> Bucket: " << args[i].bucket_name << std::endl; + std::cout << "Thread " << i << " -> Bucket: " << args[i].bucket_name + << std::endl; } std::cout << std::endl; // Start all threads for (int i = 0; i < NUM_THREADS; ++i) { - if (bthread_start_background(&threads[i], NULL, thread_worker, &args[i]) != 0) { + if (bthread_start_background(&threads[i], NULL, thread_worker, &args[i]) != + 0) { std::cout << RED << "Failed to create thread " << i << RESET << std::endl; return -1; } @@ -287,7 +307,8 @@ int main(int argc, char* argv[]) { std::cout << GREEN << "All 20 threads started!" << RESET << std::endl; // Wait for all threads to complete - std::cout << YELLOW << "Waiting for all threads to complete..." << RESET << std::endl; + std::cout << YELLOW << "Waiting for all threads to complete..." << RESET + << std::endl; for (int i = 0; i < NUM_THREADS; ++i) { bthread_join(threads[i], NULL); } diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index c08cf9855b..0ceb988c12 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -20,15 +20,21 @@ #include //for crc32 Vbucket_id // Debug flag for enabling debug statements -static bool DBUG = true; // Set to true to enable debug logs +static bool DBUG = true; // Set to true to enable debug logs // Debug print macro -#define DEBUG_PRINT(msg) do { if (DBUG) { std::cout << "[DEBUG] " << msg << std::endl; } } while(0) +#define DEBUG_PRINT(msg) \ + do { \ + if (DBUG) { \ + std::cout << "[DEBUG] " << msg << std::endl; \ + } \ + } while (0) + +#include #include "brpc/policy/couchbase_protocol.h" #include "brpc/proto_base.pb.h" #include "butil/logging.h" -#include #include "butil/macros.h" #include "butil/string_printf.h" #include "butil/sys_byteorder.h" @@ -50,24 +56,24 @@ CouchbaseMetadataTracking* CouchbaseOperations::CouchbaseRequest::metadata_tracking = &common_metadata_tracking; -bool brpc::CouchbaseMetadataTracking::set_bucket_to_collection_manifest( +bool brpc::CouchbaseMetadataTracking::setBucketToCollectionManifest( string server, string bucket, CouchbaseMetadataTracking::CollectionManifest manifest) { // Then update the collection manifest with proper locking { - UniqueLock write_lock(rw_bucket_to_collection_manifest_mutex); - bucket_to_collection_manifest[server][bucket] = manifest; + UniqueLock write_lock(rw_bucket_to_collection_manifest_mutex_); + bucket_to_collection_manifest_[server][bucket] = manifest; } return true; } -bool brpc::CouchbaseMetadataTracking::get_bucket_to_collection_manifest( +bool brpc::CouchbaseMetadataTracking::getBucketToCollectionManifest( string server, string bucket, CouchbaseMetadataTracking::CollectionManifest* manifest) { - SharedLock read_lock(rw_bucket_to_collection_manifest_mutex); - auto it1 = bucket_to_collection_manifest.find(server); - if (it1 == bucket_to_collection_manifest.end()) { + SharedLock read_lock(rw_bucket_to_collection_manifest_mutex_); + auto it1 = bucket_to_collection_manifest_.find(server); + if (it1 == bucket_to_collection_manifest_.end()) { return false; } auto it2 = it1->second.find(bucket); @@ -78,28 +84,29 @@ bool brpc::CouchbaseMetadataTracking::get_bucket_to_collection_manifest( return true; } -bool brpc::CouchbaseMetadataTracking::get_manifest_to_collection_id( +bool brpc::CouchbaseMetadataTracking::getManifestToCollectionId( CouchbaseMetadataTracking::CollectionManifest* manifest, string scope, string collection, uint8_t* collection_id) { if (manifest == nullptr || collection_id == nullptr) { DEBUG_PRINT("Invalid input: manifest or collection_id is null"); return false; } - auto it1 = manifest->scope_to_collectionID_map.find(scope); - if (it1 == manifest->scope_to_collectionID_map.end()) { + auto it1 = manifest->scope_to_collection_id_map.find(scope); + if (it1 == manifest->scope_to_collection_id_map.end()) { DEBUG_PRINT("Scope: " << scope << " not found in manifest"); return false; } auto it2 = it1->second.find(collection); if (it2 == it1->second.end()) { - DEBUG_PRINT("Collection: " << collection << " not found in scope: " << scope); + DEBUG_PRINT("Collection: " << collection + << " not found in scope: " << scope); return false; } *collection_id = it2->second; return true; } -bool CouchbaseMetadataTracking::json_to_collection_manifest( +bool CouchbaseMetadataTracking::jsonToCollectionManifest( const string& json, CouchbaseMetadataTracking::CollectionManifest* manifest) { if (manifest == nullptr) { @@ -109,7 +116,7 @@ bool CouchbaseMetadataTracking::json_to_collection_manifest( // Clear existing data manifest->uid.clear(); - manifest->scope_to_collectionID_map.clear(); + manifest->scope_to_collection_id_map.clear(); if (json.empty()) { DEBUG_PRINT("JSON string is empty"); @@ -162,7 +169,8 @@ bool CouchbaseMetadataTracking::json_to_collection_manifest( // Extract collections if (!scope.HasMember("collections") || !scope["collections"].IsArray()) { - DEBUG_PRINT("Missing or invalid 'collections' field in scope '" << scope_name << "'"); + DEBUG_PRINT("Missing or invalid 'collections' field in scope '" + << scope_name << "'"); return false; } @@ -174,20 +182,23 @@ bool CouchbaseMetadataTracking::json_to_collection_manifest( const BUTIL_RAPIDJSON_NAMESPACE::Value& collection = collections[j]; if (!collection.IsObject()) { - DEBUG_PRINT("Collection at index " << j << " in scope '" << scope_name << "' is not an object"); + DEBUG_PRINT("Collection at index " << j << " in scope '" << scope_name + << "' is not an object"); return false; } // Extract collection name if (!collection.HasMember("name") || !collection["name"].IsString()) { - DEBUG_PRINT("Missing or invalid 'name' field in collection at index " << j << " in scope '" << scope_name << "'"); + DEBUG_PRINT("Missing or invalid 'name' field in collection at index " + << j << " in scope '" << scope_name << "'"); return false; } string collection_name = collection["name"].GetString(); // Extract collection uid (hex string) if (!collection.HasMember("uid") || !collection["uid"].IsString()) { - DEBUG_PRINT("Missing or invalid 'uid' field in collection '" << collection_name << "' in scope '" << scope_name << "'"); + DEBUG_PRINT("Missing or invalid 'uid' field in collection '" + << collection_name << "' in scope '" << scope_name << "'"); return false; } string collection_uid_str = collection["uid"].GetString(); @@ -198,12 +209,18 @@ bool CouchbaseMetadataTracking::json_to_collection_manifest( // Convert hex string to integer unsigned long uid_val = std::stoul(collection_uid_str, nullptr, 16); if (uid_val > 255) { - DEBUG_PRINT("Collection uid '" << collection_uid_str << "' exceeds uint8_t range in collection '" << collection_name << "' in scope '" << scope_name << "'"); + DEBUG_PRINT( + "Collection uid '" + << collection_uid_str << "' exceeds uint8_t range in collection '" + << collection_name << "' in scope '" << scope_name << "'"); return false; } collection_id = static_cast(uid_val); } catch (const std::exception& e) { - DEBUG_PRINT("Failed to parse collection uid '" << collection_uid_str << "' as hex in collection '" << collection_name << "' in scope '" << scope_name << ": " << e.what()); + DEBUG_PRINT("Failed to parse collection uid '" + << collection_uid_str << "' as hex in collection '" + << collection_name << "' in scope '" << scope_name << ": " + << e.what()); return false; } @@ -212,14 +229,15 @@ bool CouchbaseMetadataTracking::json_to_collection_manifest( } // Add scope and its collections to manifest - manifest->scope_to_collectionID_map[scope_name] = std::move(collection_map); + manifest->scope_to_collection_id_map[scope_name] = + std::move(collection_map); } return true; } -uint32_t CouchbaseOperations::CouchbaseRequest::hash_crc32(const char* key, - size_t key_length) { +uint32_t CouchbaseOperations::CouchbaseRequest::hashCrc32(const char* key, + size_t key_length) { static const uint32_t crc32tab[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, @@ -279,14 +297,14 @@ uint32_t CouchbaseOperations::CouchbaseRequest::hash_crc32(const char* key, #endif } -void CouchbaseOperations::CouchbaseRequest::SharedCtor() { +void CouchbaseOperations::CouchbaseRequest::sharedCtor() { _pipelined_count = 0; _cached_size_ = 0; } -void CouchbaseOperations::CouchbaseRequest::SharedDtor() {} +void CouchbaseOperations::CouchbaseRequest::sharedDtor() {} -void CouchbaseOperations::CouchbaseRequest::SetCachedSize(int size) const { +void CouchbaseOperations::CouchbaseRequest::setCachedSize(int size) const { _cached_size_ = size; } @@ -325,10 +343,10 @@ void CouchbaseOperations::CouchbaseRequest::Clear() { // return true; // } -bool CouchbaseOperations::CouchbaseRequest::SelectBucketRequest( +bool CouchbaseOperations::CouchbaseRequest::selectBucketRequest( const butil::StringPiece& bucket_name) { if (bucket_name.empty()) { - DEBUG_PRINT("Empty bucket name"); + DEBUG_PRINT("Empty bucket name"); return false; } // construct the request header @@ -343,11 +361,11 @@ bool CouchbaseOperations::CouchbaseRequest::SelectBucketRequest( 0, 0}; if (_buf.append(&header, sizeof(header))) { - DEBUG_PRINT("Failed to append header to buffer"); + DEBUG_PRINT("Failed to append header to buffer"); return false; } if (_buf.append(bucket_name.data(), bucket_name.size())) { - DEBUG_PRINT("Failed to append bucket name to buffer"); + DEBUG_PRINT("Failed to append bucket name to buffer"); return false; } ++_pipelined_count; @@ -359,7 +377,7 @@ bool CouchbaseOperations::CouchbaseRequest::SelectBucketRequest( // This is typically the first request sent after connecting to the server. // It includes the agent name and a randomly generated connection ID in JSON // format. -bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { +bool CouchbaseOperations::CouchbaseRequest::helloRequest() { std::string agent = "brpc/1.0.0 ("; #ifdef __APPLE__ agent += "Darwin/"; @@ -383,7 +401,7 @@ bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { if (!urandom || fread(raw_id, 1, CONNECTION_ID_SIZE, urandom) != CONNECTION_ID_SIZE) { if (urandom) fclose(urandom); - DEBUG_PRINT("Failed to generate random connection id"); + DEBUG_PRINT("Failed to generate random connection id"); return false; } fclose(urandom); @@ -421,31 +439,31 @@ bool CouchbaseOperations::CouchbaseRequest::HelloRequest() { }; if (_buf.append(&header, sizeof(header))) { - DEBUG_PRINT("Failed to append Hello header to buffer"); + DEBUG_PRINT("Failed to append Hello header to buffer"); return false; } if (_buf.append(key.data(), key_len)) { - DEBUG_PRINT("Failed to append Hello JSON key to buffer"); + DEBUG_PRINT("Failed to append Hello JSON key to buffer"); return false; } if (_buf.append(reinterpret_cast(features), value_len)) { - DEBUG_PRINT("Failed to append Hello features to buffer"); + DEBUG_PRINT("Failed to append Hello features to buffer"); return false; } ++_pipelined_count; return true; } -bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest( +bool CouchbaseOperations::CouchbaseRequest::authenticateRequest( const butil::StringPiece& username, const butil::StringPiece& password) { if (username.empty() || password.empty()) { - DEBUG_PRINT("Empty username or password"); + DEBUG_PRINT("Empty username or password"); return false; } - // insert the features to get enabled, calling function HelloRequest() will do + // insert the features to get enabled, calling function helloRequest() will do // this. - if (!HelloRequest()) { - DEBUG_PRINT("Failed to send HelloRequest for authentication"); + if (!helloRequest()) { + DEBUG_PRINT("Failed to send helloRequest for authentication"); return false; } // Construct the request header @@ -473,7 +491,7 @@ bool CouchbaseOperations::CouchbaseRequest::AuthenticateRequest( auth_str.append(kPadding, sizeof(kPadding)); auth_str.append(password.data(), password.size()); if (_buf.append(auth_str.data(), auth_str.size())) { - DEBUG_PRINT("Failed to append auth string to buffer"); + DEBUG_PRINT("Failed to append auth string to buffer"); return false; } ++_pipelined_count; @@ -562,11 +580,11 @@ CouchbaseOperations::CouchbaseRequest::GetMetadata() const { return metadata; } -void CouchbaseOperations::CouchbaseResponse::SharedCtor() { _cached_size_ = 0; } +void CouchbaseOperations::CouchbaseResponse::sharedCtor() { _cached_size_ = 0; } -void CouchbaseOperations::CouchbaseResponse::SharedDtor() {} +void CouchbaseOperations::CouchbaseResponse::sharedDtor() {} -void CouchbaseOperations::CouchbaseResponse::SetCachedSize(int size) const { +void CouchbaseOperations::CouchbaseResponse::setCachedSize(int size) const { _cached_size_ = size; } @@ -616,7 +634,7 @@ bool CouchbaseOperations::CouchbaseResponse::IsInitialized() const { return !_buf.empty(); } -void CouchbaseOperations::CouchbaseResponse::Swap(CouchbaseResponse* other) { +void CouchbaseOperations::CouchbaseResponse::swap(CouchbaseResponse* other) { if (other != this) { _buf.swap(other->_buf); std::swap(_cached_size_, other->_cached_size_); @@ -633,7 +651,7 @@ CouchbaseOperations::CouchbaseResponse::GetMetadata() const { // =================================================================== -const char* CouchbaseOperations::CouchbaseResponse::status_str(Status st) { +const char* CouchbaseOperations::CouchbaseResponse::statusStr(Status st) { switch (st) { case STATUS_SUCCESS: return "SUCCESS"; @@ -752,28 +770,28 @@ const char* CouchbaseOperations::CouchbaseResponse::status_str(Status st) { } // Helper method to format error messages with status codes -std::string CouchbaseOperations::CouchbaseResponse::format_error_message( +std::string CouchbaseOperations::CouchbaseResponse::formatErrorMessage( uint16_t status_code, const std::string& operation, const std::string& error_msg) { if (error_msg.empty()) { return butil::string_printf("%s failed with status 0x%02x (%s)", operation.c_str(), status_code, - status_str((Status)status_code)); + statusStr((Status)status_code)); } else { return butil::string_printf( "%s failed with status 0x%02x (%s): %s", operation.c_str(), status_code, - status_str((Status)status_code), error_msg.c_str()); + statusStr((Status)status_code), error_msg.c_str()); } } // MUST NOT have extras. // MUST have key. // MUST NOT have value. -bool CouchbaseOperations::CouchbaseRequest::GetOrDelete( +bool CouchbaseOperations::CouchbaseRequest::getOrDelete( uint8_t command, const butil::StringPiece& key, uint8_t coll_id) { // Collection ID uint8_t collection_id = coll_id; - uint16_t VBucket_id = hash_crc32(key.data(), key.size()); + uint16_t VBucket_id = hashCrc32(key.data(), key.size()); const policy::CouchbaseRequestHeader header = { policy::CB_MAGIC_REQUEST, command, butil::HostToNet16( @@ -799,62 +817,68 @@ bool CouchbaseOperations::CouchbaseRequest::GetOrDelete( return true; } -// collectionID fetching either from the metadata cache or if doesn't exist then fetch from the server. -bool CouchbaseOperations::CouchbaseRequest::get_cached_or_fetch_collection_id( +// collectionID fetching either from the metadata cache or if doesn't exist then +// fetch from the server. +bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( string collection_name, uint8_t* coll_id, brpc::CouchbaseMetadataTracking* metadata_tracking, brpc::Channel* channel, const string& server, const string& selected_bucket) { - if (collection_name.empty()) { - DEBUG_PRINT("Empty collection name"); - return false; - } - if (channel == nullptr) { - DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); - return false; - } - if (server.empty()) { - DEBUG_PRINT("Server is empty, make sure to call Authenticate() first"); + if (collection_name.empty()) { + DEBUG_PRINT("Empty collection name"); + return false; + } + if (channel == nullptr) { + DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); + return false; + } + if (server.empty()) { + DEBUG_PRINT("Server is empty, make sure to call Authenticate() first"); + return false; + } + if (selected_bucket.empty()) { + DEBUG_PRINT("No bucket selected, make sure to call SelectBucket() first"); + return false; + } + + brpc::CouchbaseMetadataTracking::CollectionManifest manifest; + // check if the server/bucket exists in the cached collection manifest + if (!metadata_tracking->getBucketToCollectionManifest(server, selected_bucket, + &manifest)) { + DEBUG_PRINT("No cached collection manifest found for bucket " + << selected_bucket << " on server " << server + << ", fetching from server"); + // No cached manifest found, fetch from server + if (!refreshCollectionManifest(channel, server, selected_bucket)) { return false; } - if (selected_bucket.empty()) { - DEBUG_PRINT("No bucket selected, make sure to call SelectBucket() first"); + // local cache will also be updated in refreshCollectionManifest + // get the reference to collectionID from local cache + if (!getLocalCachedCollectionId(selected_bucket, "_default", + collection_name, coll_id)) { + // collectionID not found in the latest manifest fetched from server return false; } - - brpc::CouchbaseMetadataTracking::CollectionManifest manifest; - //check if the server/bucket exists in the cached collection manifest - if (!metadata_tracking->get_bucket_to_collection_manifest( - server, selected_bucket, &manifest)) { - DEBUG_PRINT("No cached collection manifest found for bucket " << selected_bucket << " on server " << server << ", fetching from server"); - // No cached manifest found, fetch from server - if(!RefreshCollectionManifest(channel, server, selected_bucket)) { - return false; - } - // local cache will also be updated in RefreshCollectionManifest - // get the reference to collectionID from local cache - if( !getLocalCachedCollectionId(selected_bucket, "_default", collection_name, coll_id)) { - // collectionID not found in the latest manifest fetched from server - return false; - } - //collectionID has been found in the latest manifest fetched from server and is stored in coll_id - return true; - } - else { + // collectionID has been found in the latest manifest fetched from server + // and is stored in coll_id + return true; + } else { // check if collection name to id mapping exists. - if (!metadata_tracking->get_manifest_to_collection_id( + if (!metadata_tracking->getManifestToCollectionId( &manifest, "_default", collection_name, coll_id)) { // Just to verify that the collectionID does not exist in the manifest // refresh manifest from server and try again - if(!RefreshCollectionManifest(channel, server, selected_bucket)) { + if (!refreshCollectionManifest(channel, server, selected_bucket)) { return false; } - // local cache will also be updated in RefreshCollectionManifest + // local cache will also be updated in refreshCollectionManifest // get the reference to collectionID from local cache - if( !getLocalCachedCollectionId(selected_bucket, "_default", collection_name, coll_id)) { + if (!getLocalCachedCollectionId(selected_bucket, "_default", + collection_name, coll_id)) { // collectionID not found in the latest manifest fetched from server - return false; + return false; } - //collectionID has been found in the latest manifest fetched from server and is stored in coll_id + // collectionID has been found in the latest manifest fetched from server + // and is stored in coll_id return true; } // update the local cache with the manifest in global cache @@ -864,63 +888,84 @@ bool CouchbaseOperations::CouchbaseRequest::get_cached_or_fetch_collection_id( } } - -bool CouchbaseOperations::CouchbaseRequest::GetRequest( +bool CouchbaseOperations::CouchbaseRequest::getRequest( const butil::StringPiece& key, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { - DEBUG_PRINT("GetRequest called with key: " << key << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); + DEBUG_PRINT("getRequest called with key: " + << key << ", collection_name: " << collection_name + << ", server: " << server << ", bucket: " << bucket); uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. - if(local_collection_manifest_cache->empty()){ - DEBUG_PRINT("Local collection manifest cache is empty in GetRequest"); + if (local_collection_manifest_cache->empty()) { + DEBUG_PRINT("Local collection manifest cache is empty in getRequest"); // if local cache is empty, goto global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { - DEBUG_PRINT("Failed to get collection id from global cache or server in GetRequest"); + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { + DEBUG_PRINT( + "Failed to get collection id from global cache or server in " + "getRequest"); return false; } } - // check if the collection id is available in the local cache - else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { - DEBUG_PRINT("Collection id not found in local cache in GetRequest"); + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, + &coll_id)) { + DEBUG_PRINT("Collection id not found in local cache in getRequest"); // if not check in the global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { - DEBUG_PRINT("Failed to get collection id from global cache or server in GetRequest"); + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { + DEBUG_PRINT( + "Failed to get collection id from global cache or server in " + "getRequest"); return false; } } } - DEBUG_PRINT("GetRequest using coll_id: " << (int)coll_id); - return GetOrDelete(policy::CB_BINARY_GET, key, coll_id); + DEBUG_PRINT("getRequest using coll_id: " << (int)coll_id); + return getOrDelete(policy::CB_BINARY_GET, key, coll_id); } -bool CouchbaseOperations::CouchbaseRequest::DeleteRequest( +bool CouchbaseOperations::CouchbaseRequest::deleteRequest( const butil::StringPiece& key, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { - DEBUG_PRINT("DeleteRequest called with key: " << key << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); + DEBUG_PRINT("deleteRequest called with key: " + << key << ", collection_name: " << collection_name + << ", server: " << server << ", bucket: " << bucket); uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. - if(local_collection_manifest_cache->empty()){ - DEBUG_PRINT("Local collection manifest cache is empty in DeleteRequest"); + if (local_collection_manifest_cache->empty()) { + DEBUG_PRINT("Local collection manifest cache is empty in deleteRequest"); // if local cache is empty, goto global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { - DEBUG_PRINT("Failed to get collection id from global cache or server in DeleteRequest"); + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { + DEBUG_PRINT( + "Failed to get collection id from global cache or server in " + "deleteRequest"); return false; } } - // check if the collection id is available in the local cache - else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { - DEBUG_PRINT("Collection id not found in local cache in DeleteRequest"); + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, + &coll_id)) { + DEBUG_PRINT("Collection id not found in local cache in deleteRequest"); // if not check in the global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { - DEBUG_PRINT("Failed to get collection id from global cache or server in DeleteRequest"); + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { + DEBUG_PRINT( + "Failed to get collection id from global cache or server in " + "deleteRequest"); return false; } } } - DEBUG_PRINT("DeleteRequest using coll_id: " << (int)coll_id); - return GetOrDelete(policy::CB_BINARY_DELETE, key, coll_id); + DEBUG_PRINT("deleteRequest using coll_id: " << (int)coll_id); + return getOrDelete(policy::CB_BINARY_DELETE, key, coll_id); } struct FlushHeaderWithExtras { @@ -974,7 +1019,7 @@ BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); // 0| Flags | // +---------------+---------------+---------------+---------------+ // Total 4 bytes -bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, +bool CouchbaseOperations::CouchbaseResponse::popGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value) { const size_t n = _buf.size(); @@ -996,10 +1041,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, } if (header.status != (uint16_t)STATUS_SUCCESS) { if (DBUG && header.extras_length != 0) { - DEBUG_PRINT("GET response must not have flags"); + DEBUG_PRINT("GET response must not have flags"); } if (DBUG && header.key_length != 0) { - DEBUG_PRINT("GET response must not have key"); + DEBUG_PRINT("GET response must not have key"); } const int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; @@ -1012,9 +1057,9 @@ bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, if (value_size > 0) { std::string error_msg; _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "GET operation", error_msg); + _err = formatErrorMessage(header.status, "GET operation", error_msg); } else { - _err = format_error_message(header.status, "GET operation"); + _err = formatErrorMessage(header.status, "GET operation"); } return false; } @@ -1051,11 +1096,11 @@ bool CouchbaseOperations::CouchbaseResponse::PopGet(butil::IOBuf* value, return true; } -bool CouchbaseOperations::CouchbaseResponse::PopGet(std::string* value, +bool CouchbaseOperations::CouchbaseResponse::popGet(std::string* value, uint32_t* flags, uint64_t* cas_value) { butil::IOBuf tmp; - if (PopGet(&tmp, flags, cas_value)) { + if (popGet(&tmp, flags, cas_value)) { tmp.copy_to(value); return true; } @@ -1065,12 +1110,12 @@ bool CouchbaseOperations::CouchbaseResponse::PopGet(std::string* value, // MUST NOT have extras // MUST NOT have key // MUST NOT have value -bool CouchbaseOperations::CouchbaseResponse::PopDelete() { - return PopStore(policy::CB_BINARY_DELETE, NULL); +bool CouchbaseOperations::CouchbaseResponse::popDelete() { + return popStore(policy::CB_BINARY_DELETE, NULL); } // Warning: Not tested // bool CouchbaseOperations::CouchbaseResponse::PopFlush() { -// return PopStore(policy::CB_BINARY_FLUSH, NULL); +// return popStore(policy::CB_BINARY_FLUSH, NULL); // } struct StoreHeaderWithExtras { @@ -1094,14 +1139,14 @@ const size_t STORE_EXTRAS = // 4| Expiration | // +---------------+---------------+---------------+---------------+ // Total 8 bytes -bool CouchbaseOperations::CouchbaseRequest::Store( +bool CouchbaseOperations::CouchbaseRequest::store( uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, uint8_t coll_id) { // add collection id // uint16_t collection_id = 0x00; uint8_t collection_id = coll_id; - uint16_t vBucket_id = hash_crc32(key.data(), key.size()); + uint16_t vBucket_id = hashCrc32(key.data(), key.size()); StoreHeaderWithExtras header_with_extras = { {policy::CB_MAGIC_REQUEST, command, butil::HostToNet16(key.size() + @@ -1133,7 +1178,7 @@ bool CouchbaseOperations::CouchbaseRequest::Store( // MUST NOT have extras // MUST NOT have key // MUST NOT have value -bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, +bool CouchbaseOperations::CouchbaseResponse::popStore(uint8_t command, uint64_t* cas_value) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; @@ -1151,10 +1196,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, return false; } if (DBUG && header.extras_length != 0) { - DEBUG_PRINT("STORE response must not have flags"); + DEBUG_PRINT("STORE response must not have flags"); } if (DBUG && header.key_length != 0) { - DEBUG_PRINT("STORE response must not have key"); + DEBUG_PRINT("STORE response must not have key"); } int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; @@ -1164,17 +1209,16 @@ bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, if (value_size > 0) { std::string error_msg; _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, - couchbase_binary_command_to_string(command), - error_msg); + _err = formatErrorMessage( + header.status, couchbaseBinaryCommandToString(command), error_msg); } else { - _err = format_error_message(header.status, - couchbase_binary_command_to_string(command)); + _err = formatErrorMessage(header.status, + couchbaseBinaryCommandToString(command)); } return false; } if (DBUG && value_size != 0) { - DEBUG_PRINT("STORE response must not have value, actually=" << value_size); + DEBUG_PRINT("STORE response must not have value, actually=" << value_size); } _buf.pop_front(sizeof(header) + header.total_body_length); if (cas_value) { @@ -1185,7 +1229,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopStore(uint8_t command, } const char* -CouchbaseOperations::CouchbaseResponse::couchbase_binary_command_to_string( +CouchbaseOperations::CouchbaseResponse::couchbaseBinaryCommandToString( uint8_t cmd) { switch (cmd) { case 0x1f: @@ -1275,35 +1319,47 @@ CouchbaseOperations::CouchbaseResponse::couchbase_binary_command_to_string( } } -bool CouchbaseOperations::CouchbaseRequest::UpsertRequest( +bool CouchbaseOperations::CouchbaseRequest::upsertRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { - DEBUG_PRINT("UpsertRequest called with key: " << key << ", value: " << value << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); + DEBUG_PRINT("upsertRequest called with key: " + << key << ", value: " << value + << ", collection_name: " << collection_name + << ", server: " << server << ", bucket: " << bucket); uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. - if(local_collection_manifest_cache->empty()){ - DEBUG_PRINT("Local collection manifest cache is empty in UpsertRequest"); + if (local_collection_manifest_cache->empty()) { + DEBUG_PRINT("Local collection manifest cache is empty in upsertRequest"); // if local cache is empty, goto global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { - DEBUG_PRINT("Failed to get collection id from global cache or server in UpsertRequest"); + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { + DEBUG_PRINT( + "Failed to get collection id from global cache or server in " + "upsertRequest"); return false; } } - // check if the collection id is available in the local cache - else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { - DEBUG_PRINT("Collection id not found in local cache in UpsertRequest"); + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, + &coll_id)) { + DEBUG_PRINT("Collection id not found in local cache in upsertRequest"); // if not check in the global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { - DEBUG_PRINT("Failed to get collection id from global cache or server in UpsertRequest"); + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { + DEBUG_PRINT( + "Failed to get collection id from global cache or server in " + "upsertRequest"); return false; } } } - DEBUG_PRINT("UpsertRequest using coll_id: " << (int)coll_id); - return Store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value, + DEBUG_PRINT("upsertRequest using coll_id: " << (int)coll_id); + return store(policy::CB_BINARY_SET, key, value, flags, exptime, cas_value, coll_id); } @@ -1336,7 +1392,7 @@ bool CouchbaseOperations::CouchbaseRequest::UpsertRequest( // return true; // } -bool CouchbaseOperations::CouchbaseRequest::GetCollectionManifest() { +bool CouchbaseOperations::CouchbaseRequest::getCollectionManifest() { const policy::CouchbaseRequestHeader header = { policy::CB_MAGIC_REQUEST, policy::CB_GET_COLLECTIONS_MANIFEST, @@ -1355,10 +1411,11 @@ bool CouchbaseOperations::CouchbaseRequest::GetCollectionManifest() { return true; } -bool CouchbaseOperations::CouchbaseRequest::RefreshCollectionManifest(brpc::Channel* channel, const string& server, const string& bucket) { +bool CouchbaseOperations::CouchbaseRequest::refreshCollectionManifest( + brpc::Channel* channel, const string& server, const string& bucket) { // first fetch the manifest // then compare the UID with the cached one - if (channel == nullptr) { + if (channel == nullptr) { DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); return false; } @@ -1373,110 +1430,134 @@ bool CouchbaseOperations::CouchbaseRequest::RefreshCollectionManifest(brpc::Chan CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; CouchbaseOperations::CouchbaseResponse temp_get_manifest_response; brpc::Controller temp_cntl; - temp_get_manifest_request.GetCollectionManifest(); + temp_get_manifest_request.getCollectionManifest(); channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, - &temp_get_manifest_response, NULL); + &temp_get_manifest_response, NULL); if (temp_cntl.Failed()) { - DEBUG_PRINT("Failed to get collection manifest: bRPC controller error " << temp_cntl.ErrorText()); return false; + DEBUG_PRINT("Failed to get collection manifest: bRPC controller error " + << temp_cntl.ErrorText()); + return false; } string manifest_json; - if (!temp_get_manifest_response.PopManifest(&manifest_json)) { - DEBUG_PRINT("Failed to parse response for refreshing collection Manifest: " << temp_get_manifest_response.LastError()); return false; + if (!temp_get_manifest_response.popManifest(&manifest_json)) { + DEBUG_PRINT("Failed to parse response for refreshing collection Manifest: " + << temp_get_manifest_response.lastError()); + return false; } // Compare the UID with the cached one // If they are different, refresh the cache brpc::CouchbaseMetadataTracking::CollectionManifest manifest; - if(!common_metadata_tracking.json_to_collection_manifest(manifest_json, - &manifest)){ - DEBUG_PRINT("Failed to parse collection manifest JSON"); + if (!common_metadata_tracking.jsonToCollectionManifest(manifest_json, + &manifest)) { + DEBUG_PRINT("Failed to parse collection manifest JSON"); return false; } brpc::CouchbaseMetadataTracking::CollectionManifest cached_manifest; - if(!common_metadata_tracking.get_bucket_to_collection_manifest(server, bucket, - &cached_manifest)){ + if (!common_metadata_tracking.getBucketToCollectionManifest( + server, bucket, &cached_manifest)) { // No cached manifest found, set the new one - if(!common_metadata_tracking.set_bucket_to_collection_manifest(server, bucket, - manifest)){ - DEBUG_PRINT("Failed to cache collection manifest for bucket " << bucket << " on server " << server); return false; + if (!common_metadata_tracking.setBucketToCollectionManifest(server, bucket, + manifest)) { + DEBUG_PRINT("Failed to cache collection manifest for bucket " + << bucket << " on server " << server); + return false; } - DEBUG_PRINT("Cached collection manifest for bucket " << bucket << " on server " << server); - // also update the local cache + DEBUG_PRINT("Cached collection manifest for bucket " + << bucket << " on server " << server); + // also update the local cache (*local_collection_manifest_cache)[bucket] = manifest; return true; - } - else if(manifest.uid != cached_manifest.uid) { - DEBUG_PRINT("Collection manifest has changed for bucket " << bucket << " on server " << server); - if(!common_metadata_tracking.set_bucket_to_collection_manifest(server, bucket, - manifest)){ - DEBUG_PRINT("Failed to update cached collection manifest for bucket " << bucket << " on server " << server); return false; + } else if (manifest.uid != cached_manifest.uid) { + DEBUG_PRINT("Collection manifest has changed for bucket " + << bucket << " on server " << server); + if (!common_metadata_tracking.setBucketToCollectionManifest(server, bucket, + manifest)) { + DEBUG_PRINT("Failed to update cached collection manifest for bucket " + << bucket << " on server " << server); + return false; } - DEBUG_PRINT("Updated cached collection manifest for bucket " << bucket << " on server " << server); + DEBUG_PRINT("Updated cached collection manifest for bucket " + << bucket << " on server " << server); // also update the local cache if needed. - if(local_collection_manifest_cache->find(bucket) != - local_collection_manifest_cache->end()){ + if (local_collection_manifest_cache->find(bucket) != + local_collection_manifest_cache->end()) { // if the bucket already exists in the local cache, check the UID - if( (*local_collection_manifest_cache)[bucket].uid != manifest.uid){ + if ((*local_collection_manifest_cache)[bucket].uid != manifest.uid) { // if the UID is different, update the local cache (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Updated local collection manifest cache for bucket " << bucket << " on server " << server); + DEBUG_PRINT("Updated local collection manifest cache for bucket " + << bucket << " on server " << server); } - } - else{ + } else { // if the bucket does not exist in the local cache, add it (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Added to local collection manifest cache for bucket " << bucket << " on server " << server); + DEBUG_PRINT("Added to local collection manifest cache for bucket " + << bucket << " on server " << server); } return true; - } - else{ - DEBUG_PRINT("Collection manifest is already up-to-date for bucket " << bucket << " on server " << server); - if(local_collection_manifest_cache->find(bucket) != - local_collection_manifest_cache->end()){ + } else { + DEBUG_PRINT("Collection manifest is already up-to-date for bucket " + << bucket << " on server " << server); + if (local_collection_manifest_cache->find(bucket) != + local_collection_manifest_cache->end()) { // if the bucket already exists in the local cache, check the UID - if( (*local_collection_manifest_cache)[bucket].uid != manifest.uid){ + if ((*local_collection_manifest_cache)[bucket].uid != manifest.uid) { // if the UID is different, update the local cache (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Updated local collection manifest cache for bucket " << bucket << " on server " << server); + DEBUG_PRINT("Updated local collection manifest cache for bucket " + << bucket << " on server " << server); } - } - else{ + } else { // if the bucket does not exist in the local cache, add it (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Added to local collection manifest cache for bucket " << bucket << " on server " << server); + DEBUG_PRINT("Added to local collection manifest cache for bucket " + << bucket << " on server " << server); } return false; } } -bool CouchbaseOperations::CouchbaseRequest::AddRequest( +bool CouchbaseOperations::CouchbaseRequest::addRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name, brpc::Channel* channel, const string& server, const string& bucket) { - DEBUG_PRINT("AddRequest called with key: " << key << ", value: " << value << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); + DEBUG_PRINT("addRequest called with key: " + << key << ", value: " << value + << ", collection_name: " << collection_name + << ", server: " << server << ", bucket: " << bucket); uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. - if(local_collection_manifest_cache->empty()){ - DEBUG_PRINT("Local collection manifest cache is empty in AddRequest"); + if (local_collection_manifest_cache->empty()) { + DEBUG_PRINT("Local collection manifest cache is empty in addRequest"); // if local cache is empty, goto global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { - DEBUG_PRINT("Failed to get collection id from global cache or server in AddRequest"); + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { + DEBUG_PRINT( + "Failed to get collection id from global cache or server in " + "addRequest"); return false; } } - // check if the collection id is available in the local cache - else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { - DEBUG_PRINT("Collection id not found in local cache in AddRequest"); + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, + &coll_id)) { + DEBUG_PRINT("Collection id not found in local cache in addRequest"); // if not check in the global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { - DEBUG_PRINT("Failed to get collection id from global cache or server in AddRequest"); + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { + DEBUG_PRINT( + "Failed to get collection id from global cache or server in " + "addRequest"); return false; } } } - DEBUG_PRINT("AddRequest using coll_id: " << (int)coll_id); - return Store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, + DEBUG_PRINT("addRequest using coll_id: " << (int)coll_id); + return store(policy::CB_BINARY_ADD, key, value, flags, exptime, cas_value, coll_id); } @@ -1490,7 +1571,7 @@ bool CouchbaseOperations::CouchbaseRequest::AddRequest( // const string& bucket) { // uint8_t coll_id = 0; // default collection ID // if(collection_name != "_default"){ -// if(!get_cached_or_fetch_collection_id(collection_name, &coll_id, +// if(!getCachedOrFetchCollectionId(collection_name, &coll_id, // metadata_tracking, channel, server, bucket)){ // return false; // } @@ -1500,7 +1581,7 @@ bool CouchbaseOperations::CouchbaseRequest::AddRequest( // coll_id); // } -bool CouchbaseOperations::CouchbaseRequest::AppendRequest( +bool CouchbaseOperations::CouchbaseRequest::appendRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name, brpc::Channel* channel, const string& server, @@ -1512,25 +1593,30 @@ bool CouchbaseOperations::CouchbaseRequest::AppendRequest( uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. - if(!local_collection_manifest_cache->empty()){ + if (!local_collection_manifest_cache->empty()) { // if local cache is empty, goto global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } - // check if the collection id is available in the local cache - else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, + &coll_id)) { // if not check in the global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } } - return Store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value, + return store(policy::CB_BINARY_APPEND, key, value, flags, exptime, cas_value, coll_id); } -bool CouchbaseOperations::CouchbaseRequest::PrependRequest( +bool CouchbaseOperations::CouchbaseRequest::prependRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name, brpc::Channel* channel, const string& server, @@ -1542,44 +1628,49 @@ bool CouchbaseOperations::CouchbaseRequest::PrependRequest( uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. - if(!local_collection_manifest_cache->empty()){ + if (!local_collection_manifest_cache->empty()) { // if local cache is empty, goto global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } - // check if the collection id is available in the local cache - else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { + // check if the collection id is available in the local cache + else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, + &coll_id)) { // if not check in the global cache or fetch from server - if (!get_cached_or_fetch_collection_id(collection_name, &coll_id, metadata_tracking, channel, server, bucket)) { + if (!getCachedOrFetchCollectionId(collection_name, &coll_id, + metadata_tracking, channel, server, + bucket)) { return false; } } } - return Store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, cas_value, + return store(policy::CB_BINARY_PREPEND, key, value, flags, exptime, cas_value, coll_id); } -bool CouchbaseOperations::CouchbaseResponse::PopUpsert(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_SET, cas_value); +bool CouchbaseOperations::CouchbaseResponse::popUpsert(uint64_t* cas_value) { + return popStore(policy::CB_BINARY_SET, cas_value); } -bool CouchbaseOperations::CouchbaseResponse::PopAdd(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_ADD, cas_value); +bool CouchbaseOperations::CouchbaseResponse::popAdd(uint64_t* cas_value) { + return popStore(policy::CB_BINARY_ADD, cas_value); } // Warning: Not tested // bool CouchbaseOperations::CouchbaseResponse::PopReplace(uint64_t* cas_value) // { -// return PopStore(policy::CB_BINARY_REPLACE, cas_value); +// return popStore(policy::CB_BINARY_REPLACE, cas_value); // } -bool CouchbaseOperations::CouchbaseResponse::PopAppend(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_APPEND, cas_value); +bool CouchbaseOperations::CouchbaseResponse::popAppend(uint64_t* cas_value) { + return popStore(policy::CB_BINARY_APPEND, cas_value); } -bool CouchbaseOperations::CouchbaseResponse::PopPrepend(uint64_t* cas_value) { - return PopStore(policy::CB_BINARY_PREPEND, cas_value); +bool CouchbaseOperations::CouchbaseResponse::popPrepend(uint64_t* cas_value) { + return popStore(policy::CB_BINARY_PREPEND, cas_value); } -bool CouchbaseOperations::CouchbaseResponse::PopSelectBucket( +bool CouchbaseOperations::CouchbaseResponse::popSelectBucket( uint64_t* cas_value, std::string bucket_name) { - if (PopStore(policy::CB_SELECT_BUCKET, cas_value) == false) { + if (popStore(policy::CB_SELECT_BUCKET, cas_value) == false) { DEBUG_PRINT("Failed to select bucket: " << _err); return false; } @@ -1588,7 +1679,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopSelectBucket( return true; } // Collection-related response method -bool CouchbaseOperations::CouchbaseResponse::PopCollectionId( +bool CouchbaseOperations::CouchbaseResponse::popCollectionId( uint8_t* collection_id) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; @@ -1619,9 +1710,9 @@ bool CouchbaseOperations::CouchbaseResponse::PopCollectionId( std::string err_msg; _buf.cutn(&err_msg, value_size); _err = - format_error_message(header.status, "Collection ID request", err_msg); + formatErrorMessage(header.status, "Collection ID request", err_msg); } else { - _err = format_error_message(header.status, "Collection ID request"); + _err = formatErrorMessage(header.status, "Collection ID request"); } return false; } @@ -1659,7 +1750,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopCollectionId( return true; } -bool CouchbaseOperations::CouchbaseResponse::PopManifest( +bool CouchbaseOperations::CouchbaseResponse::popManifest( std::string* manifest_json) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; @@ -1695,10 +1786,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopManifest( if (value_size > 0) { std::string err_msg; _buf.cutn(&err_msg, value_size); - _err = format_error_message(header.status, "Get Collections Manifest", - err_msg); + _err = formatErrorMessage(header.status, "Get Collections Manifest", + err_msg); } else { - _err = format_error_message(header.status, "Get Collections Manifest"); + _err = formatErrorMessage(header.status, "Get Collections Manifest"); } return false; } @@ -1750,7 +1841,7 @@ const size_t INCR_EXTRAS = // 16| Expiration | // +---------------+---------------+---------------+---------------+ // Total 20 bytes -bool CouchbaseOperations::CouchbaseRequest::Counter( +bool CouchbaseOperations::CouchbaseRequest::counter( uint8_t command, const butil::StringPiece& key, uint64_t delta, uint64_t initial_value, uint32_t exptime) { IncrHeaderWithExtras header_with_extras = { @@ -1806,7 +1897,7 @@ bool CouchbaseOperations::CouchbaseRequest::Counter( // | | // +---------------+---------------+---------------+---------------+ // Total 8 bytes -bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, +bool CouchbaseOperations::CouchbaseResponse::popCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value) { const size_t n = _buf.size(); @@ -1827,10 +1918,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, return false; } if (DBUG && header.extras_length != 0) { - DEBUG_PRINT("INCR/DECR response must not have flags"); + DEBUG_PRINT("INCR/DECR response must not have flags"); } if (DBUG && header.key_length != 0) { - DEBUG_PRINT("INCR/DECR response must not have key"); + DEBUG_PRINT("INCR/DECR response must not have key"); } const int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; @@ -1844,9 +1935,9 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, std::string error_msg; _buf.cutn(&error_msg, value_size); _err = - format_error_message(header.status, "Counter operation", error_msg); + formatErrorMessage(header.status, "Counter operation", error_msg); } else { - _err = format_error_message(header.status, "Counter operation"); + _err = formatErrorMessage(header.status, "Counter operation"); } } return false; @@ -1868,11 +1959,11 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, // Warning: Not tested // bool CouchbaseOperations::CouchbaseResponse::PopIncrement(uint64_t* // new_value, uint64_t* cas_value) { -// return PopCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); +// return popCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); // } // bool CouchbaseOperations::CouchbaseResponse::PopDecrement(uint64_t* // new_value, uint64_t* cas_value) { -// return PopCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); +// return popCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); // } // MUST have extras. @@ -1922,7 +2013,7 @@ bool CouchbaseOperations::CouchbaseResponse::PopCounter(uint8_t command, // MUST NOT have extras. // MUST NOT have key. // MUST NOT have value. -bool CouchbaseOperations::CouchbaseRequest::VersionRequest() { +bool CouchbaseOperations::CouchbaseRequest::versionRequest() { const policy::CouchbaseRequestHeader header = {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_VERSION, 0, @@ -1944,10 +2035,10 @@ bool CouchbaseOperations::CouchbaseRequest::VersionRequest() { // MUST have value. // Warning: Not tested // bool CouchbaseOperations::CouchbaseResponse::PopTouch() { -// return PopStore(policy::CB_BINARY_TOUCH, NULL); +// return popStore(policy::CB_BINARY_TOUCH, NULL); // } -bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { +bool CouchbaseOperations::CouchbaseResponse::popVersion(std::string* version) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; if (n < sizeof(header)) { @@ -1966,10 +2057,10 @@ bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { return false; } if (DBUG && header.extras_length != 0) { - DEBUG_PRINT("VERSION response must not have flags"); + DEBUG_PRINT("VERSION response must not have flags"); } if (DBUG && header.key_length != 0) { - DEBUG_PRINT("VERSION response must not have key"); + DEBUG_PRINT("VERSION response must not have key"); } const int value_size = (int)header.total_body_length - (int)header.extras_length - (int)header.key_length; @@ -1982,9 +2073,9 @@ bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { if (value_size > 0) { std::string error_msg; _buf.cutn(&error_msg, value_size); - _err = format_error_message(header.status, "Version request", error_msg); + _err = formatErrorMessage(header.status, "Version request", error_msg); } else { - _err = format_error_message(header.status, "Version request"); + _err = formatErrorMessage(header.status, "Version request"); } return false; } @@ -1996,47 +2087,56 @@ bool CouchbaseOperations::CouchbaseResponse::PopVersion(std::string* version) { return true; } -bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, - const string& value, string collection_name, - CouchbaseOperations::Result *result, brpc::Channel *channel, - const string& server, const string& bucket, - CouchbaseOperations::CouchbaseRequest *request, - CouchbaseOperations::CouchbaseResponse *response) { - if(channel == nullptr){ +bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, + const string& value, string collection_name, + CouchbaseOperations::Result* result, brpc::Channel* channel, + const string& server, const string& bucket, + CouchbaseOperations::CouchbaseRequest* request, + CouchbaseOperations::CouchbaseResponse* response) { + if (channel == nullptr) { DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); - result->error_message = "No channel found, make sure to call Authenticate() first"; + result->error_message = + "No channel found, make sure to call Authenticate() first"; return false; } - if(server.empty()){ + if (server.empty()) { DEBUG_PRINT("Server is empty, make sure to call Authenticate() first"); - result->error_message = "Server is empty, make sure to call Authenticate() first"; + result->error_message = + "Server is empty, make sure to call Authenticate() first"; return false; } - if(bucket.empty()){ + if (bucket.empty()) { DEBUG_PRINT("No bucket selected, make sure to call SelectBucket() first"); - result->error_message = "No bucket selected, make sure to call SelectBucket() first"; + result->error_message = + "No bucket selected, make sure to call SelectBucket() first"; return false; } brpc::Controller cntl; bool request_created = false; - switch(op_type){ + switch (op_type) { case CouchbaseOperations::GET: - request_created = request->GetRequest(key, collection_name, channel, server, bucket); + request_created = + request->getRequest(key, collection_name, channel, server, bucket); break; case CouchbaseOperations::UPSERT: - request_created = request->UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request_created = request->upsertRequest( + key, value, 0, 0, 0, collection_name, channel, server, bucket); break; case CouchbaseOperations::ADD: - request_created = request->AddRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request_created = request->addRequest( + key, value, 0, 0, 0, collection_name, channel, server, bucket); break; case CouchbaseOperations::APPEND: - request_created = request->AppendRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request_created = request->appendRequest( + key, value, 0, 0, 0, collection_name, channel, server, bucket); break; case CouchbaseOperations::PREPEND: - request_created = request->PrependRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request_created = request->prependRequest( + key, value, 0, 0, 0, collection_name, channel, server, bucket); break; case CouchbaseOperations::DELETE: - request_created = request->DeleteRequest(key, collection_name, channel, server, bucket); + request_created = + request->deleteRequest(key, collection_name, channel, server, bucket); break; default: DEBUG_PRINT("Unsupported operation type"); @@ -2045,71 +2145,82 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, result->error_message = "Unsupported operation type"; return false; } - if(!request_created){ + if (!request_created) { DEBUG_PRINT("CollectionID does not exist." << op_type); result->success = false; result->value = ""; - result->error_message = "CollectionID does not exist." + std::to_string(op_type); - result->status_code = 0x88; // using 0x88 as the only possible failure code that indicates the collectionID is not found + result->error_message = + "CollectionID does not exist." + std::to_string(op_type); + result->status_code = 0x88; // using 0x88 as the only possible failure code + // that indicates the collectionID is not found return false; } channel->CallMethod(NULL, &cntl, request, response, NULL); if (cntl.Failed()) { - DEBUG_PRINT("Failed to perform operation on key: " << key << " to Couchbase: " << cntl.ErrorText()); + DEBUG_PRINT("Failed to perform operation on key: " + << key << " to Couchbase: " << cntl.ErrorText()); result->success = false; result->value = ""; result->error_message = cntl.ErrorText(); return false; } - if(op_type == CouchbaseOperations::GET) { + if (op_type == CouchbaseOperations::GET) { string value; uint32_t flags = 0; uint64_t cas = 0; - if (response->PopGet(&value, &flags, &cas) == false) { + if (response->popGet(&value, &flags, &cas) == false) { result->success = false; result->value = ""; - result->error_message = response->LastError(); + result->error_message = response->lastError(); result->status_code = response->_status_code; - if(result->status_code == 0x88) { - DEBUG_PRINT("CollectionID does not exist on server, need to refresh collection manifest from server"); - // could have called sendRequest recursively, - // but if somehow the collectionID keeps on chaning, it would lead to infinite recursion and stack overflow in the end. - // so we retry once here instead and return failure if it still fails. - - // (0x88) unknown collection, this means that the collection_manifest has been updated on the server side. - // The collectionID present in the local cache/global cache is no longer valid. - // This can happen if a collection is deleted and recreated with the same name. - if (!request->RefreshCollectionManifest(channel, server, bucket)) { + if (result->status_code == 0x88) { + DEBUG_PRINT( + "CollectionID does not exist on server, need to refresh collection " + "manifest from server"); + // could have called sendRequest recursively, + // but if somehow the collectionID keeps on chaning, it would lead to + // infinite recursion and stack overflow in the end. so we retry once + // here instead and return failure if it still fails. + + // (0x88) unknown collection, this means that the collection_manifest + // has been updated on the server side. The collectionID present in the + // local cache/global cache is no longer valid. This can happen if a + // collection is deleted and recreated with the same name. + if (!request->refreshCollectionManifest(channel, server, bucket)) { DEBUG_PRINT("Failed to refresh collection manifest"); result->error_message = "Failed to refresh collection manifest"; } else { DEBUG_PRINT("Successfully refreshed collection manifest"); - //retry the request; + // retry the request; request->Clear(); response->Clear(); cntl.Reset(); - if(!request->GetRequest(key, collection_name, channel, server, bucket)){ + if (!request->getRequest(key, collection_name, channel, server, + bucket)) { DEBUG_PRINT("CollectionID does not exist."); result->success = false; result->value = ""; result->error_message = "CollectionID does not exist."; - result->status_code = 0x88; // using 0x88 as the only possible failure code that indicates the collectionID is not found + result->status_code = + 0x88; // using 0x88 as the only possible failure code that + // indicates the collectionID is not found return false; } channel->CallMethod(NULL, &cntl, request, response, NULL); if (cntl.Failed()) { - DEBUG_PRINT("Failed to perform operation on key: " << key << " to Couchbase: " << cntl.ErrorText()); + DEBUG_PRINT("Failed to perform operation on key: " + << key << " to Couchbase: " << cntl.ErrorText()); result->success = false; result->value = ""; result->error_message = cntl.ErrorText(); - return false; // return on failure + return false; // return on failure } - if (response->PopGet(&value, &flags, &cas) == false) { + if (response->popGet(&value, &flags, &cas) == false) { result->success = false; result->value = ""; - result->error_message = response->LastError(); + result->error_message = response->lastError(); result->status_code = response->_status_code; - return false; // return on failure + return false; // return on failure } // Successfully got the value after retry result->success = true; @@ -2125,26 +2236,25 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, result->value = value; result->status_code = 0; return true; - } - else{ + } else { uint64_t cas_value = 0; - //pop response on the basis of operation type + // pop response on the basis of operation type bool pop_success = false; - switch(op_type){ + switch (op_type) { case CouchbaseOperations::UPSERT: - pop_success = response->PopUpsert(&cas_value); + pop_success = response->popUpsert(&cas_value); break; case CouchbaseOperations::ADD: - pop_success = response->PopAdd(&cas_value); + pop_success = response->popAdd(&cas_value); break; case CouchbaseOperations::APPEND: - pop_success = response->PopAppend(&cas_value); + pop_success = response->popAppend(&cas_value); break; case CouchbaseOperations::PREPEND: - pop_success = response->PopPrepend(&cas_value); + pop_success = response->popPrepend(&cas_value); break; case CouchbaseOperations::DELETE: - pop_success = response->PopDelete(); + pop_success = response->popDelete(); break; default: DEBUG_PRINT("Unsupported operation type in response pop"); @@ -2153,88 +2263,98 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, result->error_message = "Unsupported operation type in response pop"; return false; } - if(!pop_success){ + if (!pop_success) { result->success = false; result->value = ""; - result->error_message = response->LastError(); + result->error_message = response->lastError(); result->status_code = response->_status_code; - if(result->status_code == 0x88) { - // (0x88) unknown collection, this typically means that the collection_manifest has been updated on the server side. - // and the client have a stale copy of collection manifest. - // In this case, we need to refresh the collection manifest and retry the operation. - if (!request->RefreshCollectionManifest(channel, server, bucket)) { + if (result->status_code == 0x88) { + // (0x88) unknown collection, this typically means that the + // collection_manifest has been updated on the server side. and the + // client have a stale copy of collection manifest. In this case, we + // need to refresh the collection manifest and retry the operation. + if (!request->refreshCollectionManifest(channel, server, bucket)) { DEBUG_PRINT("Failed to refresh collection manifest"); result->error_message = "Failed to refresh collection manifest"; return false; } - // could have called sendRequest recursively, - // but if somehow the collectionID keeps on chaning, it would lead to infinite recursion and stack overflow in the end. - // so we retry once here instead and return failure if it still fails. + // could have called sendRequest recursively, + // but if somehow the collectionID keeps on chaning, it would lead to + // infinite recursion and stack overflow in the end. so we retry once + // here instead and return failure if it still fails. DEBUG_PRINT("Successfully refreshed collection manifest"); - //retry the request; + // retry the request; request->Clear(); response->Clear(); - switch(op_type){ + switch (op_type) { case CouchbaseOperations::UPSERT: - request->UpsertRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request->upsertRequest(key, value, 0, 0, 0, collection_name, + channel, server, bucket); break; case CouchbaseOperations::ADD: - request->AddRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request->addRequest(key, value, 0, 0, 0, collection_name, channel, + server, bucket); break; case CouchbaseOperations::APPEND: - request->AppendRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request->appendRequest(key, value, 0, 0, 0, collection_name, + channel, server, bucket); break; case CouchbaseOperations::PREPEND: - request->PrependRequest(key, value, 0, 0, 0, collection_name, channel, server, bucket); + request->prependRequest(key, value, 0, 0, 0, collection_name, + channel, server, bucket); break; case CouchbaseOperations::DELETE: - request->DeleteRequest(key, collection_name, channel, server, bucket); + request->deleteRequest(key, collection_name, channel, server, + bucket); break; default: DEBUG_PRINT("Unsupported operation type in response pop"); result->success = false; result->value = ""; - result->error_message = "Unsupported operation type in response pop"; + result->error_message = + "Unsupported operation type in response pop"; return false; } channel->CallMethod(NULL, &cntl, request, response, NULL); if (cntl.Failed()) { - DEBUG_PRINT("Failed to perform operation on key: " << key << " to Couchbase: " << cntl.ErrorText()); + DEBUG_PRINT("Failed to perform operation on key: " + << key << " to Couchbase: " << cntl.ErrorText()); result->success = false; result->value = ""; result->error_message = cntl.ErrorText(); - return false; // return on failure + return false; // return on failure } pop_success = false; - switch(op_type){ + switch (op_type) { case CouchbaseOperations::UPSERT: - pop_success = response->PopUpsert(&cas_value); + pop_success = response->popUpsert(&cas_value); break; case CouchbaseOperations::ADD: - pop_success = response->PopAdd(&cas_value); + pop_success = response->popAdd(&cas_value); break; case CouchbaseOperations::APPEND: - pop_success = response->PopAppend(&cas_value); + pop_success = response->popAppend(&cas_value); break; case CouchbaseOperations::PREPEND: - pop_success = response->PopPrepend(&cas_value); + pop_success = response->popPrepend(&cas_value); break; case CouchbaseOperations::DELETE: - pop_success = response->PopDelete(); + pop_success = response->popDelete(); break; default: DEBUG_PRINT("Unsupported operation type in response pop"); result->success = false; result->value = ""; - result->error_message = "Unsupported operation type in response pop"; + result->error_message = + "Unsupported operation type in response pop"; return false; } - if(!pop_success){ + if (!pop_success) { result->success = false; result->value = ""; - result->error_message = response->LastError(); + result->error_message = response->lastError(); result->status_code = response->_status_code; - return false; // return on failure + return false; // return on failure } // Successfully performed the operation after retry result->success = true; @@ -2253,84 +2373,87 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, return true; } } -CouchbaseOperations::Result CouchbaseOperations::Get(const string& key, +CouchbaseOperations::Result CouchbaseOperations::get(const string& key, string collection_name) { // create CouchbaseRequest and CouchbaseResponse objects and then using the // channel which is created for this thread in authenticate() use it to call() - CouchbaseRequest request(&local_bucket_to_collection_manifest); + CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - sendRequest(CouchbaseOperations::GET, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response); + sendRequest(CouchbaseOperations::GET, key, "", collection_name, &result, + channel_, server_address_, selected_bucket_, &request, &response); return result; } -bool CouchbaseOperations::CouchbaseRequest::getLocalCachedCollectionId(const string& bucket, const string& scope, - const string& collection, - uint8_t* collection_id) { +bool CouchbaseOperations::CouchbaseRequest::getLocalCachedCollectionId( + const string& bucket, const string& scope, const string& collection, + uint8_t* collection_id) { if (bucket.empty() || scope.empty() || collection.empty()) { DEBUG_PRINT("Bucket, scope, and collection names must be non-empty"); return false; } auto it = local_collection_manifest_cache->find(bucket); - if(it != local_collection_manifest_cache->end()) { + if (it != local_collection_manifest_cache->end()) { CouchbaseMetadataTracking::CollectionManifest& manifest = it->second; - if(manifest.scope_to_collectionID_map.find(scope)!= manifest.scope_to_collectionID_map.end()){ - auto& collection_map = manifest.scope_to_collectionID_map[scope]; - if(collection_map.find(collection) != collection_map.end()){ + if (manifest.scope_to_collection_id_map.find(scope) != + manifest.scope_to_collection_id_map.end()) { + auto& collection_map = manifest.scope_to_collection_id_map[scope]; + if (collection_map.find(collection) != collection_map.end()) { *collection_id = collection_map[collection]; return true; - } - else{ + } else { DEBUG_PRINT("Collection name not found in local cache: " << collection); return false; } - } - else{ + } else { DEBUG_PRINT("Scope name not found in local cache: " << scope); return false; } - } - else{ + } else { DEBUG_PRINT("Bucket name not found in local cache: " << bucket); return false; } } -CouchbaseOperations::Result CouchbaseOperations::Upsert( +CouchbaseOperations::Result CouchbaseOperations::upsert( const string& key, const string& value, string collection_name) { - CouchbaseRequest request(&local_bucket_to_collection_manifest); + CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - sendRequest(CouchbaseOperations::UPSERT, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); + sendRequest(CouchbaseOperations::UPSERT, key, value, collection_name, &result, + channel_, server_address_, selected_bucket_, &request, &response); return result; } -CouchbaseOperations::Result CouchbaseOperations::Delete( +CouchbaseOperations::Result CouchbaseOperations::delete_( const string& key, string collection_name) { - CouchbaseRequest request(&local_bucket_to_collection_manifest); + CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if(!sendRequest(CouchbaseOperations::DELETE, key, "", collection_name, &result, channel, server_address, selected_bucket, &request, &response)){ + if (!sendRequest(CouchbaseOperations::DELETE, key, "", collection_name, + &result, channel_, server_address_, selected_bucket_, + &request, &response)) { return result; } return result; } -CouchbaseOperations::Result CouchbaseOperations::Add(const string& key, +CouchbaseOperations::Result CouchbaseOperations::add(const string& key, const string& value, string collection_name) { - CouchbaseRequest request(&local_bucket_to_collection_manifest); + CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - sendRequest(CouchbaseOperations::ADD, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); + sendRequest(CouchbaseOperations::ADD, key, value, collection_name, &result, + channel_, server_address_, selected_bucket_, &request, &response); return result; } -CouchbaseOperations::Result CouchbaseOperations::Authenticate( +CouchbaseOperations::Result CouchbaseOperations::authenticate( const string& username, const string& password, const string& server_address, bool enable_ssl, string path_to_cert) { // Create a channel to the Couchbase server @@ -2363,7 +2486,7 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate( CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; - if (request.AuthenticateRequest(username.c_str(), password.c_str()) == + if (request.authenticateRequest(username.c_str(), password.c_str()) == false) { DEBUG_PRINT("Failed to create Authenticate request for user: " << username); delete new_channel; @@ -2380,67 +2503,76 @@ CouchbaseOperations::Result CouchbaseOperations::Authenticate( return result; } // Successfully authenticated - channel = new_channel; - this->server_address = server_address; + channel_ = new_channel; + this->server_address_ = server_address; result.success = true; result.status_code = 0; return result; } -CouchbaseOperations::Result CouchbaseOperations::SelectBucket( +CouchbaseOperations::Result CouchbaseOperations::selectBucket( const string& bucket_name) { - CouchbaseRequest request(&local_bucket_to_collection_manifest); + CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if (request.SelectBucketRequest(bucket_name.c_str()) == false) { - DEBUG_PRINT("Failed to create Select Bucket request for bucket: " << bucket_name); + if (request.selectBucketRequest(bucket_name.c_str()) == false) { + DEBUG_PRINT( + "Failed to create Select Bucket request for bucket: " << bucket_name); result.success = false; result.value = ""; return result; } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); + channel_->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { - DEBUG_PRINT("Failed to select bucket: " << bucket_name << " from Couchbase: " << cntl.ErrorText()); + DEBUG_PRINT("Failed to select bucket: " + << bucket_name << " from Couchbase: " << cntl.ErrorText()); result.success = false; result.value = ""; result.error_message = cntl.ErrorText(); return result; } - if (response.PopSelectBucket(NULL, bucket_name) == false) { + if (response.popSelectBucket(NULL, bucket_name) == false) { result.success = false; result.value = ""; - result.error_message = response.LastError(); + result.error_message = response.lastError(); result.status_code = response._status_code; return result; } // Successfully selected the bucket - selected_bucket = bucket_name; + selected_bucket_ = bucket_name; result.success = true; result.value = ""; result.status_code = 0; // fetch the collection manifest for this bucket and store it in local cache - if(request.local_collection_manifest_cache->find(bucket_name) == request.local_collection_manifest_cache->end()){ + if (request.local_collection_manifest_cache->find(bucket_name) == + request.local_collection_manifest_cache->end()) { // only fetch if not already present in the local cache CouchbaseMetadataTracking::CollectionManifest manifest; - if(!common_metadata_tracking.get_bucket_to_collection_manifest(server_address, bucket_name, &manifest)){ - DEBUG_PRINT("Collection manifest for bucket: " << bucket_name << " not found in global cache, the local cache"); - - // manifest for this bucket/server is not cached yet, will fetch it from server now. - // refresh will also update the local cache with the fetched manifest - request.RefreshCollectionManifest(channel, server_address, bucket_name); - // We simply try once to prefetch the manifest, before any collection operation. - // If it fails, it will be lazily updated when a collection operation is performed. - } - else{ - // update the local cache with the cache manifest from global cache(common_metadata_tracking) - DEBUG_PRINT("Updated local cache collection manifest for bucket: " << bucket_name); + if (!common_metadata_tracking.getBucketToCollectionManifest( + server_address_, bucket_name, &manifest)) { + DEBUG_PRINT("Collection manifest for bucket: " + << bucket_name + << " not found in global cache, the local cache"); + + // manifest for this bucket/server is not cached yet, will fetch it from + // server now. refresh will also update the local cache with the fetched + // manifest + request.refreshCollectionManifest(channel_, server_address_, bucket_name); + // We simply try once to prefetch the manifest, before any collection + // operation. If it fails, it will be lazily updated when a collection + // operation is performed. + } else { + // update the local cache with the cache manifest from global + // cache(common_metadata_tracking) + DEBUG_PRINT("Updated local cache collection manifest for bucket: " + << bucket_name); (*request.local_collection_manifest_cache)[bucket_name] = manifest; } - } - else{ - DEBUG_PRINT("Collection manifest for bucket: " << bucket_name << " already present in local cache"); + } else { + DEBUG_PRINT("Collection manifest for bucket: " + << bucket_name << " already present in local cache"); } DEBUG_PRINT("Bucket selected successfully " << bucket_name); return result; @@ -2471,7 +2603,7 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket( // if(response.PopReplace(&cas_value) == false){ // result.success = false; // result.value = ""; -// result.error_message = response.LastError(); +// result.error_message = response.lastError(); // return result; // } // // Successfully replaced the value @@ -2480,23 +2612,26 @@ CouchbaseOperations::Result CouchbaseOperations::SelectBucket( // return result; // } -CouchbaseOperations::Result CouchbaseOperations::Append( +CouchbaseOperations::Result CouchbaseOperations::append( const string& key, const string& value, string collection_name) { - CouchbaseRequest request(&local_bucket_to_collection_manifest); + CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - sendRequest(CouchbaseOperations::APPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); + sendRequest(CouchbaseOperations::APPEND, key, value, collection_name, &result, + channel_, server_address_, selected_bucket_, &request, &response); return result; } -CouchbaseOperations::Result CouchbaseOperations::Prepend( +CouchbaseOperations::Result CouchbaseOperations::prepend( const string& key, const string& value, string collection_name) { - CouchbaseRequest request(&local_bucket_to_collection_manifest); + CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - sendRequest(CouchbaseOperations::PREPEND, key, value, collection_name, &result, channel, server_address, selected_bucket, &request, &response); + sendRequest(CouchbaseOperations::PREPEND, key, value, collection_name, + &result, channel_, server_address_, selected_bucket_, &request, + &response); return result; } @@ -2526,7 +2661,7 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend( // if(response.PopIncrement(&new_value, &cas_value) == false){ // result.success = false; // result.value = ""; -// result.error_message = response.LastError(); +// result.error_message = response.lastError(); // return result; // } // // Successfully incremented the value @@ -2561,7 +2696,7 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend( // if(response.PopDecrement(&new_value, &cas_value) == false){ // result.success = false; // result.value = ""; -// result.error_message = response.LastError(); +// result.error_message = response.lastError(); // return result; // } // // Successfully decremented the value @@ -2594,7 +2729,7 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend( // if(response.PopTouch() == false){ // result.success = false; // result.value = ""; -// result.error_message = response.LastError(); +// result.error_message = response.lastError(); // return result; // } // // Successfully touched the key @@ -2626,7 +2761,7 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend( // if(response.PopFlush() == false){ // result.success = false; // result.value = ""; -// result.error_message = response.LastError(); +// result.error_message = response.lastError(); // return result; // } // // Successfully flushed @@ -2635,18 +2770,18 @@ CouchbaseOperations::Result CouchbaseOperations::Prepend( // return result; // } -CouchbaseOperations::Result CouchbaseOperations::Version() { +CouchbaseOperations::Result CouchbaseOperations::version() { CouchbaseRequest request; CouchbaseResponse response; brpc::Controller cntl; CouchbaseOperations::Result result; - if (request.VersionRequest() == false) { + if (request.versionRequest() == false) { DEBUG_PRINT("Failed to create Version request"); result.success = false; result.value = ""; return result; } - channel->CallMethod(NULL, &cntl, &request, &response, NULL); + channel_->CallMethod(NULL, &cntl, &request, &response, NULL); if (cntl.Failed()) { DEBUG_PRINT("Failed to get version from Couchbase: " << cntl.ErrorText()); result.success = false; @@ -2655,10 +2790,10 @@ CouchbaseOperations::Result CouchbaseOperations::Version() { return result; } string version; - if (response.PopVersion(&version) == false) { + if (response.popVersion(&version) == false) { result.success = false; result.value = ""; - result.error_message = response.LastError(); + result.error_message = response.lastError(); result.status_code = response._status_code; return result; } @@ -2669,9 +2804,9 @@ CouchbaseOperations::Result CouchbaseOperations::Version() { return result; } -bool CouchbaseOperations::BeginPipeline() { +bool CouchbaseOperations::beginPipeline() { if (pipeline_active) { - DEBUG_PRINT("Pipeline already active. Call ClearPipeline() first."); + DEBUG_PRINT("Pipeline already active. Call clearPipeline() first."); return false; } @@ -2685,11 +2820,10 @@ bool CouchbaseOperations::BeginPipeline() { return true; } -bool CouchbaseOperations::PipelineRequest(operation_type op_type, +bool CouchbaseOperations::pipelineRequest(operation_type op_type, const string& key, const string& value, string collection_name) { - if (!pipeline_active) { DEBUG_PRINT("Pipeline not active. Call BeginPipeline() first."); return false; @@ -2697,49 +2831,49 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, switch (op_type) { case GET: - if (pipeline_request_couchbase_req.GetRequest(key, collection_name, channel, - server_address, - selected_bucket) == false) { + if (pipeline_request_couchbase_req.getRequest( + key, collection_name, channel_, server_address_, + selected_bucket_) == false) { return false; } pipeline_operations_queue.push(GET); break; case UPSERT: - if (pipeline_request_couchbase_req.UpsertRequest(key, value, 0, 0, 0, collection_name, - channel, server_address, - selected_bucket) == false) { + if (pipeline_request_couchbase_req.upsertRequest( + key, value, 0, 0, 0, collection_name, channel_, server_address_, + selected_bucket_) == false) { return false; } pipeline_operations_queue.push(UPSERT); break; case ADD: - if (pipeline_request_couchbase_req.AddRequest(key, value, 0, 0, 0, collection_name, - channel, server_address, - selected_bucket) == false) { + if (pipeline_request_couchbase_req.addRequest( + key, value, 0, 0, 0, collection_name, channel_, server_address_, + selected_bucket_) == false) { return false; } pipeline_operations_queue.push(ADD); break; case APPEND: - if (pipeline_request_couchbase_req.AppendRequest(key, value, 0, 0, 0, collection_name, - channel, server_address, - selected_bucket) == false) { + if (pipeline_request_couchbase_req.appendRequest( + key, value, 0, 0, 0, collection_name, channel_, server_address_, + selected_bucket_) == false) { return false; } pipeline_operations_queue.push(APPEND); break; case PREPEND: - if (pipeline_request_couchbase_req.PrependRequest(key, value, 0, 0, 0, collection_name, - channel, server_address, - selected_bucket) == false) { + if (pipeline_request_couchbase_req.prependRequest( + key, value, 0, 0, 0, collection_name, channel_, server_address_, + selected_bucket_) == false) { return false; } pipeline_operations_queue.push(PREPEND); break; case DELETE: - if (pipeline_request_couchbase_req.DeleteRequest(key, collection_name, channel, - server_address, - selected_bucket) == false) { + if (pipeline_request_couchbase_req.deleteRequest( + key, collection_name, channel_, server_address_, + selected_bucket_) == false) { return false; } pipeline_operations_queue.push(DELETE); @@ -2750,7 +2884,7 @@ bool CouchbaseOperations::PipelineRequest(operation_type op_type, } return true; } -vector CouchbaseOperations::ExecutePipeline() { +vector CouchbaseOperations::executePipeline() { vector results; if (!pipeline_active || pipeline_operations_queue.empty()) { @@ -2759,7 +2893,8 @@ vector CouchbaseOperations::ExecutePipeline() { } brpc::Controller cntl; - channel->CallMethod(NULL, &cntl, &pipeline_request_couchbase_req, &pipeline_response_couchbase_resp, NULL); + channel_->CallMethod(NULL, &cntl, &pipeline_request_couchbase_req, + &pipeline_response_couchbase_resp, NULL); if (cntl.Failed()) { DEBUG_PRINT("Pipeline execution failed: " << cntl.ErrorText()); @@ -2775,12 +2910,13 @@ vector CouchbaseOperations::ExecutePipeline() { results.push_back(failure_result); } - ClearPipeline(); + clearPipeline(); return results; } // Process each operation in the order they were added - CouchbaseOperations::CouchbaseResponse* response = &pipeline_response_couchbase_resp; + CouchbaseOperations::CouchbaseResponse* response = + &pipeline_response_couchbase_resp; while (!pipeline_operations_queue.empty()) { CouchbaseOperations::Result result; operation_type op_type = pipeline_operations_queue.front(); @@ -2790,10 +2926,10 @@ vector CouchbaseOperations::ExecutePipeline() { string value; uint32_t flags = 0; uint64_t cas = 0; - if (response->PopGet(&value, &flags, &cas) == false) { + if (response->popGet(&value, &flags, &cas) == false) { result.success = false; result.value = ""; - result.error_message = response->LastError(); + result.error_message = response->lastError(); result.status_code = response->_status_code; } else { result.success = true; @@ -2803,10 +2939,10 @@ vector CouchbaseOperations::ExecutePipeline() { break; } case UPSERT: { - if (response->PopUpsert(NULL) == false) { + if (response->popUpsert(NULL) == false) { result.success = false; result.value = ""; - result.error_message = response->LastError(); + result.error_message = response->lastError(); result.status_code = response->_status_code; } else { result.success = true; @@ -2816,10 +2952,10 @@ vector CouchbaseOperations::ExecutePipeline() { break; } case ADD: { - if (response->PopAdd(NULL) == false) { + if (response->popAdd(NULL) == false) { result.success = false; result.value = ""; - result.error_message = response->LastError(); + result.error_message = response->lastError(); result.status_code = response->_status_code; } else { result.success = true; @@ -2830,10 +2966,10 @@ vector CouchbaseOperations::ExecutePipeline() { } case APPEND: { uint64_t cas_value; - if (response->PopAppend(&cas_value) == false) { + if (response->popAppend(&cas_value) == false) { result.success = false; result.value = ""; - result.error_message = response->LastError(); + result.error_message = response->lastError(); result.status_code = response->_status_code; } else { result.success = true; @@ -2844,10 +2980,10 @@ vector CouchbaseOperations::ExecutePipeline() { } case PREPEND: { uint64_t cas_value; - if (response->PopPrepend(&cas_value) == false) { + if (response->popPrepend(&cas_value) == false) { result.success = false; result.value = ""; - result.error_message = response->LastError(); + result.error_message = response->lastError(); result.status_code = response->_status_code; } else { result.success = true; @@ -2857,10 +2993,10 @@ vector CouchbaseOperations::ExecutePipeline() { break; } case DELETE: { - if (response->PopDelete() == false) { + if (response->popDelete() == false) { result.success = false; result.value = ""; - result.error_message = response->LastError(); + result.error_message = response->lastError(); result.status_code = response->_status_code; } else { result.success = true; @@ -2885,7 +3021,7 @@ vector CouchbaseOperations::ExecutePipeline() { return results; } -bool CouchbaseOperations::ClearPipeline() { +bool CouchbaseOperations::clearPipeline() { while (!pipeline_operations_queue.empty()) { pipeline_operations_queue.pop(); } diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index aa33df92ab..040017bed5 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -121,28 +121,28 @@ class CouchbaseMetadataTracking { string uid; // uid of the manifest, it can be used to track if the manifest // is updated unordered_map> - scope_to_collectionID_map; // scope -> (collection -> collection_id) + scope_to_collection_id_map; // scope -> (collection -> collection_id) }; private: unordered_map> - bucket_to_collection_manifest; - ReaderWriterLock rw_bucket_to_collection_manifest_mutex; + bucket_to_collection_manifest_; + ReaderWriterLock rw_bucket_to_collection_manifest_mutex_; public: CouchbaseMetadataTracking() {} - ~CouchbaseMetadataTracking() { bucket_to_collection_manifest.clear(); } - bool set_bucket_to_collection_manifest(string server, string bucket, - CollectionManifest manifest); + ~CouchbaseMetadataTracking() { bucket_to_collection_manifest_.clear(); } + bool setBucketToCollectionManifest(string server, string bucket, + CollectionManifest manifest); - bool get_bucket_to_collection_manifest(string server, string bucket, - CollectionManifest* manifest); - bool get_manifest_to_collection_id(CollectionManifest* manifest, string scope, - string collection, uint8_t* collection_id); + bool getBucketToCollectionManifest(string server, string bucket, + CollectionManifest* manifest); + bool getManifestToCollectionId(CollectionManifest* manifest, string scope, + string collection, uint8_t* collection_id); - bool json_to_collection_manifest(const string& json, - CollectionManifest* manifest); + bool jsonToCollectionManifest(const string& json, + CollectionManifest* manifest); } static common_metadata_tracking; class CouchbaseOperations { public: @@ -159,21 +159,21 @@ class CouchbaseOperations { bool success; string error_message; string value; - uint16_t status_code; // 0x00 if success + uint16_t status_code; // 0x00 if success }; - Result Get(const string& key, string collection_name = "_default"); - Result Upsert(const string& key, const string& value, + Result get(const string& key, string collection_name = "_default"); + Result upsert(const string& key, const string& value, string collection_name = "_default"); - Result Add(const string& key, const string& value, + Result add(const string& key, const string& value, string collection_name = "_default"); // Warning: Not tested - // Result Replace(const string& key, const string& value, string + // Result replace(const string& key, const string& value, string // collection_name = "_default"); - Result Append(const string& key, const string& value, + Result append(const string& key, const string& value, string collection_name = "_default"); - Result Prepend(const string& key, const string& value, + Result prepend(const string& key, const string& value, string collection_name = "_default"); - Result Delete(const string& key, string collection_name = "_default"); + Result delete_(const string& key, string collection_name = "_default"); // Warning: Not tested // Result Increment(const string& key, uint64_t delta, uint64_t initial_value, // uint32_t exptime, string collection_name = "_default"); Result @@ -181,39 +181,43 @@ class CouchbaseOperations { // uint32_t exptime, string collection_name = "_default"); Result Touch(const // string& key, uint32_t exptime, string collection_name = "_default"); Result // Flush(uint32_t timeout = 0); - Result Version(); - Result Authenticate(const string& username, const string& password, - const string& server_address, bool enable_ssl=false, - string path_to_cert=""); - Result SelectBucket(const string& bucket_name); + Result version(); + Result authenticate(const string& username, const string& password, + const string& server_address, bool enable_ssl = false, + string path_to_cert = ""); + Result selectBucket(const string& bucket_name); // Pipeline management - bool BeginPipeline(); - bool PipelineRequest(operation_type op_type, const string& key, + bool beginPipeline(); + bool pipelineRequest(operation_type op_type, const string& key, const string& value = "", string collection_name = "_default"); - vector ExecutePipeline(); // Return by value instead of pointer - bool ClearPipeline(); + vector executePipeline(); // Return by value instead of pointer + bool clearPipeline(); // Pipeline status - bool IsPipelineActive() const { return pipeline_active; } - size_t GetPipelineSize() const { return pipeline_operations_queue.size(); } + bool isPipelineActive() const { return pipeline_active; } + size_t getPipelineSize() const { return pipeline_operations_queue.size(); } - CouchbaseOperations() : pipeline_request_couchbase_req(&local_bucket_to_collection_manifest), pipeline_active(false) {} + CouchbaseOperations() + : pipeline_request_couchbase_req(&local_bucket_to_collection_manifest_), + pipeline_active(false) {} ~CouchbaseOperations() {} - bool get_local_cached_collection_id(const string& bucket, const string& scope, - const string& collection, uint8_t* coll_id); + bool getLocalCachedCollectionId(const string& bucket, const string& scope, + const string& collection, uint8_t* coll_id); private: friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); friend void policy::SerializeCouchbaseRequest( butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); - brpc::Channel* channel; - string server_address; - string selected_bucket; + brpc::Channel* channel_; + string server_address_; + string selected_bucket_; - unordered_map local_bucket_to_collection_manifest; + unordered_map + local_bucket_to_collection_manifest_; class CouchbaseRequest : public NonreflectableMessage { private: @@ -221,36 +225,41 @@ class CouchbaseOperations { int _pipelined_count; butil::IOBuf _buf; mutable int _cached_size_; - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; - bool GetOrDelete(uint8_t command, const butil::StringPiece& key, + void sharedCtor(); + void sharedDtor(); + void setCachedSize(int size) const PB_425_OVERRIDE; + bool getOrDelete(uint8_t command, const butil::StringPiece& key, uint8_t coll_id = 0); - bool Counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, + bool counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, uint64_t initial_value, uint32_t exptime); - bool Store(uint8_t command, const butil::StringPiece& key, + bool store(uint8_t command, const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, uint8_t coll_id = 0); - uint32_t hash_crc32(const char* key, size_t key_length); + uint32_t hashCrc32(const char* key, size_t key_length); public: - - unordered_map *local_collection_manifest_cache; - - CouchbaseRequest(unordered_map *local_cache_reference) : NonreflectableMessage() { + unordered_map* + local_collection_manifest_cache; + + CouchbaseRequest( + unordered_map* + local_cache_reference) + : NonreflectableMessage() { metadata_tracking = &common_metadata_tracking; local_collection_manifest_cache = local_cache_reference; - SharedCtor(); + sharedCtor(); } CouchbaseRequest() : NonreflectableMessage() { metadata_tracking = &common_metadata_tracking; - SharedCtor(); + sharedCtor(); } - ~CouchbaseRequest() { SharedDtor(); } + ~CouchbaseRequest() { sharedDtor(); } CouchbaseRequest(const CouchbaseRequest& from) : NonreflectableMessage(from) { - SharedCtor(); + sharedCtor(); MergeFrom(from); } @@ -259,46 +268,45 @@ class CouchbaseOperations { return *this; } - bool SelectBucketRequest(const butil::StringPiece& bucket_name); - bool AuthenticateRequest(const butil::StringPiece& username, + bool selectBucketRequest(const butil::StringPiece& bucket_name); + bool authenticateRequest(const butil::StringPiece& username, const butil::StringPiece& password); - bool HelloRequest(); + bool helloRequest(); // Using GetCollectionManifest instead of fetching collection ID directly // bool GetCollectionId(const butil::StringPiece& scope_name, // const butil::StringPiece& collection_name); - bool GetScopeId(const butil::StringPiece& scope_name); + bool getScopeId(const butil::StringPiece& scope_name); - bool GetCollectionManifest(); + bool getCollectionManifest(); bool getLocalCachedCollectionId(const string& bucket, const string& scope, - const string& collection, uint8_t* coll_id); - - bool get_cached_or_fetch_collection_id( - string collection_name, uint8_t* coll_id, - brpc::CouchbaseMetadataTracking* metadata_tracking, - brpc::Channel* channel, const string& server, - const string& selected_bucket); - - bool RefreshCollectionManifest(brpc::Channel* channel, - const string& server, - const string& bucket); + const string& collection, uint8_t* coll_id); + + bool getCachedOrFetchCollectionId( + string collection_name, uint8_t* coll_id, + brpc::CouchbaseMetadataTracking* metadata_tracking, + brpc::Channel* channel, const string& server, + const string& selected_bucket); + + bool refreshCollectionManifest(brpc::Channel* channel, const string& server, + const string& bucket); // Collection-aware document operations - bool GetRequest(const butil::StringPiece& key, + bool getRequest(const butil::StringPiece& key, string collection_name = "_default", brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - bool UpsertRequest(const butil::StringPiece& key, + bool upsertRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name = "_default", brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - bool AddRequest(const butil::StringPiece& key, + bool addRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name = "_default", @@ -313,21 +321,21 @@ class CouchbaseOperations { // brpc::Channel* channel = nullptr, const string& server = "", // const string& bucket = ""); - bool AppendRequest(const butil::StringPiece& key, + bool appendRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name = "_default", brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - bool PrependRequest(const butil::StringPiece& key, + bool prependRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, string collection_name = "_default", brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); - bool DeleteRequest(const butil::StringPiece& key, + bool deleteRequest(const butil::StringPiece& key, string collection_name = "_default", brpc::Channel* channel = nullptr, const string& server = "", const string& bucket = ""); @@ -351,12 +359,12 @@ class CouchbaseOperations { // brpc::Channel* channel = nullptr, const string& server = "", // const string& bucket = ""); - bool VersionRequest(); + bool versionRequest(); - int pipelined_count() const { return _pipelined_count; } + int pipelinedCount() const { return _pipelined_count; } - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } + butil::IOBuf& rawBuffer() { return _buf; } + const butil::IOBuf& rawBuffer() const { return _buf; } void Swap(CouchbaseRequest* other); void MergeFrom(const CouchbaseRequest& from) override; void Clear() override; @@ -377,24 +385,24 @@ class CouchbaseOperations { static brpc::CouchbaseMetadataTracking* metadata_tracking; butil::IOBuf _buf; mutable int _cached_size_; - bool PopCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); - bool PopStore(uint8_t command, uint64_t* cas_value); + bool popCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); + bool popStore(uint8_t command, uint64_t* cas_value); - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PB_425_OVERRIDE; + void sharedCtor(); + void sharedDtor(); + void setCachedSize(int size) const PB_425_OVERRIDE; public: uint16_t _status_code; CouchbaseResponse() : NonreflectableMessage() { - SharedCtor(); + sharedCtor(); } - ~CouchbaseResponse() { SharedDtor(); } + ~CouchbaseResponse() { sharedDtor(); } CouchbaseResponse(const CouchbaseResponse& from) : NonreflectableMessage(from) { metadata_tracking = &common_metadata_tracking; - SharedCtor(); + sharedCtor(); MergeFrom(from); } inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { @@ -463,7 +471,7 @@ class CouchbaseOperations { STATUS_SUBDOC_DELETED_DOCUMENT_CANT_HAVE_VALUE = 0xd7, STATUS_XATTR_EINVAL = 0xe0 }; - const char* couchbase_binary_command_to_string(uint8_t cmd); + const char* couchbaseBinaryCommandToString(uint8_t cmd); void MergeFrom(const CouchbaseResponse& from) override; void Clear() override; bool IsInitialized() const PB_527_OVERRIDE; @@ -477,48 +485,49 @@ class CouchbaseOperations { ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - butil::IOBuf& raw_buffer() { return _buf; } - const butil::IOBuf& raw_buffer() const { return _buf; } - static const char* status_str(Status); + butil::IOBuf& rawBuffer() { return _buf; } + const butil::IOBuf& rawBuffer() const { return _buf; } + static const char* statusStr(Status); // Helper method to format error messages with status codes - static string format_error_message(uint16_t status_code, - const string& operation, - const string& error_msg = ""); + static string formatErrorMessage(uint16_t status_code, + const string& operation, + const string& error_msg = ""); // Add methods to handle response parsing - void Swap(CouchbaseResponse* other); - bool PopGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool PopGet(string* value, uint32_t* flags, uint64_t* cas_value); - const string& LastError() const { return _err; } - bool PopUpsert(uint64_t* cas_value); - bool PopAdd(uint64_t* cas_value); + void swap(CouchbaseResponse* other); + bool popGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); + bool popGet(string* value, uint32_t* flags, uint64_t* cas_value); + const string& lastError() const { return _err; } + bool popUpsert(uint64_t* cas_value); + bool popAdd(uint64_t* cas_value); // Warning: Not tested - // bool PopReplace(uint64_t* cas_value); - bool PopAppend(uint64_t* cas_value); - bool PopPrepend(uint64_t* cas_value); - bool PopSelectBucket(uint64_t* cas_value, std::string bucket_name); + // bool popReplace(uint64_t* cas_value); + bool popAppend(uint64_t* cas_value); + bool popPrepend(uint64_t* cas_value); + bool popSelectBucket(uint64_t* cas_value, std::string bucket_name); // Collection-related response methods - bool PopCollectionId(uint8_t* collection_id); + bool popCollectionId(uint8_t* collection_id); - bool PopManifest(std::string* manifest_json); + bool popManifest(std::string* manifest_json); - bool PopDelete(); + bool popDelete(); // Warning: Not tested - // bool PopFlush(); - // bool PopIncrement(uint64_t* new_value, uint64_t* cas_value); - // bool PopDecrement(uint64_t* new_value, uint64_t* cas_value); - // bool PopTouch(); - bool PopVersion(string* version); + // bool popFlush(); + // bool popIncrement(uint64_t* new_value, uint64_t* cas_value); + // bool popDecrement(uint64_t* new_value, uint64_t* cas_value); + // bool popTouch(); + bool popVersion(string* version); }; - friend bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, - const string& value, string collection_name, - CouchbaseOperations::Result *result, brpc::Channel *channel, - const string& server, const string& bucket, - CouchbaseRequest *request, - CouchbaseResponse *response); + friend bool sendRequest(CouchbaseOperations::operation_type op_type, + const string& key, const string& value, + string collection_name, + CouchbaseOperations::Result* result, + brpc::Channel* channel, const string& server, + const string& bucket, CouchbaseRequest* request, + CouchbaseResponse* response); // Pipeline management - per instance queue pipeline_operations_queue; diff --git a/src/brpc/policy/couchbase_protocol.cpp b/src/brpc/policy/couchbase_protocol.cpp index 20f7ff93bf..6ccb1d888a 100644 --- a/src/brpc/policy/couchbase_protocol.cpp +++ b/src/brpc/policy/couchbase_protocol.cpp @@ -187,7 +187,8 @@ void ProcessCouchbaseResponse(InputMessageBase* msg_base) { cntl->SetFailed(ERESPONSE, "Must be CouchbaseResponse"); } else { // We work around ParseFrom of pb which is just a placeholder. - ((CouchbaseOperations::CouchbaseResponse*)cntl->response())->raw_buffer() = msg->meta.movable(); + ((CouchbaseOperations::CouchbaseResponse*)cntl->response())->rawBuffer() = + msg->meta.movable(); if (msg->pi.count != accessor.pipelined_count()) { cntl->SetFailed(ERESPONSE, "pipelined_count=%d of response does " @@ -206,13 +207,15 @@ void SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, if (request == NULL) { return cntl->SetFailed(EREQUEST, "request is NULL"); } - if (request->GetDescriptor() != CouchbaseOperations::CouchbaseRequest::descriptor()) { + if (request->GetDescriptor() != + CouchbaseOperations::CouchbaseRequest::descriptor()) { return cntl->SetFailed(EREQUEST, "Must be CouchbaseRequest"); } - const CouchbaseOperations::CouchbaseRequest* mr = (const CouchbaseOperations::CouchbaseRequest*)request; + const CouchbaseOperations::CouchbaseRequest* mr = + (const CouchbaseOperations::CouchbaseRequest*)request; // We work around SerializeTo of pb which is just a placeholder. - *buf = mr->raw_buffer(); - ControllerPrivateAccessor(cntl).set_pipelined_count(mr->pipelined_count()); + *buf = mr->rawBuffer(); + ControllerPrivateAccessor(cntl).set_pipelined_count(mr->pipelinedCount()); } void PackCouchbaseRequest(butil::IOBuf* buf, SocketMessage**, From 5a55549a3c04d5cd3ea3506926f142da78c6556a Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 6 Oct 2025 02:13:24 +0530 Subject: [PATCH 30/49] removed unnecessary code --- src/brpc/couchbase.cpp | 105 +---------------------------------------- src/brpc/couchbase.h | 24 ++-------- 2 files changed, 4 insertions(+), 125 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 0ceb988c12..7353365efe 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -498,61 +498,6 @@ bool CouchbaseOperations::CouchbaseRequest::authenticateRequest( return true; } -bool CouchbaseOperations::CouchbaseRequest::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { - DEBUG_PRINT("You're not supposed to parse a CouchbaseRequest"); - - // simple approach just making it work. - butil::IOBuf tmp; - const void* data = NULL; - int size = 0; - while (input->GetDirectBufferPointer(&data, &size)) { - tmp.append(data, size); - input->Skip(size); - } - const butil::IOBuf saved = tmp; - int count = 0; - for (; !tmp.empty(); ++count) { - char aux_buf[sizeof(policy::CouchbaseRequestHeader)]; - const policy::CouchbaseRequestHeader* header = - (const policy::CouchbaseRequestHeader*)tmp.fetch(aux_buf, - sizeof(aux_buf)); - if (header == NULL) { - return false; - } - if (header->magic != (uint8_t)policy::CB_MAGIC_REQUEST) { - return false; - } - uint32_t total_body_length = butil::NetToHost32(header->total_body_length); - if (tmp.size() < sizeof(*header) + total_body_length) { - return false; - } - tmp.pop_front(sizeof(*header) + total_body_length); - } - _buf.append(saved); - _pipelined_count += count; - return true; -} - -void CouchbaseOperations::CouchbaseRequest::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - DEBUG_PRINT("You're not supposed to serialize a CouchbaseRequest"); - - // simple approach just making it work. - butil::IOBufAsZeroCopyInputStream wrapper(_buf); - const void* data = NULL; - int size = 0; - while (wrapper.Next(&data, &size)) { - output->WriteRaw(data, size); - } -} - -size_t CouchbaseOperations::CouchbaseRequest::ByteSizeLong() const { - int total_size = static_cast(_buf.size()); - _cached_size_ = total_size; - return total_size; -} - void CouchbaseOperations::CouchbaseRequest::MergeFrom( const CouchbaseRequest& from) { CHECK_NE(&from, this); @@ -572,14 +517,6 @@ void CouchbaseOperations::CouchbaseRequest::Swap(CouchbaseRequest* other) { } } -::google::protobuf::Metadata -CouchbaseOperations::CouchbaseRequest::GetMetadata() const { - ::google::protobuf::Metadata metadata{}; - metadata.descriptor = CouchbaseRequestBase::descriptor(); - metadata.reflection = nullptr; - return metadata; -} - void CouchbaseOperations::CouchbaseResponse::sharedCtor() { _cached_size_ = 0; } void CouchbaseOperations::CouchbaseResponse::sharedDtor() {} @@ -590,38 +527,6 @@ void CouchbaseOperations::CouchbaseResponse::setCachedSize(int size) const { void CouchbaseOperations::CouchbaseResponse::Clear() {} -bool CouchbaseOperations::CouchbaseResponse::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { - DEBUG_PRINT("You're not supposed to parse a CouchbaseResponse"); - - // simple approach just making it work. - const void* data = NULL; - int size = 0; - while (input->GetDirectBufferPointer(&data, &size)) { - _buf.append(data, size); - input->Skip(size); - } - return true; -} - -void CouchbaseOperations::CouchbaseResponse::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - DEBUG_PRINT("You're not supposed to serialize a CouchbaseResponse"); - - // simple approach just making it work. - butil::IOBufAsZeroCopyInputStream wrapper(_buf); - const void* data = NULL; - int size = 0; - while (wrapper.Next(&data, &size)) { - output->WriteRaw(data, size); - } -} - -size_t CouchbaseOperations::CouchbaseResponse::ByteSizeLong() const { - int total_size = static_cast(_buf.size()); - _cached_size_ = total_size; - return total_size; -} void CouchbaseOperations::CouchbaseResponse::MergeFrom( const CouchbaseResponse& from) { @@ -641,14 +546,6 @@ void CouchbaseOperations::CouchbaseResponse::swap(CouchbaseResponse* other) { } } -::google::protobuf::Metadata -CouchbaseOperations::CouchbaseResponse::GetMetadata() const { - ::google::protobuf::Metadata metadata{}; - metadata.descriptor = CouchbaseResponseBase::descriptor(); - metadata.reflection = nullptr; - return metadata; -} - // =================================================================== const char* CouchbaseOperations::CouchbaseResponse::statusStr(Status st) { @@ -796,7 +693,7 @@ bool CouchbaseOperations::CouchbaseRequest::getOrDelete( policy::CB_MAGIC_REQUEST, command, butil::HostToNet16( key.size() + - 1), // collection id is part of key, so adding it in the key length + 1), // Key 0, // extras length policy::CB_BINARY_RAW_BYTES, // data type butil::HostToNet16(VBucket_id), diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 040017bed5..a1fff500e4 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -227,7 +227,7 @@ class CouchbaseOperations { mutable int _cached_size_; void sharedCtor(); void sharedDtor(); - void setCachedSize(int size) const PB_425_OVERRIDE; + void setCachedSize(int size) const; bool getOrDelete(uint8_t command, const butil::StringPiece& key, uint8_t coll_id = 0); bool counter(uint8_t command, const butil::StringPiece& key, uint64_t delta, @@ -364,19 +364,11 @@ class CouchbaseOperations { int pipelinedCount() const { return _pipelined_count; } butil::IOBuf& rawBuffer() { return _buf; } - const butil::IOBuf& rawBuffer() const { return _buf; } + const butil::IOBuf& rawBuffer() const { return _buf; } // used in couchbase_protocol serialization. void Swap(CouchbaseRequest* other); void MergeFrom(const CouchbaseRequest& from) override; void Clear() override; bool IsInitialized() const PB_527_OVERRIDE; - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream* - output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; }; class CouchbaseResponse : public NonreflectableMessage { @@ -390,7 +382,7 @@ class CouchbaseOperations { void sharedCtor(); void sharedDtor(); - void setCachedSize(int size) const PB_425_OVERRIDE; + void setCachedSize(int size) const; public: uint16_t _status_code; @@ -476,17 +468,7 @@ class CouchbaseOperations { void Clear() override; bool IsInitialized() const PB_527_OVERRIDE; - size_t ByteSizeLong() const override; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PB_310_OVERRIDE; - void SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream* - output) const PB_310_OVERRIDE; - int GetCachedSize() const PB_425_OVERRIDE { return _cached_size_; } - - ::google::protobuf::Metadata GetMetadata() const PB_527_OVERRIDE; - butil::IOBuf& rawBuffer() { return _buf; } - const butil::IOBuf& rawBuffer() const { return _buf; } static const char* statusStr(Status); // Helper method to format error messages with status codes From 0d8b7b2d5b4be5346824ffcd6113f6c78d4af5d8 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 6 Oct 2025 11:27:48 +0530 Subject: [PATCH 31/49] updated comments --- src/brpc/couchbase.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 7353365efe..7ccecc791c 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -1341,8 +1341,6 @@ bool CouchbaseOperations::CouchbaseRequest::refreshCollectionManifest( << temp_get_manifest_response.lastError()); return false; } - // Compare the UID with the cached one - // If they are different, refresh the cache brpc::CouchbaseMetadataTracking::CollectionManifest manifest; if (!common_metadata_tracking.jsonToCollectionManifest(manifest_json, &manifest)) { @@ -1364,7 +1362,10 @@ bool CouchbaseOperations::CouchbaseRequest::refreshCollectionManifest( // also update the local cache (*local_collection_manifest_cache)[bucket] = manifest; return true; - } else if (manifest.uid != cached_manifest.uid) { + } + // Compare the UID with the cached one + // If they are different, refresh the cache + else if (manifest.uid != cached_manifest.uid) { DEBUG_PRINT("Collection manifest has changed for bucket " << bucket << " on server " << server); if (!common_metadata_tracking.setBucketToCollectionManifest(server, bucket, From 4d4d181c663f7dac1146509d83974a97be2e2e71 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 6 Oct 2025 11:38:51 +0530 Subject: [PATCH 32/49] updated comments --- src/brpc/couchbase.cpp | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 7ccecc791c..603bb4528f 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -1376,22 +1376,10 @@ bool CouchbaseOperations::CouchbaseRequest::refreshCollectionManifest( } DEBUG_PRINT("Updated cached collection manifest for bucket " << bucket << " on server " << server); - // also update the local cache if needed. - if (local_collection_manifest_cache->find(bucket) != - local_collection_manifest_cache->end()) { - // if the bucket already exists in the local cache, check the UID - if ((*local_collection_manifest_cache)[bucket].uid != manifest.uid) { - // if the UID is different, update the local cache - (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Updated local collection manifest cache for bucket " - << bucket << " on server " << server); - } - } else { - // if the bucket does not exist in the local cache, add it - (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Added to local collection manifest cache for bucket " - << bucket << " on server " << server); - } + // update the local cache as well + (*local_collection_manifest_cache)[bucket] = manifest; + DEBUG_PRINT("Added to local collection manifest cache for bucket " + << bucket << " on server " << server); return true; } else { DEBUG_PRINT("Collection manifest is already up-to-date for bucket " From 80da22c4bf2ccff5cfd3a356961d593fba03923d Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 6 Oct 2025 12:05:51 +0530 Subject: [PATCH 33/49] updated documentation --- docs/en/couchbase_example.md | 119 ++++++++++++++++++----------------- src/brpc/couchbase.cpp | 2 +- 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index f4e6e1d68d..8b36f609ce 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -30,9 +30,9 @@ Design goals: | **High-Level API** | `CouchbaseOperations` class | **Recommended**: Simple methods returning `Result` structs | | **SSL/TLS Support** | Built-in SSL encryption | **Required** for Couchbase Capella, optional for local clusters | | Authentication | SASL `PLAIN` with SSL | Each `CouchbaseOperations` instance requires authentication | -| Bucket selection | `SelectBucket()` method | Required before document operations | -| Basic KV | `Add()`, `Upsert()`, `Delete()`, `Get()` | Clean API with `Result` struct error handling | -| **Pipeline Operations** | `BeginPipeline()`, `PipelineRequest()`, `ExecutePipeline()` | **NEW**: Batch multiple operations in single network call for improved performance | +| Bucket selection | `selectBucket()` method | Required before document operations | +| Basic KV | `add()`, `upsert()`, `delete_()`, `get()` | Clean API with `Result` struct error handling | +| **Pipeline Operations** | `beginPipeline()`, `pipelineRequest()`, `executePipeline()` | **NEW**: Batch multiple operations in single network call for improved performance | | Collections | Collection-scoped CRUD operations | Pass collection name as optional parameter (defaults to "_default") | | Error Handling | `Result.success` + `Result.error_message` | Human-readable error messages with status codes | @@ -102,7 +102,7 @@ Overall packet structure:- brpc::CouchbaseOperations couchbase_ops; // 1. Authenticate (REQUIRED for each instance) -brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( +brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticate( username, password, server_address, enable_ssl, cert_path); if (!auth_result.success) { LOG(ERROR) << "Auth failed: " << auth_result.error_message; @@ -110,14 +110,14 @@ if (!auth_result.success) { } // 2. Select bucket (REQUIRED) -brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.SelectBucket("my_bucket"); +brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.selectBucket("my_bucket"); if (!bucket_result.success) { LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; return -1; } // 3. Perform operations -brpc::CouchbaseOperations::Result add_result = couchbase_ops.Add("user::123", json_value); +brpc::CouchbaseOperations::Result add_result = couchbase_ops.add("user::123", json_value); if (add_result.success) { std::cout << "Document added successfully!" << std::endl; } else { @@ -128,7 +128,7 @@ if (add_result.success) { #### SSL Authentication (Essential for Couchbase Capella): ```cpp // For Couchbase Capella (cloud) - SSL is REQUIRED -brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( +brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticate( username, password, "cluster.cloud.couchbase.com:11207", // SSL port @@ -140,11 +140,11 @@ brpc::CouchbaseOperations::Result auth_result = couchbase_ops.Authenticate( #### Collection Operations: ```cpp // Default collection -auto result = couchbase_ops.Get("doc::1"); +auto result = couchbase_ops.get("doc::1"); // Specific collection -auto result = couchbase_ops.Get("doc::1", "my_collection"); -auto add_result = couchbase_ops.Add("doc::2", value, "my_collection"); +auto result = couchbase_ops.get("doc::1", "my_collection"); +auto add_result = couchbase_ops.add("doc::2", value, "my_collection"); ``` #### Pipeline Operations (Performance Optimization): @@ -152,25 +152,25 @@ The pipeline API allows batching multiple operations into a single network call, ```cpp // Begin a new pipeline -if (!couchbase_ops.BeginPipeline()) { +if (!couchbase_ops.beginPipeline()) { LOG(ERROR) << "Failed to begin pipeline"; return -1; } // Add multiple operations to the pipeline (not executed yet) bool success = true; -success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "key1", "value1"); -success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, "key2", "value2"); -success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "key1"); -success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, "key3"); +success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "key1", "value1"); +success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::UPSERT, "key2", "value2"); +success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "key1"); +success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE, "key3"); if (!success) { - couchbase_ops.ClearPipeline(); // Clean up on error + couchbase_ops.clearPipeline(); // Clean up on error return -1; } // Execute all operations in a single network call -std::vector results = couchbase_ops.ExecutePipeline(); +std::vector results = couchbase_ops.executePipeline(); // Process results in the same order as requests for (size_t i = 0; i < results.size(); ++i) { @@ -188,19 +188,19 @@ for (size_t i = 0; i < results.size(); ++i) { **Pipeline with Collections**: ```cpp // Pipeline operations can also use collections -couchbase_ops.BeginPipeline(); -couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "doc1", "value1", "my_collection"); -couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "doc1", "", "my_collection"); -auto results = couchbase_ops.ExecutePipeline(); +couchbase_ops.beginPipeline(); +couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "doc1", "value1", "my_collection"); +couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "doc1", "", "my_collection"); +auto results = couchbase_ops.executePipeline(); ``` **Pipeline Management Methods**: -- `BeginPipeline()` - Start a new pipeline session -- `PipelineRequest(op_type, key, value, collection)` - Add operation to pipeline -- `ExecutePipeline()` - Execute all operations and return results -- `ClearPipeline()` - Clear pipeline without executing (cleanup) -- `IsPipelineActive()` - Check if pipeline is active -- `GetPipelineSize()` - Get number of queued operations +- `beginPipeline()` - Start a new pipeline session +- `pipelineRequest(op_type, key, value, collection)` - Add operation to pipeline +- `executePipeline()` - Execute all operations and return results +- `clearPipeline()` - Clear pipeline without executing (cleanup) +- `isPipelineActive()` - Check if pipeline is active +- `getPipelineSize()` - Get number of queued operations **Performance Benefits**: - **Reduced Network Overhead**: Multiple operations in single network round-trip @@ -210,10 +210,11 @@ auto results = couchbase_ops.ExecutePipeline(); #### Error Handling Pattern: ```cpp -brpc::CouchbaseOperations::Result result = couchbase_ops.SomeOperation(...); +brpc::CouchbaseOperations::Result result = couchbase_ops.someOperation(...); if (!result.success) { // Handle error LOG(ERROR) << "Operation failed: " << result.error_message; + LOG(ERROR) << "Error Code: "<< result.status_code; //what is the error code received. } else { // Use result.value if applicable (for Get operations) std::cout << "Retrieved value: " << result.value << std::endl; @@ -229,8 +230,8 @@ These classses are private to the `CouchbaseOpeartions` and is not exposed to th ```cpp CouchbaseRequest req; req.Authenticate(user, pass); // SASL PLAIN -req.SelectBucket("travel-sample"); -req.Add("doc::1", json_body, flags, exptime, /*cas*/0); +req.selectBucketRequest("travel-sample"); +req.addRequest("doc::1", json_body, flags, exptime, /*cas*/0); req.Get("doc::1"); // Pipeline GET after ADD channel.CallMethod(nullptr, &cntl, &req, &resp, nullptr); @@ -352,7 +353,7 @@ Create collections (7.0+): **SSL Configuration (Optional for Local)**: ```cpp // Local without SSL -auto result = couchbase_ops.Authenticate(username, password, "localhost:11210", false, ""); +auto result = couchbase_ops.authenticate(username, password, "localhost:11210", false, ""); ``` #### B. Couchbase Capella (Cloud) - **SSL Required** @@ -372,7 +373,7 @@ auto result = couchbase_ops.Authenticate(username, password, "localhost:11210", **Capella SSL Authentication Example**: ```cpp // Couchbase Capella - SSL is MANDATORY -auto result = couchbase_ops.Authenticate( +auto result = couchbase_ops.authenticate( "your_username", "your_password", "your-cluster.cloud.couchbase.com:11207", // SSL port @@ -403,12 +404,12 @@ Pipeline operations allow you to batch multiple Couchbase operations into a sing | Method | Description | Usage | |--------|-------------|-------| -| `BeginPipeline()` | Start a new pipeline session | Must call before adding operations | -| `PipelineRequest(op_type, key, value, collection)` | Add operation to pipeline | Supports all CRUD operations | -| `ExecutePipeline()` | Execute all queued operations | Returns `vector` in request order | -| `ClearPipeline()` | Clear pipeline without executing | Use for cleanup on errors | -| `IsPipelineActive()` | Check if pipeline is active | Returns `bool` | -| `GetPipelineSize()` | Get number of queued operations | Returns `size_t` | +| `beginPipeline()` | Start a new pipeline session | Must call before adding operations | +| `pipelineRequest(op_type, key, value, collection)` | Add operation to pipeline | Supports all CRUD operations | +| `executePipeline()` | Execute all queued operations | Returns `vector` in request order | +| `clearPipeline()` | Clear pipeline without executing | Use for cleanup on errors | +| `isPipelineActive()` | Check if pipeline is active | Returns `bool` | +| `getPipelineSize()` | Get number of queued operations | Returns `size_t` | #### Basic Pipeline Example @@ -419,25 +420,25 @@ brpc::CouchbaseOperations couchbase_ops; // ... authenticate and select bucket ... // 1. Begin pipeline -if (!couchbase_ops.BeginPipeline()) { +if (!couchbase_ops.beginPipeline()) { LOG(ERROR) << "Failed to begin pipeline"; return -1; } // 2. Add operations to pipeline (not executed yet) bool success = true; -success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "user1", "{\"name\":\"John\"}"); -success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "user2", "{\"name\":\"Jane\"}"); -success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "user1"); -success &= couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, "user3", "{\"name\":\"Bob\"}"); +success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "user1", "{\"name\":\"John\"}"); +success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "user2", "{\"name\":\"Jane\"}"); +success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "user1"); +success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::UPSERT, "user3", "{\"name\":\"Bob\"}"); if (!success) { - couchbase_ops.ClearPipeline(); + couchbase_ops.clearPipeline(); return -1; } // 3. Execute all operations in single network call -std::vector results = couchbase_ops.ExecutePipeline(); +std::vector results = couchbase_ops.executePipeline(); // 4. Process results (same order as requests) for (size_t i = 0; i < results.size(); ++i) { @@ -458,14 +459,14 @@ for (size_t i = 0; i < results.size(); ++i) { ```cpp // Pipeline with collection operations -couchbase_ops.BeginPipeline(); +couchbase_ops.beginPipeline(); // Add operations to specific collection -couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "doc1", "value1", "my_collection"); -couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "doc1", "", "my_collection"); -couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::DELETE, "doc2", "", "my_collection"); +couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "doc1", "value1", "my_collection"); +couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "doc1", "", "my_collection"); +couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE, "doc2", "", "my_collection"); -auto results = couchbase_ops.ExecutePipeline(); +auto results = couchbase_ops.executePipeline(); // Process results... ``` @@ -474,14 +475,14 @@ auto results = couchbase_ops.ExecutePipeline(); Pipeline operations provide granular error handling - each operation can succeed or fail independently: ```cpp -couchbase_ops.BeginPipeline(); +couchbase_ops.beginPipeline(); // Some operations may succeed, others may fail -couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::ADD, "existing_key", "value"); // May fail if key exists -couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::GET, "nonexistent_key"); // May fail if key doesn't exist -couchbase_ops.PipelineRequest(brpc::CouchbaseOperations::UPSERT, "new_key", "value"); // Should succeed +couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "existing_key", "value"); // May fail if key exists +couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "nonexistent_key"); // May fail if key doesn't exist +couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::UPSERT, "new_key", "value"); // Should succeed -auto results = couchbase_ops.ExecutePipeline(); +auto results = couchbase_ops.executePipeline(); // Check each result individually for (size_t i = 0; i < results.size(); ++i) { @@ -501,7 +502,7 @@ for (size_t i = 0; i < results.size(); ++i) { 1. **Batch Related Operations**: Group logically related operations together 2. **Handle Partial Failures**: Individual operations can fail while others succeed -3. **Clear on Errors**: Use `ClearPipeline()` if pipeline setup fails +3. **Clear on Errors**: Use `clearPipeline()` if pipeline setup fails 5. **Mixed Operations**: Combine different operation types (GET, ADD, UPSERT, DELETE) as needed #### When to Use Pipelines @@ -533,7 +534,7 @@ struct Result { **Recommended Pattern**: ```cpp -auto result = couchbase_ops.Add("key", "value"); +auto result = couchbase_ops.add("key", "value"); if (!result.success) { LOG(ERROR) << "Add failed: " << result.error_message; // Handle error appropriately @@ -587,14 +588,14 @@ int main() { } // Select bucket - auto bucket_result = couchbase_ops.SelectBucket(bucket_name); + auto bucket_result = couchbase_ops.selectBucket(bucket_name); if (!bucket_result.success) { LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; return -1; } // Perform operations with error handling - auto result = couchbase_ops.Add("key", "value", "collection_name"); + auto result = couchbase_ops.add("key", "value", "collection_name"); if (result.success) { std::cout << "Success!" << std::endl; } else { diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 603bb4528f..8e563eb075 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -2711,7 +2711,7 @@ bool CouchbaseOperations::pipelineRequest(operation_type op_type, const string& value, string collection_name) { if (!pipeline_active) { - DEBUG_PRINT("Pipeline not active. Call BeginPipeline() first."); + DEBUG_PRINT("Pipeline not active. Call beginPipeline() first."); return false; } From ec37f12f7594bd3c21d76718ced493dc874fa8f9 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 6 Oct 2025 12:12:14 +0530 Subject: [PATCH 34/49] updated documentation --- docs/en/couchbase_example.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 8b36f609ce..b4debd5794 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -93,7 +93,7 @@ Overall packet structure:- --- ### 4. High-Level API (`CouchbaseOperations`) -**Recommended Approach**: Use the `CouchbaseOperations` class for simple, thread-safe operations. +**Approach**: Use the `CouchbaseOperations` class for operations. It shall be noted that the instances of `CouchbaseOperations` should not be shared between threads, for some cases it might work but not recommended. #### Basic Usage: ```cpp From f4696096838ca9b23cd110c8ee7fed7cc2f6b1d0 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 6 Oct 2025 12:18:24 +0530 Subject: [PATCH 35/49] updated documentation --- docs/en/couchbase_example.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index b4debd5794..3108b313b8 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -608,7 +608,7 @@ int main() { --- ### 12. Summary and References -This implementation provides both high-level and low-level APIs for Couchbase KV and collection operations. Couchbase (the company) contributed to this implementation, but it is not officially supported; it is "[Community Supported](https://docs.couchbase.com/server/current/third-party/integrations.html#support-model)". +This implementation provides high-level APIs for Couchbase KV and collection operations. Couchbase (the company) contributed to this implementation, but it is not officially supported; it is "[Community Supported](https://docs.couchbase.com/server/current/third-party/integrations.html#support-model)". - **High-level API**: Recommended for most applications - simple, with built-in SSL support - **SSL Support**: Essential for Couchbase Capella and secure local deployments From 6988d330b5acca186e7d33374991d881e9d2c3db Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 6 Oct 2025 12:19:17 +0530 Subject: [PATCH 36/49] updated documentation --- docs/en/couchbase_example.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 3108b313b8..95361d2d1e 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -633,8 +633,8 @@ This implementation provides high-level APIs for Couchbase KV and collection ope > // Each thread creates its own instance > void worker_thread() { > brpc::CouchbaseOperations ops; // ✅ Thread-local instance -> ops.Authenticate(...); -> ops.Get("key"); // Safe +> ops.authenticate(...); +> ops.get("key"); // Safe > } > ``` > @@ -642,7 +642,7 @@ This implementation provides high-level APIs for Couchbase KV and collection ope > ```cpp > brpc::CouchbaseOperations global_ops; // ❌ Shared instance > void worker_thread() { -> global_ops.Get("key"); // ❌ RACE CONDITION - WILL CRASH! +> global_ops.get("key"); // ❌ RACE CONDITION - WILL CRASH! > } > ``` > From 2912a63b152a559b243743fcc05364fffbadf0d5 Mon Sep 17 00:00:00 2001 From: Giriraj Singh Date: Mon, 6 Oct 2025 20:46:04 +0530 Subject: [PATCH 37/49] Updated documentation --- docs/en/couchbase_example.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 95361d2d1e..2dc1bac250 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -1,6 +1,6 @@ ## Couchbase bRPC Binary Protocol Integration -This document explains the implementation of Couchbase Binary Protocol support added to this branch of bRPC, the available high-level operations, collection support, SSL authentication, and how to run the provided example client against either a local Couchbase Server cluster or a Couchbase Capella (cloud) deployment. However, the couchbase binary protocol implementation in bRPC requires us to do fine-grained optimizations which has been already done in the couchbase-cxx-client SDK, so we also added the support of couchbase using couchbase-cxx-SDK in bRPC and is available at [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc). +This document explains the implementation of Couchbase Binary Protocol support added to this branch of bRPC, the available high-level operations, collection support, SSL authentication, and how to run the provided example client against either a local Couchbase Server cluster or a Couchbase Capella (cloud) deployment. However, the couchbase binary protocol implementation in bRPC currently do not have fine-grained optimizations which has been already done in the couchbase-cxx-client SDK which also have query support, better error handling and much more optimized operations. So, we also added the support of couchbase using couchbase-cxx-SDK in bRPC and is available at [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc). --- ### 1. Overview @@ -648,4 +648,4 @@ This implementation provides high-level APIs for Couchbase KV and collection ope > > **Why?** `CouchbaseOperations` contains mutable state (pipeline queues, buffers, connection state) that is NOT thread-safe. Sharing instances will cause data corruption, pipeline interference, and application crashes. -Contributions and issue reports are welcome! \ No newline at end of file +Contributions and issue reports are welcome! From 2e13a7ebd81325abac4606d83293e0445632ffa2 Mon Sep 17 00:00:00 2001 From: Giriraj Singh Date: Mon, 6 Oct 2025 20:47:51 +0530 Subject: [PATCH 38/49] Updated documentation --- docs/en/couchbase_example.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 2dc1bac250..84833e7216 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -36,11 +36,10 @@ Design goals: | Collections | Collection-scoped CRUD operations | Pass collection name as optional parameter (defaults to "_default") | | Error Handling | `Result.success` + `Result.error_message` | Human-readable error messages with status codes | -**Key Differences from Low-Level API:** - **Simplified**: No need to manage channels, controllers, or response parsing - **Thread-Safe Per Instance**: Each `CouchbaseOperations` instance can be used independently ⚠️ **BUT NEVER SHARE BETWEEN THREADS** -- **Error Handling**: Simple boolean success with descriptive error messages -- **SSL Built-in**: Automatic SSL handling for secure connections +- **Error Handling**: Simple boolean success with descriptive error messages and error codes +- **SSL Built-in**: SSL handling for secure connections --- From 592db28b01db01559f3bf0fe1cf4ed06bd255cac Mon Sep 17 00:00:00 2001 From: Giriraj Singh Date: Mon, 6 Oct 2025 20:56:38 +0530 Subject: [PATCH 39/49] Update documentation --- docs/en/couchbase_example.md | 167 +++++------------------------------ 1 file changed, 21 insertions(+), 146 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 84833e7216..0d171e0764 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -149,6 +149,24 @@ auto add_result = couchbase_ops.add("doc::2", value, "my_collection"); #### Pipeline Operations (Performance Optimization): The pipeline API allows batching multiple operations into a single network call, significantly improving performance for bulk operations: +#### How Pipeline Operations Work + +1. **Begin Pipeline**: Start a new pipeline session +2. **Add Operations**: Queue multiple operations without executing them +3. **Execute Pipeline**: Send all operations in a single network call +4. **Process Results**: Handle results in the same order as requests + +#### Pipeline API Methods + +| Method | Description | Usage | +|--------|-------------|-------| +| `beginPipeline()` | Start a new pipeline session | Must call before adding operations | +| `pipelineRequest(op_type, key, value, collection)` | Add operation to pipeline | Supports all CRUD operations | +| `executePipeline()` | Execute all queued operations | Returns `vector` in request order | +| `clearPipeline()` | Clear pipeline without executing | Use for cleanup on errors | +| `isPipelineActive()` | Check if pipeline is active | Returns `bool` | +| `getPipelineSize()` | Get number of queued operations | Returns `size_t` | + ```cpp // Begin a new pipeline if (!couchbase_ops.beginPipeline()) { @@ -193,20 +211,6 @@ couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "doc1", "", "my_co auto results = couchbase_ops.executePipeline(); ``` -**Pipeline Management Methods**: -- `beginPipeline()` - Start a new pipeline session -- `pipelineRequest(op_type, key, value, collection)` - Add operation to pipeline -- `executePipeline()` - Execute all operations and return results -- `clearPipeline()` - Clear pipeline without executing (cleanup) -- `isPipelineActive()` - Check if pipeline is active -- `getPipelineSize()` - Get number of queued operations - -**Performance Benefits**: -- **Reduced Network Overhead**: Multiple operations in single network round-trip -- **Better Throughput**: Especially beneficial for bulk operations -- **Preserved Order**: Results returned in same order as requests -- **Error Isolation**: Individual operation failures don't affect others - #### Error Handling Pattern: ```cpp brpc::CouchbaseOperations::Result result = couchbase_ops.someOperation(...); @@ -388,137 +392,8 @@ auto result = couchbase_ops.authenticate( - Ensure firewall allows outbound connections on port 11207 --- -### 9. Pipeline Operations (Performance Optimization) - -Pipeline operations allow you to batch multiple Couchbase operations into a single network call, significantly improving performance for bulk operations. - -#### How Pipeline Operations Work -1. **Begin Pipeline**: Start a new pipeline session -2. **Add Operations**: Queue multiple operations without executing them -3. **Execute Pipeline**: Send all operations in a single network call -4. **Process Results**: Handle results in the same order as requests - -#### Pipeline API Methods - -| Method | Description | Usage | -|--------|-------------|-------| -| `beginPipeline()` | Start a new pipeline session | Must call before adding operations | -| `pipelineRequest(op_type, key, value, collection)` | Add operation to pipeline | Supports all CRUD operations | -| `executePipeline()` | Execute all queued operations | Returns `vector` in request order | -| `clearPipeline()` | Clear pipeline without executing | Use for cleanup on errors | -| `isPipelineActive()` | Check if pipeline is active | Returns `bool` | -| `getPipelineSize()` | Get number of queued operations | Returns `size_t` | - -#### Basic Pipeline Example - -```cpp -#include - -brpc::CouchbaseOperations couchbase_ops; -// ... authenticate and select bucket ... - -// 1. Begin pipeline -if (!couchbase_ops.beginPipeline()) { - LOG(ERROR) << "Failed to begin pipeline"; - return -1; -} - -// 2. Add operations to pipeline (not executed yet) -bool success = true; -success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "user1", "{\"name\":\"John\"}"); -success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "user2", "{\"name\":\"Jane\"}"); -success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "user1"); -success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::UPSERT, "user3", "{\"name\":\"Bob\"}"); - -if (!success) { - couchbase_ops.clearPipeline(); - return -1; -} - -// 3. Execute all operations in single network call -std::vector results = couchbase_ops.executePipeline(); - -// 4. Process results (same order as requests) -for (size_t i = 0; i < results.size(); ++i) { - const auto& result = results[i]; - if (result.success) { - std::cout << "Operation " << i << " succeeded"; - if (!result.value.empty()) { - std::cout << " - Value: " << result.value; - } - std::cout << std::endl; - } else { - std::cout << "Operation " << i << " failed: " << result.error_message << std::endl; - } -} -``` - -#### Collection-Scoped Pipeline Operations - -```cpp -// Pipeline with collection operations -couchbase_ops.beginPipeline(); - -// Add operations to specific collection -couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "doc1", "value1", "my_collection"); -couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "doc1", "", "my_collection"); -couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE, "doc2", "", "my_collection"); - -auto results = couchbase_ops.executePipeline(); -// Process results... -``` - -#### Pipeline Error Handling - -Pipeline operations provide granular error handling - each operation can succeed or fail independently: - -```cpp -couchbase_ops.beginPipeline(); - -// Some operations may succeed, others may fail -couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "existing_key", "value"); // May fail if key exists -couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "nonexistent_key"); // May fail if key doesn't exist -couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::UPSERT, "new_key", "value"); // Should succeed - -auto results = couchbase_ops.executePipeline(); - -// Check each result individually -for (size_t i = 0; i < results.size(); ++i) { - if (!results[i].success) { - LOG(WARNING) << "Operation " << i << " failed: " << results[i].error_message; - // Handle individual failures as needed - } -} -``` - -#### Performance Benefits - -- **Reduced Network Latency**: Multiple operations in single round-trip -- **Better Throughput**: Especially beneficial for bulk operations - -#### Pipeline Best Practices - -1. **Batch Related Operations**: Group logically related operations together -2. **Handle Partial Failures**: Individual operations can fail while others succeed -3. **Clear on Errors**: Use `clearPipeline()` if pipeline setup fails -5. **Mixed Operations**: Combine different operation types (GET, ADD, UPSERT, DELETE) as needed - -#### When to Use Pipelines - -**Ideal Use Cases**: -- Bulk data loading/migration -- Batch processing workflows -- Multi-document transactions (where order matters) -- Performance-critical applications with multiple operations - -**Not Recommended For**: -- Single operations (use regular methods) -- Operations requiring immediate results for decision making -- Very large batches that might timeout - ---- -### 10. Error Handling Patterns +### 9. Error Handling Patterns #### High-Level API (Recommended) The `CouchbaseOperations` class uses a simple `Result` struct: @@ -551,7 +426,7 @@ if (get_result.success) { ``` --- -### 11. Best Practices +### 10. Best Practices #### Thread Safety > ⚠️ **: THREAD SAFETY REQUIREMENTS** @@ -606,7 +481,7 @@ int main() { ``` --- -### 12. Summary and References +### 11. Summary and References This implementation provides high-level APIs for Couchbase KV and collection operations. Couchbase (the company) contributed to this implementation, but it is not officially supported; it is "[Community Supported](https://docs.couchbase.com/server/current/third-party/integrations.html#support-model)". - **High-level API**: Recommended for most applications - simple, with built-in SSL support From ddf97c4d7d5b62ae13a8f49d4043517f20971148 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 16 Oct 2025 11:46:08 +0530 Subject: [PATCH 40/49] Added features and fixed bugs in multithreaded environment Using connection_groups to differentiate between connections across CouchbaseOperations instances to different buckets. Renamed CollectionManifestTracker class to CollectionManifestManager and all the related functionality inside it as before refreshing method was outside this class Added two different authenticate method authenticate(not secure) and authenticateSSL(secure) --- src/brpc/couchbase.cpp | 320 +++++++++++++++---------- src/brpc/couchbase.h | 57 +++-- src/brpc/policy/couchbase_protocol.cpp | 22 +- 3 files changed, 239 insertions(+), 160 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 8e563eb075..e684e81e79 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -19,8 +19,10 @@ #include //for crc32 Vbucket_id +#define CB_ADD(a,b) (a+b) + // Debug flag for enabling debug statements -static bool DBUG = true; // Set to true to enable debug logs +static bool DBUG = false; // Set to true to enable debug logs // Debug print macro #define DEBUG_PRINT(msg) \ @@ -52,13 +54,13 @@ constexpr size_t RANDOM_ID_HEX_SIZE = 67; // 33 bytes * 2 + null terminator } // namespace // Static member definitions -CouchbaseMetadataTracking* +CouchbaseManifestManager* CouchbaseOperations::CouchbaseRequest::metadata_tracking = &common_metadata_tracking; -bool brpc::CouchbaseMetadataTracking::setBucketToCollectionManifest( +bool brpc::CouchbaseManifestManager::setBucketToCollectionManifest( string server, string bucket, - CouchbaseMetadataTracking::CollectionManifest manifest) { + CouchbaseManifestManager::CollectionManifest manifest) { // Then update the collection manifest with proper locking { UniqueLock write_lock(rw_bucket_to_collection_manifest_mutex_); @@ -68,9 +70,9 @@ bool brpc::CouchbaseMetadataTracking::setBucketToCollectionManifest( return true; } -bool brpc::CouchbaseMetadataTracking::getBucketToCollectionManifest( +bool brpc::CouchbaseManifestManager::getBucketToCollectionManifest( string server, string bucket, - CouchbaseMetadataTracking::CollectionManifest* manifest) { + CouchbaseManifestManager::CollectionManifest* manifest) { SharedLock read_lock(rw_bucket_to_collection_manifest_mutex_); auto it1 = bucket_to_collection_manifest_.find(server); if (it1 == bucket_to_collection_manifest_.end()) { @@ -84,8 +86,8 @@ bool brpc::CouchbaseMetadataTracking::getBucketToCollectionManifest( return true; } -bool brpc::CouchbaseMetadataTracking::getManifestToCollectionId( - CouchbaseMetadataTracking::CollectionManifest* manifest, string scope, +bool brpc::CouchbaseManifestManager::getManifestToCollectionId( + CouchbaseManifestManager::CollectionManifest* manifest, string scope, string collection, uint8_t* collection_id) { if (manifest == nullptr || collection_id == nullptr) { DEBUG_PRINT("Invalid input: manifest or collection_id is null"); @@ -106,9 +108,9 @@ bool brpc::CouchbaseMetadataTracking::getManifestToCollectionId( return true; } -bool CouchbaseMetadataTracking::jsonToCollectionManifest( +bool CouchbaseManifestManager::jsonToCollectionManifest( const string& json, - CouchbaseMetadataTracking::CollectionManifest* manifest) { + CouchbaseManifestManager::CollectionManifest* manifest) { if (manifest == nullptr) { DEBUG_PRINT("Invalid input: manifest is null"); return false; @@ -236,6 +238,108 @@ bool CouchbaseMetadataTracking::jsonToCollectionManifest( return true; } +bool CouchbaseManifestManager::refreshCollectionManifest( + brpc::Channel* channel, const string& server, const string& bucket, + unordered_map* local_collection_manifest_cache) { + // first fetch the manifest + // then compare the UID with the cached one + if (channel == nullptr) { + DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); + return false; + } + if (server.empty()) { + DEBUG_PRINT("Server is empty, make sure to call Authenticate() first"); + return false; + } + if (bucket.empty()) { + DEBUG_PRINT("No bucket selected, make sure to call SelectBucket() first"); + return false; + } + CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; + CouchbaseOperations::CouchbaseResponse temp_get_manifest_response; + brpc::Controller temp_cntl; + temp_get_manifest_request.getCollectionManifest(); + channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, + &temp_get_manifest_response, NULL); + if (temp_cntl.Failed()) { + DEBUG_PRINT("Failed to get collection manifest: bRPC controller error " + << temp_cntl.ErrorText()); + return false; + } + string manifest_json; + if (!temp_get_manifest_response.popManifest(&manifest_json)) { + DEBUG_PRINT("Failed to parse response for refreshing collection Manifest: " + << temp_get_manifest_response.lastError()); + return false; + } + brpc::CouchbaseManifestManager::CollectionManifest manifest; + if (!common_metadata_tracking.jsonToCollectionManifest(manifest_json, + &manifest)) { + DEBUG_PRINT("Failed to parse collection manifest JSON"); + return false; + } + brpc::CouchbaseManifestManager::CollectionManifest cached_manifest; + if (!common_metadata_tracking.getBucketToCollectionManifest( + server, bucket, &cached_manifest)) { + // No cached manifest found, set the new one + if (!common_metadata_tracking.setBucketToCollectionManifest(server, bucket, + manifest)) { + DEBUG_PRINT("Failed to cache collection manifest for bucket " + << bucket << " on server " << server); + return false; + } + DEBUG_PRINT("Cached collection manifest for bucket " + << bucket << " on server " << server); + // also update the local cache + if (local_collection_manifest_cache != nullptr) { + (*local_collection_manifest_cache)[bucket] = manifest; + } + return true; + } + // Compare the UID with the cached one + // If they are different, refresh the cache + else if (manifest.uid != cached_manifest.uid) { + DEBUG_PRINT("Collection manifest has changed for bucket " + << bucket << " on server " << server); + if (!common_metadata_tracking.setBucketToCollectionManifest(server, bucket, + manifest)) { + DEBUG_PRINT("Failed to update cached collection manifest for bucket " + << bucket << " on server " << server); + return false; + } + DEBUG_PRINT("Updated cached collection manifest for bucket " + << bucket << " on server " << server); + // update the local cache as well + if (local_collection_manifest_cache != nullptr) { + (*local_collection_manifest_cache)[bucket] = manifest; + DEBUG_PRINT("Added to local collection manifest cache for bucket " + << bucket << " on server " << server); + } + return true; + } else { + DEBUG_PRINT("Collection manifest is already up-to-date for bucket " + << bucket << " on server " << server); + if (local_collection_manifest_cache != nullptr) { + if (local_collection_manifest_cache->find(bucket) != + local_collection_manifest_cache->end()) { + // if the bucket already exists in the local cache, check the UID + if ((*local_collection_manifest_cache)[bucket].uid != manifest.uid) { + // if the UID is different, update the local cache + (*local_collection_manifest_cache)[bucket] = manifest; + DEBUG_PRINT("Updated local collection manifest cache for bucket " + << bucket << " on server " << server); + } + } else { + // if the bucket does not exist in the local cache, add it + (*local_collection_manifest_cache)[bucket] = manifest; + DEBUG_PRINT("Added to local collection manifest cache for bucket " + << bucket << " on server " << server); + } + } + return false; + } +} + uint32_t CouchbaseOperations::CouchbaseRequest::hashCrc32(const char* key, size_t key_length) { static const uint32_t crc32tab[256] = { @@ -718,8 +822,9 @@ bool CouchbaseOperations::CouchbaseRequest::getOrDelete( // fetch from the server. bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( string collection_name, uint8_t* coll_id, - brpc::CouchbaseMetadataTracking* metadata_tracking, brpc::Channel* channel, - const string& server, const string& selected_bucket) { + brpc::CouchbaseManifestManager* metadata_tracking, brpc::Channel* channel, + const string& server, const string& selected_bucket, + unordered_map* local_cache) { if (collection_name.empty()) { DEBUG_PRINT("Empty collection name"); return false; @@ -737,7 +842,7 @@ bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( return false; } - brpc::CouchbaseMetadataTracking::CollectionManifest manifest; + brpc::CouchbaseManifestManager::CollectionManifest manifest; // check if the server/bucket exists in the cached collection manifest if (!metadata_tracking->getBucketToCollectionManifest(server, selected_bucket, &manifest)) { @@ -745,7 +850,7 @@ bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( << selected_bucket << " on server " << server << ", fetching from server"); // No cached manifest found, fetch from server - if (!refreshCollectionManifest(channel, server, selected_bucket)) { + if (!metadata_tracking->refreshCollectionManifest(channel, server, selected_bucket, local_cache)) { return false; } // local cache will also be updated in refreshCollectionManifest @@ -764,7 +869,7 @@ bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( &manifest, "_default", collection_name, coll_id)) { // Just to verify that the collectionID does not exist in the manifest // refresh manifest from server and try again - if (!refreshCollectionManifest(channel, server, selected_bucket)) { + if (!metadata_tracking->refreshCollectionManifest(channel, server, selected_bucket, local_cache)) { return false; } // local cache will also be updated in refreshCollectionManifest @@ -799,7 +904,7 @@ bool CouchbaseOperations::CouchbaseRequest::getRequest( // if local cache is empty, goto global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "getRequest"); @@ -813,7 +918,7 @@ bool CouchbaseOperations::CouchbaseRequest::getRequest( // if not check in the global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "getRequest"); @@ -839,7 +944,7 @@ bool CouchbaseOperations::CouchbaseRequest::deleteRequest( // if local cache is empty, goto global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "deleteRequest"); @@ -853,7 +958,7 @@ bool CouchbaseOperations::CouchbaseRequest::deleteRequest( // if not check in the global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "deleteRequest"); @@ -1233,7 +1338,7 @@ bool CouchbaseOperations::CouchbaseRequest::upsertRequest( // if local cache is empty, goto global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "upsertRequest"); @@ -1247,7 +1352,7 @@ bool CouchbaseOperations::CouchbaseRequest::upsertRequest( // if not check in the global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "upsertRequest"); @@ -1308,101 +1413,6 @@ bool CouchbaseOperations::CouchbaseRequest::getCollectionManifest() { return true; } -bool CouchbaseOperations::CouchbaseRequest::refreshCollectionManifest( - brpc::Channel* channel, const string& server, const string& bucket) { - // first fetch the manifest - // then compare the UID with the cached one - if (channel == nullptr) { - DEBUG_PRINT("No channel found, make sure to call Authenticate() first"); - return false; - } - if (server.empty()) { - DEBUG_PRINT("Server is empty, make sure to call Authenticate() first"); - return false; - } - if (bucket.empty()) { - DEBUG_PRINT("No bucket selected, make sure to call SelectBucket() first"); - return false; - } - CouchbaseOperations::CouchbaseRequest temp_get_manifest_request; - CouchbaseOperations::CouchbaseResponse temp_get_manifest_response; - brpc::Controller temp_cntl; - temp_get_manifest_request.getCollectionManifest(); - channel->CallMethod(NULL, &temp_cntl, &temp_get_manifest_request, - &temp_get_manifest_response, NULL); - if (temp_cntl.Failed()) { - DEBUG_PRINT("Failed to get collection manifest: bRPC controller error " - << temp_cntl.ErrorText()); - return false; - } - string manifest_json; - if (!temp_get_manifest_response.popManifest(&manifest_json)) { - DEBUG_PRINT("Failed to parse response for refreshing collection Manifest: " - << temp_get_manifest_response.lastError()); - return false; - } - brpc::CouchbaseMetadataTracking::CollectionManifest manifest; - if (!common_metadata_tracking.jsonToCollectionManifest(manifest_json, - &manifest)) { - DEBUG_PRINT("Failed to parse collection manifest JSON"); - return false; - } - brpc::CouchbaseMetadataTracking::CollectionManifest cached_manifest; - if (!common_metadata_tracking.getBucketToCollectionManifest( - server, bucket, &cached_manifest)) { - // No cached manifest found, set the new one - if (!common_metadata_tracking.setBucketToCollectionManifest(server, bucket, - manifest)) { - DEBUG_PRINT("Failed to cache collection manifest for bucket " - << bucket << " on server " << server); - return false; - } - DEBUG_PRINT("Cached collection manifest for bucket " - << bucket << " on server " << server); - // also update the local cache - (*local_collection_manifest_cache)[bucket] = manifest; - return true; - } - // Compare the UID with the cached one - // If they are different, refresh the cache - else if (manifest.uid != cached_manifest.uid) { - DEBUG_PRINT("Collection manifest has changed for bucket " - << bucket << " on server " << server); - if (!common_metadata_tracking.setBucketToCollectionManifest(server, bucket, - manifest)) { - DEBUG_PRINT("Failed to update cached collection manifest for bucket " - << bucket << " on server " << server); - return false; - } - DEBUG_PRINT("Updated cached collection manifest for bucket " - << bucket << " on server " << server); - // update the local cache as well - (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Added to local collection manifest cache for bucket " - << bucket << " on server " << server); - return true; - } else { - DEBUG_PRINT("Collection manifest is already up-to-date for bucket " - << bucket << " on server " << server); - if (local_collection_manifest_cache->find(bucket) != - local_collection_manifest_cache->end()) { - // if the bucket already exists in the local cache, check the UID - if ((*local_collection_manifest_cache)[bucket].uid != manifest.uid) { - // if the UID is different, update the local cache - (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Updated local collection manifest cache for bucket " - << bucket << " on server " << server); - } - } else { - // if the bucket does not exist in the local cache, add it - (*local_collection_manifest_cache)[bucket] = manifest; - DEBUG_PRINT("Added to local collection manifest cache for bucket " - << bucket << " on server " << server); - } - return false; - } -} - bool CouchbaseOperations::CouchbaseRequest::addRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, @@ -1420,7 +1430,7 @@ bool CouchbaseOperations::CouchbaseRequest::addRequest( // if local cache is empty, goto global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "addRequest"); @@ -1434,7 +1444,7 @@ bool CouchbaseOperations::CouchbaseRequest::addRequest( // if not check in the global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "addRequest"); @@ -1458,7 +1468,7 @@ bool CouchbaseOperations::CouchbaseRequest::addRequest( // uint8_t coll_id = 0; // default collection ID // if(collection_name != "_default"){ // if(!getCachedOrFetchCollectionId(collection_name, &coll_id, -// metadata_tracking, channel, server, bucket)){ +// metadata_tracking, channel, server, bucket, local_collection_manifest_cache)){ // return false; // } // } @@ -1483,7 +1493,7 @@ bool CouchbaseOperations::CouchbaseRequest::appendRequest( // if local cache is empty, goto global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { return false; } } @@ -1493,7 +1503,7 @@ bool CouchbaseOperations::CouchbaseRequest::appendRequest( // if not check in the global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { return false; } } @@ -1518,7 +1528,7 @@ bool CouchbaseOperations::CouchbaseRequest::prependRequest( // if local cache is empty, goto global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { return false; } } @@ -1528,7 +1538,7 @@ bool CouchbaseOperations::CouchbaseRequest::prependRequest( // if not check in the global cache or fetch from server if (!getCachedOrFetchCollectionId(collection_name, &coll_id, metadata_tracking, channel, server, - bucket)) { + bucket, local_collection_manifest_cache)) { return false; } } @@ -1537,6 +1547,12 @@ bool CouchbaseOperations::CouchbaseRequest::prependRequest( coll_id); } +bool CouchbaseOperations::CouchbaseResponse::popAuthenticate(uint64_t* cas_value) { + return popStore(policy::CB_BINARY_SASL_AUTH, cas_value); +} +bool CouchbaseOperations::CouchbaseResponse::popHello(uint64_t* cas_value) { + return popStore(policy::CB_HELLO_SELECT_FEATURES, cas_value); +} bool CouchbaseOperations::CouchbaseResponse::popUpsert(uint64_t* cas_value) { return popStore(policy::CB_BINARY_SET, cas_value); } @@ -2072,7 +2088,7 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, // has been updated on the server side. The collectionID present in the // local cache/global cache is no longer valid. This can happen if a // collection is deleted and recreated with the same name. - if (!request->refreshCollectionManifest(channel, server, bucket)) { + if (!request->metadata_tracking->refreshCollectionManifest(channel, server, bucket, request->local_collection_manifest_cache)) { DEBUG_PRINT("Failed to refresh collection manifest"); result->error_message = "Failed to refresh collection manifest"; } else { @@ -2159,7 +2175,7 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, // collection_manifest has been updated on the server side. and the // client have a stale copy of collection manifest. In this case, we // need to refresh the collection manifest and retry the operation. - if (!request->refreshCollectionManifest(channel, server, bucket)) { + if (!request->metadata_tracking->refreshCollectionManifest(channel, server, bucket, request->local_collection_manifest_cache)) { DEBUG_PRINT("Failed to refresh collection manifest"); result->error_message = "Failed to refresh collection manifest"; return false; @@ -2281,7 +2297,7 @@ bool CouchbaseOperations::CouchbaseRequest::getLocalCachedCollectionId( } auto it = local_collection_manifest_cache->find(bucket); if (it != local_collection_manifest_cache->end()) { - CouchbaseMetadataTracking::CollectionManifest& manifest = it->second; + CouchbaseManifestManager::CollectionManifest& manifest = it->second; if (manifest.scope_to_collection_id_map.find(scope) != manifest.scope_to_collection_id_map.end()) { auto& collection_map = manifest.scope_to_collection_id_map[scope]; @@ -2341,13 +2357,32 @@ CouchbaseOperations::Result CouchbaseOperations::add(const string& key, CouchbaseOperations::Result CouchbaseOperations::authenticate( const string& username, const string& password, - const string& server_address, bool enable_ssl, string path_to_cert) { + const string& server_address, const string& bucket_name) { + return authenticateAll(username, password, server_address, bucket_name, false, + ""); +} + +CouchbaseOperations::Result CouchbaseOperations::authenticateSSL( + const string& username, const string& password, + const string& server_address, const string& bucket_name, + string path_to_cert) { + return authenticateAll(username, password, server_address, bucket_name, true, + path_to_cert); +} + +CouchbaseOperations::Result CouchbaseOperations::authenticateAll( + const string& username, const string& password, + const string& server_address, const string& bucket_name, bool enable_ssl, string path_to_cert) { // Create a channel to the Couchbase server brpc::ChannelOptions options; options.protocol = brpc::PROTOCOL_COUCHBASE; options.connection_type = "single"; options.timeout_ms = 1000; // 1 second options.max_retry = 3; + + // CRITICAL: Set unique connection_group to prevent connection sharing + // Each CouchbaseOperations instance connected to same bucket gets its own connection group + options.connection_group = server_address + bucket_name; // enable_ssl if (enable_ssl) { @@ -2388,11 +2423,38 @@ CouchbaseOperations::Result CouchbaseOperations::authenticate( result.error_message = cntl.ErrorText(); return result; } + uint64_t cas; + if(response.popHello(&cas) == false) { + DEBUG_PRINT("Failed to receive HELO response from Couchbase: " + << response.lastError()); + delete new_channel; + result.success = false; + result.value = ""; + result.error_message = response.lastError(); + result.status_code = response._status_code; + return result; + } + if (response.popAuthenticate(&cas) == false) { + DEBUG_PRINT("Failed to authenticate user: " + << username << " to Couchbase: " << response.lastError()); + result.success = false; + result.value = ""; + result.error_message = response.lastError(); + result.status_code = response._status_code; + return result; + } // Successfully authenticated channel_ = new_channel; this->server_address_ = server_address; result.success = true; result.status_code = 0; + + DEBUG_PRINT("Instance " << reinterpret_cast(this) + << " authenticated with unique connection_group:" + << server_address_ + bucket_name); + + // select the bucket + result = selectBucket(bucket_name); return result; } @@ -2435,7 +2497,7 @@ CouchbaseOperations::Result CouchbaseOperations::selectBucket( if (request.local_collection_manifest_cache->find(bucket_name) == request.local_collection_manifest_cache->end()) { // only fetch if not already present in the local cache - CouchbaseMetadataTracking::CollectionManifest manifest; + CouchbaseManifestManager::CollectionManifest manifest; if (!common_metadata_tracking.getBucketToCollectionManifest( server_address_, bucket_name, &manifest)) { DEBUG_PRINT("Collection manifest for bucket: " @@ -2445,7 +2507,7 @@ CouchbaseOperations::Result CouchbaseOperations::selectBucket( // manifest for this bucket/server is not cached yet, will fetch it from // server now. refresh will also update the local cache with the fetched // manifest - request.refreshCollectionManifest(channel_, server_address_, bucket_name); + request.metadata_tracking->refreshCollectionManifest(channel_, server_address_, bucket_name, request.local_collection_manifest_cache); // We simply try once to prefetch the manifest, before any collection // operation. If it fails, it will be lazily updated when a collection // operation is performed. diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index a1fff500e4..754341c902 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -115,7 +115,8 @@ class UniqueLock { ~UniqueLock() { lock_.unlock(); } }; -class CouchbaseMetadataTracking { +// manager +class CouchbaseManifestManager { public: struct CollectionManifest { string uid; // uid of the manifest, it can be used to track if the manifest @@ -131,8 +132,8 @@ class CouchbaseMetadataTracking { ReaderWriterLock rw_bucket_to_collection_manifest_mutex_; public: - CouchbaseMetadataTracking() {} - ~CouchbaseMetadataTracking() { bucket_to_collection_manifest_.clear(); } + CouchbaseManifestManager() {} + ~CouchbaseManifestManager() { bucket_to_collection_manifest_.clear(); } bool setBucketToCollectionManifest(string server, string bucket, CollectionManifest manifest); @@ -143,6 +144,9 @@ class CouchbaseMetadataTracking { bool jsonToCollectionManifest(const string& json, CollectionManifest* manifest); + bool refreshCollectionManifest(brpc::Channel* channel, const string& server, + const string& bucket, + unordered_map* local_cache = nullptr); } static common_metadata_tracking; class CouchbaseOperations { public: @@ -182,9 +186,11 @@ class CouchbaseOperations { // string& key, uint32_t exptime, string collection_name = "_default"); Result // Flush(uint32_t timeout = 0); Result version(); - Result authenticate(const string& username, const string& password, - const string& server_address, bool enable_ssl = false, + Result authenticateSSL(const string& username, const string& password, + const string& server_address, const string& bucket_name, string path_to_cert = ""); + Result authenticate(const string& username, const string& password, + const string& server_address, const string& bucket_name); Result selectBucket(const string& bucket_name); // Pipeline management @@ -207,6 +213,9 @@ class CouchbaseOperations { const string& collection, uint8_t* coll_id); private: + CouchbaseOperations::Result authenticateAll( + const string& username, const string& password, + const string& server_address, const string& bucket_name, bool enable_ssl, string path_to_cert); friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); friend void policy::SerializeCouchbaseRequest( butil::IOBuf* buf, Controller* cntl, @@ -216,12 +225,14 @@ class CouchbaseOperations { string selected_bucket_; unordered_map + CouchbaseManifestManager::CollectionManifest> local_bucket_to_collection_manifest_; + public: + // these classes have been made public so that normal user can also create advanced bRPC programs as per their requirements. class CouchbaseRequest : public NonreflectableMessage { - private: - static brpc::CouchbaseMetadataTracking* metadata_tracking; + public: + static brpc::CouchbaseManifestManager* metadata_tracking; int _pipelined_count; butil::IOBuf _buf; mutable int _cached_size_; @@ -240,12 +251,12 @@ class CouchbaseOperations { public: unordered_map* + CouchbaseManifestManager::CollectionManifest>* local_collection_manifest_cache; CouchbaseRequest( unordered_map* + CouchbaseManifestManager::CollectionManifest>* local_cache_reference) : NonreflectableMessage() { metadata_tracking = &common_metadata_tracking; @@ -258,13 +269,16 @@ class CouchbaseOperations { } ~CouchbaseRequest() { sharedDtor(); } CouchbaseRequest(const CouchbaseRequest& from) - : NonreflectableMessage(from) { + : NonreflectableMessage() { + metadata_tracking = &common_metadata_tracking; sharedCtor(); MergeFrom(from); } inline CouchbaseRequest& operator=(const CouchbaseRequest& from) { - CopyFrom(from); + if (this != &from) { + MergeFrom(from); + } return *this; } @@ -286,12 +300,10 @@ class CouchbaseOperations { bool getCachedOrFetchCollectionId( string collection_name, uint8_t* coll_id, - brpc::CouchbaseMetadataTracking* metadata_tracking, + brpc::CouchbaseManifestManager* metadata_tracking, brpc::Channel* channel, const string& server, - const string& selected_bucket); - - bool refreshCollectionManifest(brpc::Channel* channel, const string& server, - const string& bucket); + const string& selected_bucket, + unordered_map* local_cache); // Collection-aware document operations bool getRequest(const butil::StringPiece& key, @@ -372,9 +384,10 @@ class CouchbaseOperations { }; class CouchbaseResponse : public NonreflectableMessage { + public: + static brpc::CouchbaseManifestManager* metadata_tracking; private: string _err; - static brpc::CouchbaseMetadataTracking* metadata_tracking; butil::IOBuf _buf; mutable int _cached_size_; bool popCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); @@ -392,13 +405,15 @@ class CouchbaseOperations { } ~CouchbaseResponse() { sharedDtor(); } CouchbaseResponse(const CouchbaseResponse& from) - : NonreflectableMessage(from) { + : NonreflectableMessage() { metadata_tracking = &common_metadata_tracking; sharedCtor(); MergeFrom(from); } inline CouchbaseResponse& operator=(const CouchbaseResponse& from) { - CopyFrom(from); + if (this != &from) { + MergeFrom(from); + } return *this; } @@ -488,6 +503,8 @@ class CouchbaseOperations { bool popAppend(uint64_t* cas_value); bool popPrepend(uint64_t* cas_value); bool popSelectBucket(uint64_t* cas_value, std::string bucket_name); + bool popAuthenticate(uint64_t* cas_value); + bool popHello(uint64_t* cas_value); // Collection-related response methods bool popCollectionId(uint8_t* collection_id); diff --git a/src/brpc/policy/couchbase_protocol.cpp b/src/brpc/policy/couchbase_protocol.cpp index 6ccb1d888a..ab2d3c3efb 100644 --- a/src/brpc/policy/couchbase_protocol.cpp +++ b/src/brpc/policy/couchbase_protocol.cpp @@ -134,16 +134,16 @@ ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket* socket, msg->meta.append(&local_header, sizeof(local_header)); source->pop_front(sizeof(*header)); source->cutn(&msg->meta, total_body_length); - if (header->command == CB_BINARY_SASL_AUTH) { - if (header->status != 0) { - LOG(ERROR) << "Failed to authenticate the couchbase Server."; - return MakeParseError(PARSE_ERROR_NO_RESOURCE, - "Fail to authenticate with the couchbase Server"); - } - msg = static_cast(socket->release_parsing_context()); - msg->pi = pi; - return MakeMessage(msg); - } else { + // if (header->command == CB_BINARY_SASL_AUTH) { + // if (header->status != 0) { + // LOG(ERROR) << "Failed to authenticate the couchbase Server."; + // return MakeParseError(PARSE_ERROR_NO_RESOURCE, + // "Fail to authenticate with the couchbase Server"); + // } + // msg = static_cast(socket->release_parsing_context()); + // msg->pi = pi; + // return MakeMessage(msg); + // } else { if (++msg->pi.count >= pi.count) { CHECK_EQ(msg->pi.count, pi.count); msg = @@ -153,7 +153,7 @@ ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket* socket, } else { socket->GivebackPipelinedInfo(pi); } - } + // } } } From 8cab3d14d51b0199e51e8b73831eb3a6ea913385 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Fri, 17 Oct 2025 01:58:55 +0530 Subject: [PATCH 41/49] Updated multithreaded and single threaded code. Added an example where a single instance is being shared across the threads when operating on single bucket. --- example/couchbase_c++/couchbase_client.cpp | 136 +++++++++--------- .../multithreaded_couchbase_client.cpp | 79 ++++++++-- 2 files changed, 136 insertions(+), 79 deletions(-) diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index 949aea5d47..a3dbca6842 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -28,71 +28,8 @@ #define RESET "\033[0m" DEFINE_string(server, "localhost:11210", "IP Address of server"); - -int main() { - // Create CouchbaseOperations instance for high-level operations - brpc::CouchbaseOperations couchbase_ops; - - std::cout << GREEN << "Using high-level CouchbaseOperations interface" - << RESET << std::endl; - - // Ask username and password for authentication - std::string username = "Administrator"; - std::string password = "password"; - while (username.empty() || password.empty()) { - std::cout << "Enter Couchbase username: "; - std::cin >> username; - if (username.empty()) { - std::cout << "Username cannot be empty. Please enter again." << std::endl; - continue; - } - std::cout << "Enter Couchbase password: "; - std::cin >> password; - if (password.empty()) { - std::cout << "Password cannot be empty. Please enter again." << std::endl; - continue; - } - } - - // Use high-level authentication method - // when connecting to capella use couchbase_ops.authenticate(username, - // password, FLAGS_server, true, "path/to/cert.txt"); - brpc::CouchbaseOperations::Result auth_result = - couchbase_ops.authenticate(username, password, FLAGS_server); - if (!auth_result.success) { - LOG(ERROR) << "Authentication failed: " << auth_result.error_message; - return -1; - } - - std::cout - << GREEN - << "Authentication successful, proceeding with Couchbase operations..." - << RESET << std::endl; - - // Select bucket - std::string bucket_name = "testing"; - while (bucket_name.empty()) { - std::cout << "Enter Couchbase bucket name: "; - std::cin >> bucket_name; - if (bucket_name.empty()) { - std::cout << "Bucket name cannot be empty. Please enter again." - << std::endl; - continue; - } - } - - // Use high-level bucket selection method - brpc::CouchbaseOperations::Result bucket_result = - couchbase_ops.selectBucket(bucket_name); - if (!bucket_result.success) { - LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; - return -1; - } else { - std::cout << GREEN << "Bucket Selection Successful" << RESET << std::endl; - } - - // Add operation using high-level method - std::string add_key = "user::test_brpc_binprot"; +int performOperations(brpc::CouchbaseOperations& couchbase_ops){ + std::string add_key = "user::test_brpc_binprot"; std::string add_value = R"({"name": "John Doe", "age": 30, "email": "john@example.com"})"; @@ -232,7 +169,7 @@ int main() { // Retrieve Collection ID for scope `_default` and collection // `testing_collection` const std::string scope_name = "_default"; // default scope - std::string collection_name = "non_existent_collection"; // target collection + std::string collection_name = "col1"; // target collection // enter collection name as user input // std::cout << "Enter collection name (default 'testing_collection'): "; // std::string user_input; @@ -454,5 +391,72 @@ int main() { std::cout << GREEN << "\n=== All operations completed successfully! ===" << RESET << std::endl; +} +int main() { + // Create CouchbaseOperations instance for high-level operations + brpc::CouchbaseOperations couchbase_ops; + + // std::cout << GREEN << "Using high-level CouchbaseOperations interface" + // << RESET << std::endl; + + // Ask username and password for authentication + std::string username = "Administrator"; + std::string password = "password"; + while (username.empty() || password.empty()) { + std::cout << "Enter Couchbase username: "; + std::cin >> username; + if (username.empty()) { + std::cout << "Username cannot be empty. Please enter again." << std::endl; + continue; + } + std::cout << "Enter Couchbase password: "; + std::cin >> password; + if (password.empty()) { + std::cout << "Password cannot be empty. Please enter again." << std::endl; + continue; + } + } + + // Use high-level authentication method + // when connecting to capella use couchbase_ops.authenticate(username, + // password, FLAGS_server, true, "path/to/cert.txt"); + brpc::CouchbaseOperations::Result auth_result = + couchbase_ops.authenticate(username, password, FLAGS_server, "testing"); + if (!auth_result.success) { + LOG(ERROR) << "Authentication failed: " << auth_result.error_message; + return -1; + } + + std::cout + << GREEN + << "Authentication successful, proceeding with Couchbase operations..." + << RESET << std::endl; + + performOperations(couchbase_ops); + + // Change bucket Selection + std::string bucket_name = "testing"; + while (bucket_name.empty()) { + std::cout << "Enter Couchbase bucket name: "; + std::cin >> bucket_name; + if (bucket_name.empty()) { + std::cout << "Bucket name cannot be empty. Please enter again." + << std::endl; + continue; + } + } + + // Use high-level bucket selection method + brpc::CouchbaseOperations::Result bucket_result = + couchbase_ops.selectBucket(bucket_name); + if (!bucket_result.success) { + LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; + return -1; + } else { + std::cout << GREEN << "Bucket Selection Successful" << RESET << std::endl; + } + performOperations(couchbase_ops); + // Add operation using high-level method + return 0; } diff --git a/example/couchbase_c++/multithreaded_couchbase_client.cpp b/example/couchbase_c++/multithreaded_couchbase_client.cpp index df40aab48c..bc8bcd725d 100644 --- a/example/couchbase_c++/multithreaded_couchbase_client.cpp +++ b/example/couchbase_c++/multithreaded_couchbase_client.cpp @@ -40,8 +40,7 @@ const int THREADS_PER_BUCKET = 5; struct { std::string username = "Administrator"; std::string password = "password"; - std::vector bucket_names = {"testing0", "testing1", "testing2", - "testing3"}; + std::vector bucket_names = {"t0", "t1", "t2", "t3"}; } g_config; // Simple thread statistics @@ -79,6 +78,7 @@ struct ThreadArgs { int thread_id; int bucket_id; std::string bucket_name; + brpc::CouchbaseOperations* couchbase_ops; ThreadStats* stats; }; @@ -140,6 +140,7 @@ void perform_crud_operations_col1(brpc::CouchbaseOperations& couchbase_ops, stats->operations_successful++; } else { stats->operations_failed++; + cout << "UPSERT failed: " << result.error_message << std::endl; return; } @@ -151,6 +152,7 @@ void perform_crud_operations_col1(brpc::CouchbaseOperations& couchbase_ops, stats->operations_successful++; } else { stats->operations_failed++; + cout << "GET failed: " << result.error_message << std::endl; return; } @@ -177,24 +179,26 @@ void* thread_worker(void* arg) { // Authentication brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticate( - g_config.username, g_config.password, "127.0.0.1:11210", false, ""); - + g_config.username, g_config.password, "127.0.0.1:11210", args->bucket_name); + // for SSL authentication use below line instead + // brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticateSSL(username, password, "127.0.0.1:11210", args->bucket_name, "/path/to/cert.txt"); + if (!auth_result.success) { std::cout << RED << "Thread " << args->thread_id << ": Auth failed - " << auth_result.error_message << RESET << std::endl; return NULL; } - // Select bucket - brpc::CouchbaseOperations::Result bucket_result = - couchbase_ops.selectBucket(args->bucket_name); + // // Select bucket + // brpc::CouchbaseOperations::Result bucket_result = + // couchbase_ops.selectBucket(args->bucket_name); - if (!bucket_result.success) { - std::cout << RED << "Thread " << args->thread_id - << ": Bucket selection failed - " << bucket_result.error_message - << RESET << std::endl; - return NULL; - } + // if (!bucket_result.success) { + // std::cout << RED << "Thread " << args->thread_id + // << ": Bucket selection failed - " << bucket_result.error_message + // << RESET << std::endl; + // return NULL; + // } std::cout << GREEN << "Thread " << args->thread_id << " connected to bucket " << args->bucket_name << RESET << std::endl; @@ -227,6 +231,24 @@ void* thread_worker(void* arg) { return NULL; } +void* shared_object_thread_worker(void *arg){ + ThreadArgs* shared_args = static_cast(arg); + brpc::CouchbaseOperations* shared_couchbase_ops = shared_args->couchbase_ops; + // Perform operations - 10 times on default collection, 10 times on col1 + // collection + for (int i = 0; i < 10; ++i) { + std::string base_key = + butil::string_printf("shared_thread_op_%d_thread_id_%d", i, shared_args->thread_id); + // CRUD operations on default collection + perform_crud_operations_default(*shared_couchbase_ops, base_key, shared_args->stats); + // CRUD operations on col1 collection + perform_crud_operations_col1(*shared_couchbase_ops, base_key, shared_args->stats); + // Small delay between operations + bthread_usleep(10000); // 10ms + } + return NULL; +} + // Print simple statistics void print_stats() { g_stats.aggregate_stats(); @@ -314,6 +336,37 @@ int main(int argc, char* argv[]) { } std::cout << GREEN << "All threads completed!" << RESET << std::endl; + // create a shared CouchbaseOperations instance + brpc::CouchbaseOperations shared_couchbase_ops; + brpc::CouchbaseOperations::Result result; + result = shared_couchbase_ops.authenticate( + g_config.username, g_config.password, "127.0.0.1:11210", "t0"); + if(result.success){ + std::cout << GREEN << "Shared CouchbaseOperations instance authenticated successfully!" << RESET << std::endl; + } else { + std::cout << RED << "Shared CouchbaseOperations instance authentication failed: " << result.error_message << RESET << std::endl; + return -1; + } + + for (int i = 0; i < NUM_THREADS; ++i) { + args[i].thread_id = i; + args[i].couchbase_ops = &shared_couchbase_ops; + args[i].bucket_id = 0; + args[i].bucket_name = "t0"; + // args[i].stats = &g_stats.per_thread_stats[i]; + } + + for(int i = 0; i < NUM_THREADS; ++i){ + if (bthread_start_background(&threads[i], NULL, shared_object_thread_worker, &args[i]) != + 0) { + std::cout << RED << "Failed to create shared object thread " << i << RESET << std::endl; + return -1; + } + } + for(int i = 0; i < NUM_THREADS; ++i){ + bthread_join(threads[i], NULL); + } + std::cout << GREEN << "All shared object threads completed!" << RESET << std::endl; // Print statistics print_stats(); From 1639f5d99bbf6d098e7b5c7ed4c2066d35324e8b Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Fri, 17 Oct 2025 12:21:28 +0530 Subject: [PATCH 42/49] updated documentation updated the documentation on thread safe operations and fixed small small discrepancies. --- docs/en/couchbase_example.md | 780 ++++++++++++++++++++++++++++------- 1 file changed, 632 insertions(+), 148 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 0d171e0764..55ed46adac 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -1,23 +1,21 @@ ## Couchbase bRPC Binary Protocol Integration -This document explains the implementation of Couchbase Binary Protocol support added to this branch of bRPC, the available high-level operations, collection support, SSL authentication, and how to run the provided example client against either a local Couchbase Server cluster or a Couchbase Capella (cloud) deployment. However, the couchbase binary protocol implementation in bRPC currently do not have fine-grained optimizations which has been already done in the couchbase-cxx-client SDK which also have query support, better error handling and much more optimized operations. So, we also added the support of couchbase using couchbase-cxx-SDK in bRPC and is available at [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc). +This document explains the implementation of Couchbase Binary Protocol support added to bRPC, and the available high-level operations, collection support, SSL authentication, and how to run the provided example client against either a local Couchbase Server cluster or a Couchbase Capella (cloud) deployment. However, the couchbase binary protocol implementation in bRPC currently do not have fine-grained optimizations which has been already done in the couchbase-cxx-client SDK also having query support, better error handling and much more optimized/reliable operations. So, we also added the support of couchbase using couchbase-cxx-SDK in bRPC and is available at [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc). --- ### 1. Overview The integration provides high-level APIs for communicating with Couchbase Server using its Binary Protocol, using the high-level `CouchbaseOperations` class which provides a simplified interface. -> Each thread must create and use its own `CouchbaseOperations` instance. Sharing instances will cause race conditions and unpredictable behavior. - The core pieces are: * `src/brpc/policy/couchbase_protocol.[h|cpp]` – framing + parse loop for binary responses, and request serialization. -* `src/brpc/couchbase.[h|cpp]` – high-level `CouchbaseOperations` class with simple methods, plus low-level request/response builders (`CouchbaseRequest`), parsers (`CouchbaseResponse`) and error-handlers. +* `src/brpc/couchbase.[h|cpp]` – high-level `CouchbaseOperations` class with request (`CouchbaseRequest`) and response (`CouchbaseResponse`) builders, parsers and error-handlers. * `example/couchbase_c++/couchbase_client.cpp` – an end‑to‑end example using the high-level API for authentication, bucket selection, CRUD operations, and collection‑scoped operations. -* `example/couchbase_c++/multithreaded_couchbase_client.cpp` – a multithreaded example showing how each thread creates its own `CouchbaseOperations` instance. +* `example/couchbase_c++/multithreaded_couchbase_client.cpp` – a multithreaded example where an instance of `CouchbaseOperations` is shared across the threads operating on same bucket. An another block of code where multiple threads have their own `CouchbaseOperations` instance as the threads operate on different buckets. Design goals: * **SSL Support**: Built-in SSL/TLS support for secure connections to Couchbase Capella. -* **Per-instance Authentication**: Each `CouchbaseOperations` object maintains its own authenticated session. +* **Per-instance Authentication**: Each `CouchbaseOperations` object maintains its own authenticated session if each instance connects to a different bucket, when multiple instances connect/operate on the same bucket then a single TCP socket is shared for these `CouchbaseOperations` instances because separate `connection_groups` are created on the basis of `server_name+bucket`. * **Collection Support**: Native support for collection-scoped operations. * Keep wire structs identical to the binary protocol (24‑byte header, network order numeric fields). * Future extensions for advanced features. @@ -27,17 +25,17 @@ Design goals: | Category | Supported Operations | Notes | |----------|----------------------|-------| -| **High-Level API** | `CouchbaseOperations` class | **Recommended**: Simple methods returning `Result` structs | +| **High-Level API** | `CouchbaseOperations` class | **Recommended**: Simple methods returning `Result` struct | | **SSL/TLS Support** | Built-in SSL encryption | **Required** for Couchbase Capella, optional for local clusters | -| Authentication | SASL `PLAIN` with SSL | Each `CouchbaseOperations` instance requires authentication | -| Bucket selection | `selectBucket()` method | Required before document operations | -| Basic KV | `add()`, `upsert()`, `delete_()`, `get()` | Clean API with `Result` struct error handling | +| Authentication | SASL `PLAIN` with/without SSL | `authenticate()` for non-SSL, `authenticateSSL()` for SSL connections | +| Bucket selection | Integrated with authentication | Bucket specified during authentication; `selectBucket()` also available separately | +| Basic KV | `add()`, `upsert()`, `delete_()`, `get()` | Clean API with `Result` struct error handling; | | **Pipeline Operations** | `beginPipeline()`, `pipelineRequest()`, `executePipeline()` | **NEW**: Batch multiple operations in single network call for improved performance | | Collections | Collection-scoped CRUD operations | Pass collection name as optional parameter (defaults to "_default") | -| Error Handling | `Result.success` + `Result.error_message` | Human-readable error messages with status codes | +| Error Handling | `Result.success` + `Result.error_message` + `Result.status_code` | Human-readable error messages with Couchbase status codes | - **Simplified**: No need to manage channels, controllers, or response parsing -- **Thread-Safe Per Instance**: Each `CouchbaseOperations` instance can be used independently ⚠️ **BUT NEVER SHARE BETWEEN THREADS** +- **Flexible Threading**: Share instances across threads for same bucket/server, or create separate instances for different buckets/servers - **Error Handling**: Simple boolean success with descriptive error messages and error codes - **SSL Built-in**: SSL handling for secure connections @@ -92,7 +90,7 @@ Overall packet structure:- --- ### 4. High-Level API (`CouchbaseOperations`) -**Approach**: Use the `CouchbaseOperations` class for operations. It shall be noted that the instances of `CouchbaseOperations` should not be shared between threads, for some cases it might work but not recommended. +**Approach**: Use the `CouchbaseOperations` class for operations. Instances can be shared across threads when connecting to the same bucket, or you can create separate instances in multi-threading where each thread is connecting to a separate bucket. #### Basic Usage: ```cpp @@ -100,39 +98,36 @@ Overall packet structure:- brpc::CouchbaseOperations couchbase_ops; -// 1. Authenticate (REQUIRED for each instance) +// 1. Authenticate with bucket selection (REQUIRED for each instance) brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticate( - username, password, server_address, enable_ssl, cert_path); + username, password, server_address, bucket_name); if (!auth_result.success) { LOG(ERROR) << "Auth failed: " << auth_result.error_message; return -1; } -// 2. Select bucket (REQUIRED) -brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.selectBucket("my_bucket"); -if (!bucket_result.success) { - LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; - return -1; -} - -// 3. Perform operations +// 2. Perform operations (bucket is already selected during authentication) brpc::CouchbaseOperations::Result add_result = couchbase_ops.add("user::123", json_value); if (add_result.success) { std::cout << "Document added successfully!" << std::endl; } else { std::cout << "Add failed: " << add_result.error_message << std::endl; } + +// Optional: Switch to a different bucket (if needed) +// brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.selectBucket("another_bucket"); +``` ``` #### SSL Authentication (Essential for Couchbase Capella): ```cpp // For Couchbase Capella (cloud) - SSL is REQUIRED -brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticate( +brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticateSSL( username, password, "cluster.cloud.couchbase.com:11207", // SSL port - true, // enable_ssl = true - "path/to/certificate.pem" // certificate path + bucket_name, // bucket name + "path/to/certificate.txt" // certificate path ); ``` @@ -217,7 +212,7 @@ brpc::CouchbaseOperations::Result result = couchbase_ops.someOperation(...); if (!result.success) { // Handle error LOG(ERROR) << "Operation failed: " << result.error_message; - LOG(ERROR) << "Error Code: "<< result.status_code; //what is the error code received. + LOG(ERROR) << "Error Code: " << result.status_code; // Couchbase status code } else { // Use result.value if applicable (for Get operations) std::cout << "Retrieved value: " << result.value << std::endl; @@ -225,23 +220,24 @@ if (!result.success) { ``` --- -### 5. Request/Response Class (`CouchbaseRequest`/`CouchbaseResponse`) +### 5. Request/Response Classes (`CouchbaseRequest`/`CouchbaseResponse`) -These classses are private to the `CouchbaseOpeartions` and is not exposed to the user. These classes are responsible for building the request that needs to be sent and received over the channel. A basic overview of how the request/response classes works internally has been shown below. +These classes are public in `CouchbaseOperations` and can be used for advanced bRPC programs. The high-level API uses these classes internally. They are responsible for building the request that needs to be sent and received over the channel. -#### Request Building: +#### Request Building (Advanced Usage): ```cpp CouchbaseRequest req; -req.Authenticate(user, pass); // SASL PLAIN +req.helloRequest(); // HELLO negotiation +req.authenticateRequest(user, pass); // SASL PLAIN authentication req.selectBucketRequest("travel-sample"); req.addRequest("doc::1", json_body, flags, exptime, /*cas*/0); -req.Get("doc::1"); // Pipeline GET after ADD +req.getRequest("doc::1"); // Pipeline GET after ADD channel.CallMethod(nullptr, &cntl, &req, &resp, nullptr); ``` #### Response Parsing: -Each `Pop*` method consumes the front of the internal response buffer, validating: +Each `pop*` method consumes the front of the internal response buffer, validating: 1. Header present. 2. Opcode matches expected operation. 3. Status == success (otherwise `_err` filled with formatted message). @@ -254,37 +250,550 @@ Each `Pop*` method consumes the front of the internal response buffer, validatin Uses the **high-level `CouchbaseOperations` API**: 1. **Create `CouchbaseOperations` instance** - can create more than one per thread. +```cpp +brpc::CouchbaseOperations couchbase_ops; +``` + 2. **Prompt for credentials** - username/password for authentication. -3. **SSL Authentication** - with support for Couchbase Capella certificate-based SSL. -4. **Select bucket** - required before any document operations. -5. **Basic CRUD operations**: +```cpp +std::string username = "Administrator"; +std::string password = "password"; +while (username.empty() || password.empty()) { + std::cout << "Enter Couchbase username: "; + std::cin >> username; + std::cout << "Enter Couchbase password: "; + std::cin >> password; +} +``` + +3. **Authentication with bucket selection** - `authenticate()` for local, `authenticateSSL()` for Capella. + +**Function Signatures:** +```cpp +// Non-SSL authentication +Result authenticate(const string& username, // Couchbase username + const string& password, // Couchbase password + const string& server_address, // Server host:port (e.g., "localhost:11210") + const string& bucket_name); // Target bucket name + +// SSL authentication +Result authenticateSSL(const string& username, // Couchbase username + const string& password, // Couchbase password + const string& server_address, // Server host:port (e.g., "cluster.cloud.couchbase.com:11207") + const string& bucket_name, // Target bucket name + string path_to_cert); // Path to SSL certificate file +``` + +**Usage Examples:** +```cpp +// For local Couchbase (non-SSL) +brpc::CouchbaseOperations::Result auth_result = + couchbase_ops.authenticate(username, password, FLAGS_server, "testing"); + +// For Couchbase Capella (SSL) +// brpc::CouchbaseOperations::Result auth_result = +// couchbase_ops.authenticateSSL(username, password, "cluster.cloud.couchbase.com:11207", +// "bucket_name", "path/to/cert.txt"); + +if (!auth_result.success) { + LOG(ERROR) << "Authentication failed: " << auth_result.error_message; + return -1; +} +``` + +4. **Basic CRUD operations**: - Add document (should succeed) - Try adding same key again (should fail with "key exists") - Get document (retrieve the added document) -6. **Multiple document operations** - Add several documents with different keys. -7. **Upsert operations**: + +**Function Signatures:** +```cpp +// ADD operation - creates new document, fails if key exists +Result add(const string& key, // Document key/ID + const string& value, // Document value (JSON string) + string collection_name = "_default"); // Collection name (optional, defaults to "_default") + +// GET operation - retrieves document by key +Result get(const string& key, // Document key/ID to retrieve + string collection_name = "_default"); // Collection name (optional, defaults to "_default") +``` + +**Usage Examples:** +```cpp +std::string add_key = "user::test_brpc_binprot"; +std::string add_value = R"({"name": "John Doe", "age": 30, "email": "john@example.com"})"; + +// First ADD operation (should succeed) +brpc::CouchbaseOperations::Result add_result = couchbase_ops.add(add_key, add_value); +if (add_result.success) { + std::cout << "ADD operation successful" << std::endl; +} else { + std::cout << "ADD operation failed: " << add_result.error_message << std::endl; +} + +// Second ADD operation (should fail - key exists) +brpc::CouchbaseOperations::Result add_result2 = couchbase_ops.add(add_key, add_value); +if (!add_result2.success) { + std::cout << "Second ADD failed as expected: " << add_result2.error_message << std::endl; +} + +// GET operation +brpc::CouchbaseOperations::Result get_result = couchbase_ops.get(add_key); +if (get_result.success) { + std::cout << "GET operation successful" << std::endl; + std::cout << "GET value: " << get_result.value << std::endl; +} +``` + +5. **Multiple document operations** - Add several documents with different keys. +```cpp +std::string item1_key = "binprot_item1"; +std::string item2_key = "binprot_item2"; +std::string item3_key = "binprot_item3"; + +couchbase_ops.add(item1_key, add_value); +couchbase_ops.add(item2_key, add_value); +couchbase_ops.add(item3_key, add_value); +``` + +6. **Upsert operations**: - Upsert existing document (should update) - Upsert new document (should create) - Verify with Get operations -8. **Delete operations**: + +**Function Signature:** +```cpp +// UPSERT operation - creates new document or updates existing one +Result upsert(const string& key, // Document key/ID + const string& value, // Document value (JSON string) + string collection_name = "_default"); // Collection name (optional, defaults to "_default") +``` + +**Usage Examples:** +```cpp +std::string upsert_key = "upsert_test"; +std::string upsert_value = R"({"operation": "upsert", "version": 1})"; + +// Upsert new document (will create) +brpc::CouchbaseOperations::Result upsert_result = couchbase_ops.upsert(upsert_key, upsert_value); + +// Upsert existing document (will update) +std::string updated_value = R"({"operation": "upsert", "version": 2})"; +brpc::CouchbaseOperations::Result update_result = couchbase_ops.upsert(upsert_key, updated_value); + +// Verify with GET +brpc::CouchbaseOperations::Result check_result = couchbase_ops.get(upsert_key); +``` + +7. **Delete operations**: - Delete non-existent key (should fail gracefully) - Delete existing key (should succeed) -9. **Collection-scoped operations** - Add/Get/Upsert/Delete in specific collections. -10. **Pipeline operations demo**: - - Begin pipeline and add multiple operations - - Execute batch operations in single network call - - Process results in order - - Collection-scoped pipeline operations - - Error handling and cleanup + +**Function Signature:** +```cpp +// DELETE operation - removes document by key +Result delete_(const string& key, // Document key/ID to delete + string collection_name = "_default"); // Collection name (optional, defaults to "_default") +``` + +**Usage Examples:** +```cpp +// Delete non-existent key +std::string delete_key = "non_existent_key"; +brpc::CouchbaseOperations::Result delete_result = couchbase_ops.delete_(delete_key); +if (!delete_result.success) { + std::cout << "Delete failed as expected: " << delete_result.error_message << std::endl; +} + +// Delete existing key +std::string delete_existing_key = "binprot_item1"; +brpc::CouchbaseOperations::Result delete_existing_result = couchbase_ops.delete_(delete_existing_key); +if (delete_existing_result.success) { + std::cout << "Delete existing key successful" << std::endl; +} +``` + +8. **Collection-scoped operations** - Add/Get/Upsert/Delete in specific collections. + +**Note:** All CRUD operations support an optional collection parameter. When not specified, operations default to the "_default" collection. + +**Usage Examples:** +```cpp +std::string collection_name = "testing_collection"; // Target collection name +std::string coll_key = "collection::doc1"; // Document key +std::string coll_value = R"({"collection_operation": "add", "scope": "custom"})"; // Document value + +// Collection-scoped ADD (key, value, collection_name) +brpc::CouchbaseOperations::Result coll_add_result = + couchbase_ops.add(coll_key, coll_value, collection_name); + +// Collection-scoped GET (key, collection_name) +brpc::CouchbaseOperations::Result coll_get_result = + couchbase_ops.get(coll_key, collection_name); + +// Collection-scoped UPSERT (key, value, collection_name) +brpc::CouchbaseOperations::Result coll_upsert_result = + couchbase_ops.upsert(coll_key, coll_value, collection_name); + +// Collection-scoped DELETE (key, collection_name) +brpc::CouchbaseOperations::Result coll_delete_result = + couchbase_ops.delete_(coll_key, collection_name); +``` + +9. **Pipeline operations demo**: + - Begin pipeline and add multiple operations + - Execute batch operations in single network call + - Process results in order + - Collection-scoped pipeline operations + - Error handling and cleanup + +**Function Signatures:** +```cpp +// Pipeline management functions +bool beginPipeline(); // Start a new pipeline session + +bool pipelineRequest(operation_type op_type, // Operation type (ADD, UPSERT, GET, DELETE, etc.) + const string& key, // Document key/ID + const string& value = "", // Document value (empty for GET/DELETE operations) + string collection_name = "_default"); // Collection name (optional) + +vector executePipeline(); // Execute all queued operations and return results + +bool clearPipeline(); // Clear pipeline without executing (cleanup) + +// Pipeline status functions +bool isPipelineActive() const; // Check if pipeline is active +size_t getPipelineSize() const; // Get number of queued operations +``` + +**Usage Examples:** +```cpp +// Begin pipeline +if (!couchbase_ops.beginPipeline()) { + std::cout << "Failed to begin pipeline" << std::endl; + return -1; +} + +// Add multiple operations to pipeline +std::string pipeline_key1 = "pipeline::doc1"; +std::string pipeline_key2 = "pipeline::doc2"; +std::string pipeline_value1 = R"({"operation": "pipeline_add", "id": 1})"; +std::string pipeline_value2 = R"({"operation": "pipeline_upsert", "id": 2})"; + +bool pipeline_success = true; +// pipelineRequest(operation_type, key, value, collection_name) +pipeline_success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, pipeline_key1, pipeline_value1); +pipeline_success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::UPSERT, pipeline_key2, pipeline_value2); +pipeline_success &= couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, pipeline_key1); // Empty value for GET + +if (!pipeline_success) { + couchbase_ops.clearPipeline(); // Clean up on error + return -1; +} + +// Execute pipeline - returns results in same order as requests +std::vector pipeline_results = couchbase_ops.executePipeline(); + +// Process results +for (size_t i = 0; i < pipeline_results.size(); ++i) { + if (pipeline_results[i].success) { + std::cout << "Operation " << (i + 1) << " SUCCESS"; + if (!pipeline_results[i].value.empty()) { + std::cout << " - Value: " << pipeline_results[i].value; + } + std::cout << std::endl; + } else { + std::cout << "Operation " << (i + 1) << " FAILED: " + << pipeline_results[i].error_message << std::endl; + } +} + +// Collection-scoped pipeline operations +if (couchbase_ops.beginPipeline()) { + // pipelineRequest(operation_type, key, value, collection_name) + couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::ADD, "coll_pipeline::doc1", + R"({"collection_operation": "pipeline_add", "id": 1})", collection_name); + couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::GET, "coll_pipeline::doc1", "", collection_name); + auto coll_results = couchbase_ops.executePipeline(); +} +``` + +10. **Bucket switching** - Demonstrate changing bucket selection. + +**Function Signature:** +```cpp +// SELECTBUCKET operation - switch to a different bucket on the same server +Result selectBucket(const string& bucket_name); // Target bucket name to switch to +``` + +**Usage Example:** +```cpp +std::string bucket_name = "testing"; +std::cout << "Enter Couchbase bucket name: "; +std::cin >> bucket_name; + +// selectBucket(bucket_name) - switches to the specified bucket +brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.selectBucket(bucket_name); +if (!bucket_result.success) { + LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; + return -1; +} else { + std::cout << "Bucket Selection Successful" << std::endl; +} + +// Perform operations on new bucket +performOperations(couchbase_ops); +``` #### Multithreaded Example (`multithreaded_couchbase_client.cpp`) Demonstrates: -- **16 bthreads** (4 threads per bucket across 4 buckets) -- **Each thread creates its own `CouchbaseOperations` instance** -- **Independent authentication** per thread +- **20 bthreads** (5 threads per bucket across 4 buckets) +- **Multiple threading patterns**: Each thread can create its own instance or share instances - **Concurrent operations** across multiple buckets and collections +- **Thread-safe statistics tracking** for operations +- **Collection-scoped operations** across threads -Key difference from single-threaded: Each thread must authenticate independently. +**Global Configuration**: +```cpp +const int NUM_THREADS = 20; +const int THREADS_PER_BUCKET = 5; + +// Global config structure +struct { + std::string username = "Administrator"; + std::string password = "password"; + std::vector bucket_names = {"t0", "t1", "t2", "t3"}; +} g_config; + +// Thread statistics tracking +struct ThreadStats { + std::atomic operations_attempted{0}; + std::atomic operations_successful{0}; + std::atomic operations_failed{0}; +}; + +struct GlobalStats { + ThreadStats total; + std::vector per_thread_stats; + GlobalStats() : per_thread_stats(NUM_THREADS) {} +} g_stats; +``` + +**Thread Worker Function**: +```cpp +struct ThreadArgs { + int thread_id; + int bucket_id; + std::string bucket_name; + ThreadStats* stats; +}; + +void* thread_worker(void* arg) { + ThreadArgs* args = static_cast(arg); + + // Create CouchbaseOperations instance for this thread + brpc::CouchbaseOperations couchbase_ops; + + // Authentication with assigned bucket + brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticate( + g_config.username, g_config.password, "127.0.0.1:11210", args->bucket_name); + + // For SSL authentication: + // brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticateSSL( + // g_config.username, g_config.password, "127.0.0.1:11207", args->bucket_name, "/path/to/cert.txt"); + + if (!auth_result.success) { + std::cout << "Thread " << args->thread_id << ": Auth failed - " + << auth_result.error_message << std::endl; + return NULL; + } + + // Perform CRUD operations on default collection + std::string base_key = "thread_" + std::to_string(args->thread_id); + perform_crud_operations_default(couchbase_ops, base_key, args->stats); + + // Perform collection-scoped operations + perform_crud_operations_collection(couchbase_ops, base_key, "my_collection", args->stats); + + return NULL; +} +``` + +**CRUD Operations Functions**: +```cpp +void perform_crud_operations_default(brpc::CouchbaseOperations& couchbase_ops, + const std::string& base_key, ThreadStats* stats) { + std::string key = base_key + "_default"; + std::string value = R"({"thread_id": %d, "collection": "default"})"; + + stats->operations_attempted++; + + // UPSERT operation + brpc::CouchbaseOperations::Result result = couchbase_ops.upsert(key, value); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + } + + // GET operation + stats->operations_attempted++; + result = couchbase_ops.get(key); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + } + + // DELETE operation + stats->operations_attempted++; + result = couchbase_ops.delete_(key); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + } +} + +void perform_crud_operations_collection(brpc::CouchbaseOperations& couchbase_ops, + const std::string& base_key, + const std::string& collection_name, + ThreadStats* stats) { + std::string key = base_key + "_collection"; + std::string value = R"({"thread_id": %d, "collection": ")" + collection_name + R"("})"; + + // Collection-scoped operations + stats->operations_attempted++; + brpc::CouchbaseOperations::Result result = couchbase_ops.upsert(key, value, collection_name); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + } + + stats->operations_attempted++; + result = couchbase_ops.get(key, collection_name); + if (result.success) { + stats->operations_successful++; + } else { + stats->operations_failed++; + } +} +``` + +**Main Function - Thread Management**: +```cpp +int main(int argc, char* argv[]) { + std::vector threads(NUM_THREADS); + std::vector thread_args(NUM_THREADS); + + // Create threads - 5 threads per bucket across 4 buckets + for (int i = 0; i < NUM_THREADS; ++i) { + thread_args[i].thread_id = i; + thread_args[i].bucket_id = i / THREADS_PER_BUCKET; + thread_args[i].bucket_name = g_config.bucket_names[thread_args[i].bucket_id]; + thread_args[i].stats = &g_stats.per_thread_stats[i]; + + if (bthread_start_background(&threads[i], NULL, thread_worker, &thread_args[i]) != 0) { + LOG(ERROR) << "Failed to create thread " << i; + return -1; + } + } + + // Wait for all threads to complete + for (int i = 0; i < NUM_THREADS; ++i) { + bthread_join(threads[i], NULL); + } + + // Aggregate and display statistics + g_stats.aggregate_stats(); + std::cout << "Total operations attempted: " << g_stats.total.operations_attempted.load() << std::endl; + std::cout << "Total operations successful: " << g_stats.total.operations_successful.load() << std::endl; + std::cout << "Total operations failed: " << g_stats.total.operations_failed.load() << std::endl; + + return 0; +} +``` + +**Alternative Pattern - Shared Instance Demo**: +```cpp +// Shared instance worker function +void* shared_object_thread_worker(void *arg) { + ThreadArgs* shared_args = static_cast(arg); + brpc::CouchbaseOperations* shared_couchbase_ops = shared_args->couchbase_ops; + + // Perform operations - 10 times on default collection, 10 times on col1 collection + for (int i = 0; i < 10; ++i) { + std::string base_key = butil::string_printf("shared_thread_op_%d_thread_id_%d", + i, shared_args->thread_id); + + // CRUD operations on default collection using shared instance + perform_crud_operations_default(*shared_couchbase_ops, base_key, shared_args->stats); + + // CRUD operations on col1 collection using shared instance + perform_crud_operations_col1(*shared_couchbase_ops, base_key, shared_args->stats); + + // Small delay between operations + bthread_usleep(10000); // 10ms + } + return NULL; +} + +// Main function demonstrates shared instance pattern +int main_shared_demo() { + // Create a shared CouchbaseOperations instance + brpc::CouchbaseOperations shared_couchbase_ops; + brpc::CouchbaseOperations::Result result; + + // Authenticate shared instance + result = shared_couchbase_ops.authenticate( + g_config.username, g_config.password, "127.0.0.1:11210", "t0"); + + if (result.success) { + std::cout << GREEN << "Shared CouchbaseOperations instance authenticated successfully!" + << RESET << std::endl; + } else { + std::cout << RED << "Shared CouchbaseOperations instance authentication failed: " + << result.error_message << RESET << std::endl; + return -1; + } + + // Configure all threads to use the shared instance + std::vector threads(NUM_THREADS); + std::vector args(NUM_THREADS); + + for (int i = 0; i < NUM_THREADS; ++i) { + args[i].thread_id = i; + args[i].couchbase_ops = &shared_couchbase_ops; // Point to shared instance + args[i].bucket_id = 0; + args[i].bucket_name = "t0"; // All threads use same bucket via shared instance + args[i].stats = &g_stats.per_thread_stats[i]; + } + + // Start all threads using shared instance + for (int i = 0; i < NUM_THREADS; ++i) { + if (bthread_start_background(&threads[i], NULL, shared_object_thread_worker, &args[i]) != 0) { + std::cout << RED << "Failed to create shared object thread " << i << RESET << std::endl; + return -1; + } + } + + // Wait for all threads to complete + for (int i = 0; i < NUM_THREADS; ++i) { + bthread_join(threads[i], NULL); + } + + std::cout << GREEN << "All shared object threads completed!" << RESET << std::endl; + return 0; +} +``` + +Key features: +- Demonstrates different connection patterns for multithreaded scenarios +- Shows concurrent access to different buckets and collections +- Proper resource management in multithreaded environments +- Statistics tracking across all threads +- Both separate instance and shared instance patterns --- ### 7. Building and Running the Examples @@ -302,39 +811,9 @@ make #### Run Multithreaded Example: ```bash -./multithreaded_couchbase_client --operations_per_thread=20 --sleep_ms=100 -``` - -#### Interactive Prompts: -Both examples will prompt for: -``` -Enter Couchbase username: your_username -Enter Couchbase password: ******** -Enter Couchbase bucket name: your_bucket +./multithreaded_couchbase_client ``` -For multithreaded example, additional prompts: -``` -Enter 4 bucket names: -Bucket 1: bucket1 -Bucket 2: bucket2 -Bucket 3: bucket3 -Bucket 4: bucket4 -Number of collections (0 for none): 2 -Collection 1: collection1 -Collection 2: collection2 -``` - -#### SSL Configuration: -- **Local Couchbase**: SSL is optional, set `enable_ssl = false` -- **Couchbase Capella**: SSL is **required**, download the certificate from Capella console -- Update the server address in the code or use command line flags: - ```bash - ./couchbase_client --server="your-cluster.cloud.couchbase.com:11207" - ``` - -Ensure buckets/collections exist before testing collection‑scoped operations. - --- ### 8. Setting Up Couchbase @@ -355,8 +834,8 @@ Create collections (7.0+): **SSL Configuration (Optional for Local)**: ```cpp -// Local without SSL -auto result = couchbase_ops.authenticate(username, password, "localhost:11210", false, ""); +// Local without SSL - authenticate with bucket selection +auto result = couchbase_ops.authenticate(username, password, "localhost:11210", bucket_name); ``` #### B. Couchbase Capella (Cloud) - **SSL Required** @@ -376,11 +855,11 @@ auto result = couchbase_ops.authenticate(username, password, "localhost:11210", **Capella SSL Authentication Example**: ```cpp // Couchbase Capella - SSL is MANDATORY -auto result = couchbase_ops.authenticate( +auto result = couchbase_ops.authenticateSSL( "your_username", "your_password", "your-cluster.cloud.couchbase.com:11207", // SSL port - true, // enable_ssl = true + "your_bucket_name", // bucket name "couchbase-cloud-cert.pem" // certificate file ); ``` @@ -403,6 +882,7 @@ struct Result { bool success; // true if operation succeeded string error_message; // human-readable error description string value; // returned value (for Get operations) + uint16_t status_code; // Couchbase status code (0x00 if success) }; ``` @@ -411,29 +891,31 @@ struct Result { auto result = couchbase_ops.add("key", "value"); if (!result.success) { LOG(ERROR) << "Add failed: " << result.error_message; + LOG(ERROR) << "Status code: " << result.status_code; // Handle error appropriately } else { std::cout << "Add succeeded!" << std::endl; } // For Get operations, check both success and value -auto get_result = couchbase_ops.Get("key"); +auto get_result = couchbase_ops.get("key"); if (get_result.success) { std::cout << "Retrieved: " << get_result.value << std::endl; } else { LOG(ERROR) << "Get failed: " << get_result.error_message; + LOG(ERROR) << "Status code: " << get_result.status_code; } ``` --- ### 10. Best Practices -#### Thread Safety -> ⚠️ **: THREAD SAFETY REQUIREMENTS** -> - **Each thread MUST create its own `CouchbaseOperations` instance** -> - **Each instance MUST authenticate independently** -> - **NEVER share `CouchbaseOperations` objects between threads** -> - **Sharing instances will cause race conditions, data corruption, and crashes** +#### Threading Patterns +> **💡 FLEXIBLE THREADING OPTIONS** +> - **Same bucket/server**: Share a single `CouchbaseOperations` instance across threads +> - **Different buckets**: Create separate instances for each bucket within the same server +> - **Different servers**: Create separate instances for each server connection +> - **Connection isolation**: Each instance uses unique connection groups based on server+bucket combination #### SSL Security - **Always use SSL for Couchbase Capella** (cloud deployments) @@ -446,80 +928,82 @@ if (get_result.success) { - **Use pipeline operations for bulk operations** - **Pipeline operations preserve order** - results correspond to request order -#### Code Example Template +#### Threading Examples ```cpp -#include +// Option 1: Shared instance for same bucket +brpc::CouchbaseOperations shared_ops; +shared_ops.authenticate(username, password, server_address, bucket_name); -int main() { - brpc::CouchbaseOperations couchbase_ops; - - // Authenticate (adjust SSL settings as needed) - auto auth_result = couchbase_ops.Authenticate( - username, password, server_address, enable_ssl, cert_path); - if (!auth_result.success) { - LOG(ERROR) << "Authentication failed: " << auth_result.error_message; - return -1; - } - - // Select bucket - auto bucket_result = couchbase_ops.selectBucket(bucket_name); - if (!bucket_result.success) { - LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message; - return -1; - } - - // Perform operations with error handling - auto result = couchbase_ops.add("key", "value", "collection_name"); - if (result.success) { - std::cout << "Success!" << std::endl; - } else { - LOG(ERROR) << "Operation failed: " << result.error_message; - } - - return 0; +void worker_thread_1() { + shared_ops.add("key1", "value1"); // Safe to share +} +void worker_thread_2() { + shared_ops.get("key2"); // Safe to share } + +// Option 2: Separate instances for different buckets +brpc::CouchbaseOperations ops_bucket1; +brpc::CouchbaseOperations ops_bucket2; +ops_bucket1.authenticate(username, password, server_address, "bucket1"); +ops_bucket2.authenticate(username, password, server_address, "bucket2"); + +// Option 3: Separate instances for different servers +brpc::CouchbaseOperations ops_server1; +brpc::CouchbaseOperations ops_server2; +ops_server1.authenticate(username, password, "server1:11210", bucket_name); +ops_server2.authenticate(username, password, "server2:11210", bucket_name); ``` --- ### 11. Summary and References -This implementation provides high-level APIs for Couchbase KV and collection operations. Couchbase (the company) contributed to this implementation, but it is not officially supported; it is "[Community Supported](https://docs.couchbase.com/server/current/third-party/integrations.html#support-model)". - -- **High-level API**: Recommended for most applications - simple, with built-in SSL support -- **SSL Support**: Essential for Couchbase Capella and secure local deployments -- **Thread Safety**: Each thread should create its own authenticated `CouchbaseOperations` instance -- **Collection Support**: Native support for collection-scoped operations - +This implementation provides high-level APIs for Couchbase KV operations. Couchbase (the company) contributed to this implementation, but it is not officially supported; it is "[Community Supported](https://docs.couchbase.com/server/current/third-party/integrations.html#support-model)". --- -## ⚠️ **CRITICAL THREAD SAFETY WARNING** ⚠️ - -> **🚨 NEVER SHARE `CouchbaseOperations` INSTANCES BETWEEN THREADS! 🚨** -> -> **Each thread MUST create its own `CouchbaseOperations` instance.** -> ->**Each thread can have multiple `CouchbaseOperations` instances.** +## 💡 **THREADING USAGE PATTERNS** 💡 > -> **For thread safe design please use couchbase-cxx-SDK version of bRPC, While it does not leverage many of the bRPC features around memory management and IO, it does provide a more complete set of Couchbase features and may be useful to those who have apps using bRPC with either memcached binprot or Couchbase and need some of the additional services and can be accessed at [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc).** -> -> **✅ CORRECT:** +> **✅ PATTERN 1: Shared instance when multiple threads operating on the same bucket** > ```cpp -> // Each thread creates its own instance -> void worker_thread() { -> brpc::CouchbaseOperations ops; // ✅ Thread-local instance -> ops.authenticate(...); -> ops.get("key"); // Safe +> brpc::CouchbaseOperations shared_ops; +> shared_ops.authenticate(username, password, "server:11210", "my_bucket"); +> +> void worker_thread_1() { +> shared_ops.add("key1", "value1"); // ✅ Safe to share +> } +> void worker_thread_2() { +> shared_ops.get("key2"); // ✅ Safe to share > } > ``` > -> **❌ WRONG - WILL CAUSE CRASHES:** +> **✅ PATTERN 2: Separate instances when different threads will be operating on different buckets** > ```cpp -> brpc::CouchbaseOperations global_ops; // ❌ Shared instance -> void worker_thread() { -> global_ops.get("key"); // ❌ RACE CONDITION - WILL CRASH! +> void worker_thread1() { +> brpc::CouchbaseOperations ops_bucket1; +> ops_bucket1.authenticate(username, password, "server:11210", "bucket1"); +> ops_bucket1.add("key1", "value1"); +> } +> void worker_thread2() { +> brpc::CouchbaseOperations ops_bucket2; +> ops_bucket2.authenticate(username, password, "server:11210", "bucket2"); +> ops_bucket2.add("key1", "value1"); > } > ``` > -> **Why?** `CouchbaseOperations` contains mutable state (pipeline queues, buffers, connection state) that is NOT thread-safe. Sharing instances will cause data corruption, pipeline interference, and application crashes. +> **✅ PATTERN 3: Separate instances when threads are operating on different servers.** +> ```cpp +> void worker_thread1() { +> brpc::CouchbaseOperations ops_bucket1; +> ops_server1.authenticate(username, password, "server1:11210", "bucket1"); +> ops_server1.add("key1", "value1"); +> } +> void worker_thread2() { +> brpc::CouchbaseOperations ops_server2; +> ops_server2.authenticate(username, password, "server2:11210", "bucket2"); +> ops_server2.add("key1", "value1"); +> } +> ``` +> +> **For additional Couchbase features, consider the couchbase-cxx-SDK version of bRPC, which provides a more complete set of Couchbase features and can be accessed at [Couchbaselabs-cb-brpc](https://github.com/couchbaselabs/cb_brpc/tree/couchbase_sdk_brpc).** + Contributions and issue reports are welcome! From e4242b966b24128d72ad0f71e7c3deb6b5078bdf Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 5 Nov 2025 10:56:04 +0530 Subject: [PATCH 43/49] removed commented code and updated readme to have links for cluster download certificate --- docs/en/couchbase_example.md | 4 ++-- src/brpc/policy/couchbase_protocol.cpp | 11 ----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 55ed46adac..8ad2728b4a 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -117,9 +117,9 @@ if (add_result.success) { // Optional: Switch to a different bucket (if needed) // brpc::CouchbaseOperations::Result bucket_result = couchbase_ops.selectBucket("another_bucket"); ``` -``` #### SSL Authentication (Essential for Couchbase Capella): +To know how to download the security certificate [click here](https://docs.couchbase.com/cloud/security/security-certificates.html). ```cpp // For Couchbase Capella (cloud) - SSL is REQUIRED brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticateSSL( @@ -127,7 +127,7 @@ brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticateSSL( password, "cluster.cloud.couchbase.com:11207", // SSL port bucket_name, // bucket name - "path/to/certificate.txt" // certificate path + "path/to/certificate.txt" // certificate path(can be downloaded from capella UI) ); ``` diff --git a/src/brpc/policy/couchbase_protocol.cpp b/src/brpc/policy/couchbase_protocol.cpp index ab2d3c3efb..d40c950aa2 100644 --- a/src/brpc/policy/couchbase_protocol.cpp +++ b/src/brpc/policy/couchbase_protocol.cpp @@ -134,16 +134,6 @@ ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket* socket, msg->meta.append(&local_header, sizeof(local_header)); source->pop_front(sizeof(*header)); source->cutn(&msg->meta, total_body_length); - // if (header->command == CB_BINARY_SASL_AUTH) { - // if (header->status != 0) { - // LOG(ERROR) << "Failed to authenticate the couchbase Server."; - // return MakeParseError(PARSE_ERROR_NO_RESOURCE, - // "Fail to authenticate with the couchbase Server"); - // } - // msg = static_cast(socket->release_parsing_context()); - // msg->pi = pi; - // return MakeMessage(msg); - // } else { if (++msg->pi.count >= pi.count) { CHECK_EQ(msg->pi.count, pi.count); msg = @@ -153,7 +143,6 @@ ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket* socket, } else { socket->GivebackPipelinedInfo(pi); } - // } } } From 90d5522063bd6622925b3617e4b9cd7395fbfd08 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 5 Nov 2025 11:37:23 +0530 Subject: [PATCH 44/49] removed unused code. --- src/brpc/couchbase.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index e684e81e79..cf11a9c01b 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -19,8 +19,6 @@ #include //for crc32 Vbucket_id -#define CB_ADD(a,b) (a+b) - // Debug flag for enabling debug statements static bool DBUG = false; // Set to true to enable debug logs From c1c76f12e952b82e664e7de60936bed0c838bbe4 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 10 Nov 2025 16:19:45 +0530 Subject: [PATCH 45/49] Added traditional bRPC coding approach Traditional bRPC coding approach doesn't uses high level functions but provides more control to the user fixed formatting issues. fixed the bug in couchbase.cpp where logic to check the cache is empty was inverted --- example/couchbase_c++/CMakeLists.txt | 130 ----- example/couchbase_c++/Makefile | 17 +- example/couchbase_c++/couchbase_client.cpp | 23 +- .../traditional_brpc_couchbase_client.cpp | 171 ++++++ src/brpc/couchbase.cpp | 504 +++--------------- src/brpc/couchbase.h | 35 +- src/brpc/policy/couchbase_protocol.cpp | 18 +- src/brpc/policy/couchbase_protocol.h | 300 +++++------ 8 files changed, 446 insertions(+), 752 deletions(-) delete mode 100644 example/couchbase_c++/CMakeLists.txt create mode 100644 example/couchbase_c++/traditional_brpc_couchbase_client.cpp diff --git a/example/couchbase_c++/CMakeLists.txt b/example/couchbase_c++/CMakeLists.txt deleted file mode 100644 index b0dbaee803..0000000000 --- a/example/couchbase_c++/CMakeLists.txt +++ /dev/null @@ -1,130 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -cmake_minimum_required(VERSION 2.8.10) -project(couchbase_c++ C CXX) - -option(LINK_SO "Whether examples are linked dynamically" OFF) - -execute_process( - COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'" - OUTPUT_VARIABLE OUTPUT_PATH -) - -set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) - -include(FindThreads) -include(FindProtobuf) -protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto) -# include PROTO_HEADER -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -# Search for libthrift* by best effort. If it is not found and brpc is -# compiled with thrift protocol enabled, a link error would be reported. -find_library(THRIFT_LIB NAMES thrift) -if (NOT THRIFT_LIB) - set(THRIFT_LIB "") -endif() - -find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) -if(LINK_SO) - find_library(BRPC_LIB NAMES brpc) -else() - find_library(BRPC_LIB NAMES libbrpc.a brpc) -endif() -if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) - message(FATAL_ERROR "Fail to find brpc") -endif() -include_directories(${BRPC_INCLUDE_PATH}) - -find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) -find_library(GFLAGS_LIBRARY NAMES gflags libgflags) -if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) - message(FATAL_ERROR "Fail to find gflags") -endif() -include_directories(${GFLAGS_INCLUDE_PATH}) - -if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - include(CheckFunctionExists) - CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME) - if(NOT HAVE_CLOCK_GETTIME) - set(DEFINE_CLOCK_GETTIME "-DNO_CLOCK_GETTIME_IN_MAC") - endif() -endif() - -set(CMAKE_CXX_FLAGS "${DEFINE_CLOCK_GETTIME} -DNDEBUG -O2 -D__const__=__unused__ -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") - -if(CMAKE_VERSION VERSION_LESS "3.1.3") - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") - endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") - endif() -else() - set(CMAKE_CXX_STANDARD 17) - set(CMAKE_CXX_STANDARD_REQUIRED ON) -endif() - -find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) -find_library(LEVELDB_LIB NAMES leveldb) -if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) - message(FATAL_ERROR "Fail to find leveldb") -endif() -include_directories(${LEVELDB_INCLUDE_PATH}) - -if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - set(OPENSSL_ROOT_DIR - "/usr/local/opt/openssl" # Homebrew installed OpenSSL - ) -endif() - -find_package(OpenSSL) -include_directories(${OPENSSL_INCLUDE_DIR}) - -set(DYNAMIC_LIB - ${CMAKE_THREAD_LIBS_INIT} - ${GFLAGS_LIBRARY} - ${PROTOBUF_LIBRARIES} - ${LEVELDB_LIB} - ${OPENSSL_CRYPTO_LIBRARY} - ${OPENSSL_SSL_LIBRARY} - ${THRIFT_LIB} - dl - ) - -if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - set(DYNAMIC_LIB ${DYNAMIC_LIB} - pthread - "-framework CoreFoundation" - "-framework CoreGraphics" - "-framework CoreData" - "-framework CoreText" - "-framework Security" - "-framework Foundation" - "-Wl,-U,_MallocExtension_ReleaseFreeMemory" - "-Wl,-U,_ProfilerStart" - "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii" - "-Wl,-U,_mallctl" - "-Wl,-U,_malloc_stats_print" - ) -endif() - -add_executable(couchbase_client couchbase_client.cpp) - -target_link_libraries(couchbase_client ${BRPC_LIB} ${DYNAMIC_LIB}) diff --git a/example/couchbase_c++/Makefile b/example/couchbase_c++/Makefile index 75052b5763..f41e4b6b72 100644 --- a/example/couchbase_c++/Makefile +++ b/example/couchbase_c++/Makefile @@ -26,10 +26,11 @@ COMMA=, SOPATHS=$(addprefix -Wl$(COMMA)-rpath$(COMMA), $(LIBS)) # Define targets and their sources -TARGETS = couchbase_client multithreaded_couchbase_client +TARGETS = couchbase_client multithreaded_couchbase_client traditional_brpc_couchbase_client COUCHBASE_CLIENT_OBJS = couchbase_client.o MULTITHREADED_CLIENT_OBJS = multithreaded_couchbase_client.o -ALL_OBJS = $(COUCHBASE_CLIENT_OBJS) $(MULTITHREADED_CLIENT_OBJS) +TRADITIONAL_CLIENT_OBJS = traditional_brpc_couchbase_client.o +ALL_OBJS = $(COUCHBASE_CLIENT_OBJS) $(MULTITHREADED_CLIENT_OBJS) $(TRADITIONAL_CLIENT_OBJS) ifeq ($(SYSTEM),Darwin) ifneq ("$(LINK_SO)", "") @@ -72,6 +73,14 @@ else $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ endif +traditional_brpc_couchbase_client: $(TRADITIONAL_CLIENT_OBJS) + @echo "> Linking $@" +ifneq ("$(LINK_SO)", "") + $(CXX) $(LIBPATHS) $(SOPATHS) $(LINK_OPTIONS_SO) -o $@ +else + $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ +endif + # Compilation rules couchbase_client.o: couchbase_client.cpp @echo "> Compiling $@" @@ -80,3 +89,7 @@ couchbase_client.o: couchbase_client.cpp multithreaded_couchbase_client.o: multithreaded_couchbase_client.cpp @echo "> Compiling $@" $(CXX) -c $(HDRPATHS) $(CXXFLAGS) $< -o $@ + +traditional_brpc_couchbase_client.o: traditional_brpc_couchbase_client.cpp + @echo "> Compiling $@" + $(CXX) -c $(HDRPATHS) $(CXXFLAGS) $< -o $@ \ No newline at end of file diff --git a/example/couchbase_c++/couchbase_client.cpp b/example/couchbase_c++/couchbase_client.cpp index a3dbca6842..b1dc90635c 100644 --- a/example/couchbase_c++/couchbase_client.cpp +++ b/example/couchbase_c++/couchbase_client.cpp @@ -28,8 +28,8 @@ #define RESET "\033[0m" DEFINE_string(server, "localhost:11210", "IP Address of server"); -int performOperations(brpc::CouchbaseOperations& couchbase_ops){ - std::string add_key = "user::test_brpc_binprot"; +int performOperations(brpc::CouchbaseOperations& couchbase_ops) { + std::string add_key = "user::test_brpc_binprot"; std::string add_value = R"({"name": "John Doe", "age": 30, "email": "john@example.com"})"; @@ -167,16 +167,9 @@ int performOperations(brpc::CouchbaseOperations& couchbase_ops){ } // Retrieve Collection ID for scope `_default` and collection - // `testing_collection` - const std::string scope_name = "_default"; // default scope - std::string collection_name = "col1"; // target collection - // enter collection name as user input - // std::cout << "Enter collection name (default 'testing_collection'): "; - // std::string user_input; - // std::cin >> user_input; - // if (!user_input.empty()) { - // collection_name = user_input; - // } + // `col1` + const std::string scope_name = "_default"; // default scope + std::string collection_name = "col1"; // target collection // ------------------------------------------------------------------ // Collection-scoped CRUD operations (only if collection id was retrieved) // ------------------------------------------------------------------ @@ -192,9 +185,6 @@ int performOperations(brpc::CouchbaseOperations& couchbase_ops){ << "Collection ADD failed: " << coll_add_result.error_message << RESET << std::endl; } - // Prompt user to delete the collection to test collection operations - // std::cout << "Delete & re-create the collection."; - // std::cin >> collection_name; // 2. GET from collection using high-level method brpc::CouchbaseOperations::Result coll_get_result = couchbase_ops.get(coll_key, collection_name); @@ -455,8 +445,7 @@ int main() { } else { std::cout << GREEN << "Bucket Selection Successful" << RESET << std::endl; } - performOperations(couchbase_ops); // Add operation using high-level method - + performOperations(couchbase_ops); return 0; } diff --git a/example/couchbase_c++/traditional_brpc_couchbase_client.cpp b/example/couchbase_c++/traditional_brpc_couchbase_client.cpp new file mode 100644 index 0000000000..c63c98cd4f --- /dev/null +++ b/example/couchbase_c++/traditional_brpc_couchbase_client.cpp @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +// ANSI color codes for console output +#define GREEN "\033[32m" +#define RED "\033[31m" +#define RESET "\033[0m" + +int main() { + // traditional bRPC Couchbase client + brpc::Channel channel; + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_COUCHBASE; + options.connection_type = "single"; + options.timeout_ms = 1000; // 1 second + options.max_retry = 3; + if (channel.Init("localhost:11210", &options) != 0) { + LOG(ERROR) << "Failed to initialize channel"; + return -1; + } + brpc::Controller cntl; + brpc::CouchbaseOperations::CouchbaseRequest req; + brpc::CouchbaseOperations::CouchbaseResponse res; + uint64_t cas; + req.authenticateRequest("Administrator", "password"); + channel.CallMethod(NULL, &cntl, &req, &res, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Unable to authenticate: Something went wrong" + << cntl.ErrorText(); + return -1; + } else { + if (res.popHello(&cas) && res.popAuthenticate(&cas)) { + std::cout << "Traditional bRPC Couchbase Client Authentication Successful" + << std::endl; + } else { + std::cout << "Client Authentication Failed with status code: " << std::hex + << res._status_code << std::endl; + return -1; + } + } + cntl.Reset(); + // clearing request and response + + req.Clear(); + res.Clear(); + req.selectBucketRequest("testing"); + channel.CallMethod(NULL, &cntl, &req, &res, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Unable to select bucket: Something went wrong" + << cntl.ErrorText(); + return -1; + } else { + if (res.popSelectBucket(&cas)) { + std::cout + << "Traditional bRPC Couchbase Client Bucket Selection Successful" + << std::endl; + } else { + // the status code will be updated only after you do + // popFunctionName(param). + std::cout << "Traditional bRPC Couchbase Client Bucket Selection Failed " + "with status code: " + << std::hex << res._status_code << std::endl; + std::cout << "Error Message: " << res.lastError() << std::endl; + return -1; + } + } + cntl.Reset(); + // clearing request and response + + req.Clear(); + res.Clear(); + req.addRequest( + "sample_key", + R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", + 0 /*flags*/, 0 /*exptime*/, 0 /*cas*/); + channel.CallMethod(NULL, &cntl, &req, &res, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Unable to add key-value: Something went wrong" + << cntl.ErrorText(); + return -1; + } else { + if (res.popAdd(&cas)) { + std::cout + << "Traditional bRPC Couchbase Client Key-Value Addition Successful" + << std::endl; + } else { + // the status code will be updated only after you do + // popFunctionName(param). + std::cout << "Traditional bRPC Couchbase Client Key-Value Addition " + "Failed with status code: " + << std::hex << res._status_code << std::endl; + std::cout << "Error Message: " << res.lastError() << std::endl; + return -1; + } + } + cntl.Reset(); + + // clearing request and response before doing a getRequest + req.Clear(); + res.Clear(); + req.getRequest("sample_key"); + channel.CallMethod(NULL, &cntl, &req, &res, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Unable to get value for key: Something went wrong" + << cntl.ErrorText(); + return -1; + } else { + std::string value; + uint32_t flags; + if (res.popGet(&value, &flags, &cas)) { + std::cout + << "Traditional bRPC Couchbase Client Key-Value Retrieval Successful" + << std::endl; + std::cout << "Retrieved Value: " << value << std::endl; + } else { + // note the status code will be updated only after you do + // popFunctionName(param). + std::cout << "Traditional bRPC Couchbase Client Key-Value Retrieval " + "Failed with status code: " + << std::hex << res._status_code << std::endl; + std::cout << "Error Message: " << res.lastError() << std::endl; + return -1; + } + } + cntl.Reset(); + // clearing request and response before doing a deleteRequest + + req.Clear(); + res.Clear(); + req.deleteRequest("sample_key"); + channel.CallMethod(NULL, &cntl, &req, &res, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Unable to delete key-value: Something went wrong" + << cntl.ErrorText(); + return -1; + } else { + if (res.popDelete()) { + std::cout + << "Traditional bRPC Couchbase Client Key-Value Deletion Successful" + << std::endl; + } else { + // the status code will be updated only after you do + // popFunctionName(param). + std::cout << "Traditional bRPC Couchbase Client Key-Value Deletion " + "Failed with status code: " + << std::hex << res._status_code << std::endl; + std::cout << "Error Message: " << res.lastError() << std::endl; + return -1; + } + } + return 0; +} \ No newline at end of file diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index cf11a9c01b..3e915ccdad 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -238,7 +238,8 @@ bool CouchbaseManifestManager::jsonToCollectionManifest( bool CouchbaseManifestManager::refreshCollectionManifest( brpc::Channel* channel, const string& server, const string& bucket, - unordered_map* local_collection_manifest_cache) { + unordered_map* + local_collection_manifest_cache) { // first fetch the manifest // then compare the UID with the cached one if (channel == nullptr) { @@ -294,9 +295,9 @@ bool CouchbaseManifestManager::refreshCollectionManifest( } return true; } - // Compare the UID with the cached one - // If they are different, refresh the cache - else if (manifest.uid != cached_manifest.uid) { + // Compare the UID with the cached one + // If they are different, refresh the cache + else if (manifest.uid != cached_manifest.uid) { DEBUG_PRINT("Collection manifest has changed for bucket " << bucket << " on server " << server); if (!common_metadata_tracking.setBucketToCollectionManifest(server, bucket, @@ -629,7 +630,6 @@ void CouchbaseOperations::CouchbaseResponse::setCachedSize(int size) const { void CouchbaseOperations::CouchbaseResponse::Clear() {} - void CouchbaseOperations::CouchbaseResponse::MergeFrom( const CouchbaseResponse& from) { CHECK_NE(&from, this); @@ -793,11 +793,9 @@ bool CouchbaseOperations::CouchbaseRequest::getOrDelete( uint16_t VBucket_id = hashCrc32(key.data(), key.size()); const policy::CouchbaseRequestHeader header = { policy::CB_MAGIC_REQUEST, command, - butil::HostToNet16( - key.size() + - 1), // Key - 0, // extras length - policy::CB_BINARY_RAW_BYTES, // data type + butil::HostToNet16(key.size() + 1), // Key + 0, // extras length + policy::CB_BINARY_RAW_BYTES, // data type butil::HostToNet16(VBucket_id), butil::HostToNet32(key.size() + sizeof(collection_id)), // total body length includes @@ -822,7 +820,8 @@ bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( string collection_name, uint8_t* coll_id, brpc::CouchbaseManifestManager* metadata_tracking, brpc::Channel* channel, const string& server, const string& selected_bucket, - unordered_map* local_cache) { + unordered_map* + local_cache) { if (collection_name.empty()) { DEBUG_PRINT("Empty collection name"); return false; @@ -848,7 +847,8 @@ bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( << selected_bucket << " on server " << server << ", fetching from server"); // No cached manifest found, fetch from server - if (!metadata_tracking->refreshCollectionManifest(channel, server, selected_bucket, local_cache)) { + if (!metadata_tracking->refreshCollectionManifest( + channel, server, selected_bucket, local_cache)) { return false; } // local cache will also be updated in refreshCollectionManifest @@ -867,7 +867,8 @@ bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( &manifest, "_default", collection_name, coll_id)) { // Just to verify that the collectionID does not exist in the manifest // refresh manifest from server and try again - if (!metadata_tracking->refreshCollectionManifest(channel, server, selected_bucket, local_cache)) { + if (!metadata_tracking->refreshCollectionManifest( + channel, server, selected_bucket, local_cache)) { return false; } // local cache will also be updated in refreshCollectionManifest @@ -900,9 +901,9 @@ bool CouchbaseOperations::CouchbaseRequest::getRequest( if (local_collection_manifest_cache->empty()) { DEBUG_PRINT("Local collection manifest cache is empty in getRequest"); // if local cache is empty, goto global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "getRequest"); @@ -914,9 +915,9 @@ bool CouchbaseOperations::CouchbaseRequest::getRequest( &coll_id)) { DEBUG_PRINT("Collection id not found in local cache in getRequest"); // if not check in the global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "getRequest"); @@ -940,9 +941,9 @@ bool CouchbaseOperations::CouchbaseRequest::deleteRequest( if (local_collection_manifest_cache->empty()) { DEBUG_PRINT("Local collection manifest cache is empty in deleteRequest"); // if local cache is empty, goto global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "deleteRequest"); @@ -954,9 +955,9 @@ bool CouchbaseOperations::CouchbaseRequest::deleteRequest( &coll_id)) { DEBUG_PRINT("Collection id not found in local cache in deleteRequest"); // if not check in the global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "deleteRequest"); @@ -974,51 +975,6 @@ struct FlushHeaderWithExtras { } __attribute__((packed)); BAIDU_CASSERT(sizeof(FlushHeaderWithExtras) == 28, must_match); -// MAY have extras. -// MUST NOT have key. -// MUST NOT have value. -// Extra data for flush: -// Byte/ 0 | 1 | 2 | 3 | -// / | | | | -// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| -// +---------------+---------------+---------------+---------------+ -// 0| Expiration | -// +---------------+---------------+---------------+---------------+ -// Total 4 bytes -// Warning: Not tested -// bool CouchbaseOperations::CouchbaseRequest::FlushRequest(uint32_t timeout) { -// const uint8_t FLUSH_EXTRAS = (timeout == 0 ? 0 : 4); -// FlushHeaderWithExtras header_with_extras = { -// {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_FLUSH, 0, FLUSH_EXTRAS, -// policy::CB_BINARY_RAW_BYTES, 0, butil::HostToNet32(FLUSH_EXTRAS), 0, -// 0}, -// butil::HostToNet32(timeout)}; -// if (FLUSH_EXTRAS == 0) { -// if (_buf.append(&header_with_extras.header, -// sizeof(policy::CouchbaseRequestHeader))) { -// return false; -// } -// } else { -// if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { -// return false; -// } -// } -// ++_pipelined_count; -// return true; -// } - -// (if found): -// MUST have extras. -// MAY have key. -// MAY have value. -// Extra data for the get commands: -// Byte/ 0 | 1 | 2 | 3 | -// / | | | | -// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| -// +---------------+---------------+---------------+---------------+ -// 0| Flags | -// +---------------+---------------+---------------+---------------+ -// Total 4 bytes bool CouchbaseOperations::CouchbaseResponse::popGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value) { @@ -1113,10 +1069,6 @@ bool CouchbaseOperations::CouchbaseResponse::popGet(std::string* value, bool CouchbaseOperations::CouchbaseResponse::popDelete() { return popStore(policy::CB_BINARY_DELETE, NULL); } -// Warning: Not tested -// bool CouchbaseOperations::CouchbaseResponse::PopFlush() { -// return popStore(policy::CB_BINARY_FLUSH, NULL); -// } struct StoreHeaderWithExtras { policy::CouchbaseRequestHeader header; @@ -1334,9 +1286,9 @@ bool CouchbaseOperations::CouchbaseRequest::upsertRequest( if (local_collection_manifest_cache->empty()) { DEBUG_PRINT("Local collection manifest cache is empty in upsertRequest"); // if local cache is empty, goto global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "upsertRequest"); @@ -1348,9 +1300,9 @@ bool CouchbaseOperations::CouchbaseRequest::upsertRequest( &coll_id)) { DEBUG_PRINT("Collection id not found in local cache in upsertRequest"); // if not check in the global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "upsertRequest"); @@ -1363,35 +1315,6 @@ bool CouchbaseOperations::CouchbaseRequest::upsertRequest( coll_id); } -// Using GetCollectionManifest instead of fetching collection ID directly -// bool CouchbaseOperations::CouchbaseRequest::GetCollectionId( -// const butil::StringPiece& scope_name, -// const butil::StringPiece& collection_name) { -// // Format the collection path as "scope.collection" -// std::string collection_path = -// scope_name.as_string() + "." + collection_name.as_string(); - -// const policy::CouchbaseRequestHeader header = { -// policy::CB_MAGIC_REQUEST, -// policy::CB_COLLECTIONS_GET_CID, -// butil::HostToNet16(collection_path.size()), -// 0, // no extras -// policy::CB_BINARY_RAW_BYTES, -// 0, // no vbucket -// butil::HostToNet32(collection_path.size()), -// 0, // opaque -// 0 // no CAS -// }; -// if (_buf.append(&header, sizeof(header))) { -// return false; -// } -// if (_buf.append(collection_path.data(), collection_path.size())) { -// return false; -// } -// ++_pipelined_count; -// return true; -// } - bool CouchbaseOperations::CouchbaseRequest::getCollectionManifest() { const policy::CouchbaseRequestHeader header = { policy::CB_MAGIC_REQUEST, @@ -1426,9 +1349,9 @@ bool CouchbaseOperations::CouchbaseRequest::addRequest( if (local_collection_manifest_cache->empty()) { DEBUG_PRINT("Local collection manifest cache is empty in addRequest"); // if local cache is empty, goto global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "addRequest"); @@ -1440,9 +1363,9 @@ bool CouchbaseOperations::CouchbaseRequest::addRequest( &coll_id)) { DEBUG_PRINT("Collection id not found in local cache in addRequest"); // if not check in the global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { DEBUG_PRINT( "Failed to get collection id from global cache or server in " "addRequest"); @@ -1455,26 +1378,6 @@ bool CouchbaseOperations::CouchbaseRequest::addRequest( coll_id); } -// Warning: Not tested -// bool CouchbaseOperations::CouchbaseRequest::ReplaceRequest(const -// butil::StringPiece& key, -// const butil::StringPiece& value, uint32_t -// flags, uint32_t exptime, uint64_t cas_value, -// string collection_name, -// brpc::Channel* channel, const string& server, -// const string& bucket) { -// uint8_t coll_id = 0; // default collection ID -// if(collection_name != "_default"){ -// if(!getCachedOrFetchCollectionId(collection_name, &coll_id, -// metadata_tracking, channel, server, bucket, local_collection_manifest_cache)){ -// return false; -// } -// } -// return Store(policy::CB_BINARY_REPLACE, key, value, flags, exptime, -// cas_value, -// coll_id); -// } - bool CouchbaseOperations::CouchbaseRequest::appendRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, @@ -1487,11 +1390,11 @@ bool CouchbaseOperations::CouchbaseRequest::appendRequest( uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. - if (!local_collection_manifest_cache->empty()) { + if (local_collection_manifest_cache->empty()) { // if local cache is empty, goto global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { return false; } } @@ -1499,9 +1402,9 @@ bool CouchbaseOperations::CouchbaseRequest::appendRequest( else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { // if not check in the global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { return false; } } @@ -1522,11 +1425,11 @@ bool CouchbaseOperations::CouchbaseRequest::prependRequest( uint8_t coll_id = 0; // default collection ID if (collection_name != "_default") { // check if the local cache is empty or not. - if (!local_collection_manifest_cache->empty()) { + if (local_collection_manifest_cache->empty()) { // if local cache is empty, goto global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { return false; } } @@ -1534,9 +1437,9 @@ bool CouchbaseOperations::CouchbaseRequest::prependRequest( else if (!getLocalCachedCollectionId(bucket, "_default", collection_name, &coll_id)) { // if not check in the global cache or fetch from server - if (!getCachedOrFetchCollectionId(collection_name, &coll_id, - metadata_tracking, channel, server, - bucket, local_collection_manifest_cache)) { + if (!getCachedOrFetchCollectionId( + collection_name, &coll_id, metadata_tracking, channel, server, + bucket, local_collection_manifest_cache)) { return false; } } @@ -1545,7 +1448,8 @@ bool CouchbaseOperations::CouchbaseRequest::prependRequest( coll_id); } -bool CouchbaseOperations::CouchbaseResponse::popAuthenticate(uint64_t* cas_value) { +bool CouchbaseOperations::CouchbaseResponse::popAuthenticate( + uint64_t* cas_value) { return popStore(policy::CB_BINARY_SASL_AUTH, cas_value); } bool CouchbaseOperations::CouchbaseResponse::popHello(uint64_t* cas_value) { @@ -1569,7 +1473,7 @@ bool CouchbaseOperations::CouchbaseResponse::popPrepend(uint64_t* cas_value) { return popStore(policy::CB_BINARY_PREPEND, cas_value); } bool CouchbaseOperations::CouchbaseResponse::popSelectBucket( - uint64_t* cas_value, std::string bucket_name) { + uint64_t* cas_value) { if (popStore(policy::CB_SELECT_BUCKET, cas_value) == false) { DEBUG_PRINT("Failed to select bucket: " << _err); return false; @@ -1761,31 +1665,6 @@ bool CouchbaseOperations::CouchbaseRequest::counter( return true; } -// Warning: Not tested -// bool CouchbaseOperations::CouchbaseRequest::IncrementRequest(const -// butil::StringPiece& key, uint64_t delta, -// uint64_t initial_value, uint32_t exptime, -// string collection_name, -// brpc::Channel* channel, const string& -// server, const string& bucket) { -// // Note: Counter method doesn't seem to use collection_name, may need to be -// updated if collection support is needed return -// Counter(policy::CB_BINARY_INCREMENT, key, delta, initial_value, -// exptime); -// } - -// bool CouchbaseOperations::CouchbaseRequest::DecrementRequest(const -// butil::StringPiece& key, uint64_t delta, -// uint64_t initial_value, uint32_t exptime, -// string collection_name, -// brpc::Channel* channel, const string& -// server, const string& bucket) { -// // Note: Counter method doesn't seem to use collection_name, may need to be -// updated if collection support is needed return -// Counter(policy::CB_BINARY_DECREMENT, key, delta, initial_value, -// exptime); -// } - // MUST NOT have extras. // MUST NOT have key. // MUST have value. @@ -1856,60 +1735,6 @@ bool CouchbaseOperations::CouchbaseResponse::popCounter(uint8_t command, return true; } -// Warning: Not tested -// bool CouchbaseOperations::CouchbaseResponse::PopIncrement(uint64_t* -// new_value, uint64_t* cas_value) { -// return popCounter(policy::CB_BINARY_INCREMENT, new_value, cas_value); -// } -// bool CouchbaseOperations::CouchbaseResponse::PopDecrement(uint64_t* -// new_value, uint64_t* cas_value) { -// return popCounter(policy::CB_BINARY_DECREMENT, new_value, cas_value); -// } - -// MUST have extras. -// MUST have key. -// MUST NOT have value. -// struct TouchHeaderWithExtras { -// policy::CouchbaseRequestHeader header; -// uint32_t exptime; -// } __attribute__((packed)); -// BAIDU_CASSERT(sizeof(TouchHeaderWithExtras) == 28, must_match); -// const size_t TOUCH_EXTRAS = -// sizeof(TouchHeaderWithExtras) - sizeof(policy::CouchbaseRequestHeader); - -// MAY have extras. -// MUST NOT have key. -// MUST NOT have value. -// Extra data for touch: -// Byte/ 0 | 1 | 2 | 3 | -// / | | | | -// |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7| -// +---------------+---------------+---------------+---------------+ -// 0| Expiration | -// +---------------+---------------+---------------+---------------+ -// Total 4 bytes -// Warning: Not tested -// bool CouchbaseOperations::CouchbaseRequest::TouchRequest(const -// butil::StringPiece& key, uint32_t exptime, -// string collection_name, -// brpc::Channel* channel, const string& server, -// const string& bucket) { -// TouchHeaderWithExtras header_with_extras = { -// {policy::CB_MAGIC_REQUEST, policy::CB_BINARY_TOUCH, -// butil::HostToNet16(key.size()), TOUCH_EXTRAS, -// policy::CB_BINARY_RAW_BYTES, 0, -// butil::HostToNet32(TOUCH_EXTRAS + key.size()), 0, 0}, -// butil::HostToNet32(exptime)}; -// if (_buf.append(&header_with_extras, sizeof(header_with_extras))) { -// return false; -// } -// if (_buf.append(key.data(), key.size())) { -// return false; -// } -// ++_pipelined_count; -// return true; -// } - // MUST NOT have extras. // MUST NOT have key. // MUST NOT have value. @@ -1930,14 +1755,6 @@ bool CouchbaseOperations::CouchbaseRequest::versionRequest() { return true; } -// MUST NOT have extras. -// MUST NOT have key. -// MUST have value. -// Warning: Not tested -// bool CouchbaseOperations::CouchbaseResponse::PopTouch() { -// return popStore(policy::CB_BINARY_TOUCH, NULL); -// } - bool CouchbaseOperations::CouchbaseResponse::popVersion(std::string* version) { const size_t n = _buf.size(); policy::CouchbaseResponseHeader header; @@ -2086,7 +1903,9 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, // has been updated on the server side. The collectionID present in the // local cache/global cache is no longer valid. This can happen if a // collection is deleted and recreated with the same name. - if (!request->metadata_tracking->refreshCollectionManifest(channel, server, bucket, request->local_collection_manifest_cache)) { + if (!request->metadata_tracking->refreshCollectionManifest( + channel, server, bucket, + request->local_collection_manifest_cache)) { DEBUG_PRINT("Failed to refresh collection manifest"); result->error_message = "Failed to refresh collection manifest"; } else { @@ -2173,7 +1992,9 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, // collection_manifest has been updated on the server side. and the // client have a stale copy of collection manifest. In this case, we // need to refresh the collection manifest and retry the operation. - if (!request->metadata_tracking->refreshCollectionManifest(channel, server, bucket, request->local_collection_manifest_cache)) { + if (!request->metadata_tracking->refreshCollectionManifest( + channel, server, bucket, + request->local_collection_manifest_cache)) { DEBUG_PRINT("Failed to refresh collection manifest"); result->error_message = "Failed to refresh collection manifest"; return false; @@ -2357,7 +2178,7 @@ CouchbaseOperations::Result CouchbaseOperations::authenticate( const string& username, const string& password, const string& server_address, const string& bucket_name) { return authenticateAll(username, password, server_address, bucket_name, false, - ""); + ""); } CouchbaseOperations::Result CouchbaseOperations::authenticateSSL( @@ -2365,21 +2186,23 @@ CouchbaseOperations::Result CouchbaseOperations::authenticateSSL( const string& server_address, const string& bucket_name, string path_to_cert) { return authenticateAll(username, password, server_address, bucket_name, true, - path_to_cert); + path_to_cert); } CouchbaseOperations::Result CouchbaseOperations::authenticateAll( const string& username, const string& password, - const string& server_address, const string& bucket_name, bool enable_ssl, string path_to_cert) { + const string& server_address, const string& bucket_name, bool enable_ssl, + string path_to_cert) { // Create a channel to the Couchbase server brpc::ChannelOptions options; options.protocol = brpc::PROTOCOL_COUCHBASE; options.connection_type = "single"; options.timeout_ms = 1000; // 1 second options.max_retry = 3; - + // CRITICAL: Set unique connection_group to prevent connection sharing - // Each CouchbaseOperations instance connected to same bucket gets its own connection group + // Each CouchbaseOperations instance connected to same bucket gets its own + // connection group options.connection_group = server_address + bucket_name; // enable_ssl @@ -2422,7 +2245,7 @@ CouchbaseOperations::Result CouchbaseOperations::authenticateAll( return result; } uint64_t cas; - if(response.popHello(&cas) == false) { + if (response.popHello(&cas) == false) { DEBUG_PRINT("Failed to receive HELO response from Couchbase: " << response.lastError()); delete new_channel; @@ -2433,8 +2256,8 @@ CouchbaseOperations::Result CouchbaseOperations::authenticateAll( return result; } if (response.popAuthenticate(&cas) == false) { - DEBUG_PRINT("Failed to authenticate user: " - << username << " to Couchbase: " << response.lastError()); + DEBUG_PRINT("Failed to authenticate user: " << username << " to Couchbase: " + << response.lastError()); result.success = false; result.value = ""; result.error_message = response.lastError(); @@ -2446,11 +2269,11 @@ CouchbaseOperations::Result CouchbaseOperations::authenticateAll( this->server_address_ = server_address; result.success = true; result.status_code = 0; - - DEBUG_PRINT("Instance " << reinterpret_cast(this) - << " authenticated with unique connection_group:" - << server_address_ + bucket_name); - + + DEBUG_PRINT("Instance " << reinterpret_cast(this) + << " authenticated with unique connection_group:" + << server_address_ + bucket_name); + // select the bucket result = selectBucket(bucket_name); return result; @@ -2478,7 +2301,7 @@ CouchbaseOperations::Result CouchbaseOperations::selectBucket( result.error_message = cntl.ErrorText(); return result; } - if (response.popSelectBucket(NULL, bucket_name) == false) { + if (response.popSelectBucket(NULL) == false) { result.success = false; result.value = ""; result.error_message = response.lastError(); @@ -2505,7 +2328,9 @@ CouchbaseOperations::Result CouchbaseOperations::selectBucket( // manifest for this bucket/server is not cached yet, will fetch it from // server now. refresh will also update the local cache with the fetched // manifest - request.metadata_tracking->refreshCollectionManifest(channel_, server_address_, bucket_name, request.local_collection_manifest_cache); + request.metadata_tracking->refreshCollectionManifest( + channel_, server_address_, bucket_name, + request.local_collection_manifest_cache); // We simply try once to prefetch the manifest, before any collection // operation. If it fails, it will be lazily updated when a collection // operation is performed. @@ -2524,40 +2349,6 @@ CouchbaseOperations::Result CouchbaseOperations::selectBucket( return result; } -// Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Replace(const string& key, -// const string& value, string collection_name) { -// CouchbaseRequest request; -// CouchbaseResponse response; -// brpc::Controller cntl; -// CouchbaseOperations::Result result; -// if(request.ReplaceRequest(key, value, 0, 0, 0, collection_name, channel, -// server_address, selected_bucket) == false){ -// LOG(ERROR) << "Failed to create Replace request for key: " << key; -// result.success = false; -// result.value = ""; -// return result; -// } -// channel->CallMethod(NULL, &cntl, &request, &response, NULL); -// if (cntl.Failed()) { -// LOG(ERROR) << "Failed to replace key: " << key << " to Couchbase: " << -// cntl.ErrorText(); result.success = false; result.value = ""; -// result.error_message = cntl.ErrorText(); -// return result; -// } -// uint64_t cas_value; -// if(response.PopReplace(&cas_value) == false){ -// result.success = false; -// result.value = ""; -// result.error_message = response.lastError(); -// return result; -// } -// // Successfully replaced the value -// result.success = true; -// result.value = ""; -// return result; -// } - CouchbaseOperations::Result CouchbaseOperations::append( const string& key, const string& value, string collection_name) { CouchbaseRequest request(&local_bucket_to_collection_manifest_); @@ -2581,141 +2372,6 @@ CouchbaseOperations::Result CouchbaseOperations::prepend( return result; } -// Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Increment(const string& key, -// uint64_t delta, uint64_t initial_value, uint32_t exptime, string -// collection_name) { -// CouchbaseRequest request; -// CouchbaseResponse response; -// brpc::Controller cntl; -// CouchbaseOperations::Result result; -// if(request.IncrementRequest(key, delta, initial_value, exptime, -// collection_name, channel, server_address, selected_bucket) == false){ -// LOG(ERROR) << "Failed to create Increment request for key: " << key; -// result.success = false; -// result.value = ""; -// return result; -// } -// channel->CallMethod(NULL, &cntl, &request, &response, NULL); -// if (cntl.Failed()) { -// LOG(ERROR) << "Failed to increment key: " << key << " in Couchbase: " << -// cntl.ErrorText(); result.success = false; result.value = ""; -// result.error_message = cntl.ErrorText(); -// return result; -// } -// uint64_t new_value, cas_value; -// if(response.PopIncrement(&new_value, &cas_value) == false){ -// result.success = false; -// result.value = ""; -// result.error_message = response.lastError(); -// return result; -// } -// // Successfully incremented the value -// result.success = true; -// result.value = std::to_string(new_value); -// return result; -// } - -// Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Decrement(const string& key, -// uint64_t delta, uint64_t initial_value, uint32_t exptime, string -// collection_name) { -// CouchbaseRequest request; -// CouchbaseResponse response; -// brpc::Controller cntl; -// CouchbaseOperations::Result result; -// if(request.DecrementRequest(key, delta, initial_value, exptime, -// collection_name, channel, server_address, selected_bucket) == false){ -// LOG(ERROR) << "Failed to create Decrement request for key: " << key; -// result.success = false; -// result.value = ""; -// return result; -// } -// channel->CallMethod(NULL, &cntl, &request, &response, NULL); -// if (cntl.Failed()) { -// LOG(ERROR) << "Failed to decrement key: " << key << " in Couchbase: " << -// cntl.ErrorText(); result.success = false; result.value = ""; -// result.error_message = cntl.ErrorText(); -// return result; -// } -// uint64_t new_value, cas_value; -// if(response.PopDecrement(&new_value, &cas_value) == false){ -// result.success = false; -// result.value = ""; -// result.error_message = response.lastError(); -// return result; -// } -// // Successfully decremented the value -// result.success = true; -// result.value = std::to_string(new_value); -// return result; -// } - -// Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Touch(const string& key, -// uint32_t exptime, string collection_name) { -// CouchbaseRequest request; -// CouchbaseResponse response; -// brpc::Controller cntl; -// CouchbaseOperations::Result result; -// if(request.TouchRequest(key, exptime, collection_name, channel, -// server_address, selected_bucket) == false){ -// LOG(ERROR) << "Failed to create Touch request for key: " << key; -// result.success = false; -// result.value = ""; -// return result; -// } -// channel->CallMethod(NULL, &cntl, &request, &response, NULL); -// if (cntl.Failed()) { -// LOG(ERROR) << "Failed to touch key: " << key << " in Couchbase: " << -// cntl.ErrorText(); result.success = false; result.value = ""; -// result.error_message = cntl.ErrorText(); -// return result; -// } -// if(response.PopTouch() == false){ -// result.success = false; -// result.value = ""; -// result.error_message = response.lastError(); -// return result; -// } -// // Successfully touched the key -// result.success = true; -// result.value = ""; -// return result; -// } - -// Warning: Not tested -// CouchbaseOperations::Result CouchbaseOperations::Flush(uint32_t timeout) { -// CouchbaseRequest request; -// CouchbaseResponse response; -// brpc::Controller cntl; -// CouchbaseOperations::Result result; -// if(request.FlushRequest(timeout) == false){ -// LOG(ERROR) << "Failed to create Flush request"; -// result.success = false; -// result.value = ""; -// return result; -// } -// channel->CallMethod(NULL, &cntl, &request, &response, NULL); -// if (cntl.Failed()) { -// LOG(ERROR) << "Failed to flush Couchbase: " << cntl.ErrorText(); -// result.success = false; -// result.value = ""; -// result.error_message = cntl.ErrorText(); -// return result; -// } -// if(response.PopFlush() == false){ -// result.success = false; -// result.value = ""; -// result.error_message = response.lastError(); -// return result; -// } -// // Successfully flushed -// result.success = true; -// result.value = ""; -// return result; -// } - CouchbaseOperations::Result CouchbaseOperations::version() { CouchbaseRequest request; CouchbaseResponse response; diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index 754341c902..f58c3f00d8 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -144,9 +144,9 @@ class CouchbaseManifestManager { bool jsonToCollectionManifest(const string& json, CollectionManifest* manifest); - bool refreshCollectionManifest(brpc::Channel* channel, const string& server, - const string& bucket, - unordered_map* local_cache = nullptr); + bool refreshCollectionManifest( + brpc::Channel* channel, const string& server, const string& bucket, + unordered_map* local_cache = nullptr); } static common_metadata_tracking; class CouchbaseOperations { public: @@ -187,8 +187,8 @@ class CouchbaseOperations { // Flush(uint32_t timeout = 0); Result version(); Result authenticateSSL(const string& username, const string& password, - const string& server_address, const string& bucket_name, - string path_to_cert = ""); + const string& server_address, + const string& bucket_name, string path_to_cert = ""); Result authenticate(const string& username, const string& password, const string& server_address, const string& bucket_name); Result selectBucket(const string& bucket_name); @@ -213,9 +213,12 @@ class CouchbaseOperations { const string& collection, uint8_t* coll_id); private: - CouchbaseOperations::Result authenticateAll( - const string& username, const string& password, - const string& server_address, const string& bucket_name, bool enable_ssl, string path_to_cert); + CouchbaseOperations::Result authenticateAll(const string& username, + const string& password, + const string& server_address, + const string& bucket_name, + bool enable_ssl, + string path_to_cert); friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); friend void policy::SerializeCouchbaseRequest( butil::IOBuf* buf, Controller* cntl, @@ -224,12 +227,12 @@ class CouchbaseOperations { string server_address_; string selected_bucket_; - unordered_map + unordered_map local_bucket_to_collection_manifest_; public: - // these classes have been made public so that normal user can also create advanced bRPC programs as per their requirements. + // these classes have been made public so that normal user can also create + // advanced bRPC programs as per their requirements. class CouchbaseRequest : public NonreflectableMessage { public: static brpc::CouchbaseManifestManager* metadata_tracking; @@ -303,7 +306,8 @@ class CouchbaseOperations { brpc::CouchbaseManifestManager* metadata_tracking, brpc::Channel* channel, const string& server, const string& selected_bucket, - unordered_map* local_cache); + unordered_map* + local_cache); // Collection-aware document operations bool getRequest(const butil::StringPiece& key, @@ -376,7 +380,9 @@ class CouchbaseOperations { int pipelinedCount() const { return _pipelined_count; } butil::IOBuf& rawBuffer() { return _buf; } - const butil::IOBuf& rawBuffer() const { return _buf; } // used in couchbase_protocol serialization. + const butil::IOBuf& rawBuffer() const { + return _buf; + } // used in couchbase_protocol serialization. void Swap(CouchbaseRequest* other); void MergeFrom(const CouchbaseRequest& from) override; void Clear() override; @@ -386,6 +392,7 @@ class CouchbaseOperations { class CouchbaseResponse : public NonreflectableMessage { public: static brpc::CouchbaseManifestManager* metadata_tracking; + private: string _err; butil::IOBuf _buf; @@ -502,7 +509,7 @@ class CouchbaseOperations { // bool popReplace(uint64_t* cas_value); bool popAppend(uint64_t* cas_value); bool popPrepend(uint64_t* cas_value); - bool popSelectBucket(uint64_t* cas_value, std::string bucket_name); + bool popSelectBucket(uint64_t* cas_value); bool popAuthenticate(uint64_t* cas_value); bool popHello(uint64_t* cas_value); diff --git a/src/brpc/policy/couchbase_protocol.cpp b/src/brpc/policy/couchbase_protocol.cpp index d40c950aa2..a014581ed5 100644 --- a/src/brpc/policy/couchbase_protocol.cpp +++ b/src/brpc/policy/couchbase_protocol.cpp @@ -134,15 +134,14 @@ ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket* socket, msg->meta.append(&local_header, sizeof(local_header)); source->pop_front(sizeof(*header)); source->cutn(&msg->meta, total_body_length); - if (++msg->pi.count >= pi.count) { - CHECK_EQ(msg->pi.count, pi.count); - msg = - static_cast(socket->release_parsing_context()); - msg->pi = pi; - return MakeMessage(msg); - } else { - socket->GivebackPipelinedInfo(pi); - } + if (++msg->pi.count >= pi.count) { + CHECK_EQ(msg->pi.count, pi.count); + msg = static_cast(socket->release_parsing_context()); + msg->pi = pi; + return MakeMessage(msg); + } else { + socket->GivebackPipelinedInfo(pi); + } } } @@ -213,7 +212,6 @@ void PackCouchbaseRequest(butil::IOBuf* buf, SocketMessage**, Controller* cntl, const butil::IOBuf& request, const Authenticator* auth) { if (auth) { - std::cout << "Appending authentication data to request" << std::endl; std::string auth_str; if (auth->GenerateCredential(&auth_str) != 0) { return cntl->SetFailed(EREQUEST, "Fail to generate credential"); diff --git a/src/brpc/policy/couchbase_protocol.h b/src/brpc/policy/couchbase_protocol.h index b9caee11f2..15367def0b 100644 --- a/src/brpc/policy/couchbase_protocol.h +++ b/src/brpc/policy/couchbase_protocol.h @@ -23,161 +23,151 @@ namespace brpc { namespace policy { - enum CouchbaseMagic { - CB_MAGIC_REQUEST = 0x80, - CB_MAGIC_RESPONSE = 0x81 - }; - - // Definition of the data types in the packet - // https://github.com/couchbase/kv_engine/blob/master/docs/BinaryProtocol.md - enum CouchbaseBinaryDataType { - CB_BINARY_RAW_BYTES = 0x00 - }; - - enum CouchbaseJsonDataType { - CB_JSON = 0x01 - }; - - // Definition of the different command opcodes. - // https://github.com/couchbase/kv_engine/blob/master/docs/BinaryProtocol.md - enum CouchbaseBinaryCommand { - CB_HELLO_SELECT_FEATURES = 0x1f, - CB_SELECT_BUCKET = 0x89, - CB_GET_SCOPE_ID = 0xBC, - CB_BINARY_GET = 0x00, - CB_BINARY_SET = 0x01, - CB_BINARY_ADD = 0x02, - CB_BINARY_REPLACE = 0x03, - CB_BINARY_DELETE = 0x04, - CB_BINARY_INCREMENT = 0x05, - CB_BINARY_DECREMENT = 0x06, - CB_BINARY_QUIT = 0x07, - CB_BINARY_FLUSH = 0x08, - CB_BINARY_GETQ = 0x09, - CB_BINARY_NOOP = 0x0a, - CB_BINARY_VERSION = 0x0b, - CB_BINARY_GETK = 0x0c, - CB_BINARY_GETKQ = 0x0d, - CB_BINARY_APPEND = 0x0e, - CB_BINARY_PREPEND = 0x0f, - CB_BINARY_STAT = 0x10, - CB_BINARY_SETQ = 0x11, - CB_BINARY_ADDQ = 0x12, - CB_BINARY_REPLACEQ = 0x13, - CB_BINARY_DELETEQ = 0x14, - CB_BINARY_INCREMENTQ = 0x15, - CB_BINARY_DECREMENTQ = 0x16, - CB_BINARY_QUITQ = 0x17, - CB_BINARY_FLUSHQ = 0x18, - CB_BINARY_APPENDQ = 0x19, - CB_BINARY_PREPENDQ = 0x1a, - CB_BINARY_TOUCH = 0x1c, - CB_BINARY_GAT = 0x1d, - CB_BINARY_GATQ = 0x1e, - CB_BINARY_GATK = 0x23, - CB_BINARY_GATKQ = 0x24, - - CB_BINARY_SASL_LIST_MECHS = 0x20, - CB_BINARY_SASL_AUTH = 0x21, - CB_BINARY_SASL_STEP = 0x22, - - // Collection Management Commands (Couchbase 7.0+) - CB_GET_CLUSTER_CONFIG = 0xb5, - CB_GET_COLLECTIONS_MANIFEST = 0xba, - CB_COLLECTIONS_GET_CID = 0xbb, - CB_COLLECTIONS_GET_SCOPE_ID = 0xbc, - - }; - - struct CouchbaseRequestHeader { - // Magic number identifying the package (See Couchbase Binary Protocol#Magic_Byte) - uint8_t magic; - - // Command code (See Couchbase Binary Protocol#Command_opcodes) - uint8_t command; - - // Length in bytes of the text key that follows the command extras - uint16_t key_length; - - // Length in bytes of the command extras - uint8_t extras_length; - - // Reserved for future use (See Couchbase Binary Protocol#Data_Type) - uint8_t data_type; - - // The virtual bucket for this command - uint16_t vbucket_id; - - // Length in bytes of extra + key + value - uint32_t total_body_length; - - // Will be copied back to you in the response - uint32_t opaque; - - // Data version check - uint64_t cas_value; - }; - - struct CouchbaseResponseHeader { - // Magic number identifying the package (See Couchbase Binary Protocol#Magic_Byte) - uint8_t magic; - - // Command code (See Couchbase Binary Protocol#Command_opcodes) - uint8_t command; - - // Length in bytes of the text key that follows the command extras - uint16_t key_length; - - // Length in bytes of the command extras - uint8_t extras_length; - - // Reserved for future use (See Couchbase Binary Protocol#Data_Type) - uint8_t data_type; - - // Status of the response (non-zero on error) - uint16_t status; - - // Length in bytes of extra + key + value - uint32_t total_body_length; - - // Will be copied back to you in the response - uint32_t opaque; - - // Data version check - uint64_t cas_value; - }; - - // Parse couchbase messages. - ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket *socket, bool read_eof, - const void *arg); - - // Actions to a couchbase response. - void ProcessCouchbaseResponse(InputMessageBase* msg); - - // Serialize a couchbase request. - void SerializeCouchbaseRequest(butil::IOBuf* buf, - Controller* cntl, - const google::protobuf::Message* request); - - // Pack `request' to `method' into `buf'. - void PackCouchbaseRequest(butil::IOBuf* buf, - SocketMessage**, - uint64_t correlation_id, - const google::protobuf::MethodDescriptor* method, - Controller* controller, - const butil::IOBuf& request, - const Authenticator* auth); - - //process couchbase request. - //since, there is no server side instance running, this function is not implemented. - //void ProcessCouchbaseRequest(InputMessageBase* msg); - - const std::string& GetCouchbaseMethodName( - const google::protobuf::MethodDescriptor*, - const Controller*); - - } // namespace policy -} // namespace brpc +enum CouchbaseMagic { CB_MAGIC_REQUEST = 0x80, CB_MAGIC_RESPONSE = 0x81 }; + +// Definition of the data types in the packet +// https://github.com/couchbase/kv_engine/blob/master/docs/BinaryProtocol.md +enum CouchbaseBinaryDataType { CB_BINARY_RAW_BYTES = 0x00 }; + +enum CouchbaseJsonDataType { CB_JSON = 0x01 }; + +// Definition of the different command opcodes. +// https://github.com/couchbase/kv_engine/blob/master/docs/BinaryProtocol.md +enum CouchbaseBinaryCommand { + CB_HELLO_SELECT_FEATURES = 0x1f, + CB_SELECT_BUCKET = 0x89, + CB_GET_SCOPE_ID = 0xBC, + CB_BINARY_GET = 0x00, + CB_BINARY_SET = 0x01, + CB_BINARY_ADD = 0x02, + CB_BINARY_REPLACE = 0x03, + CB_BINARY_DELETE = 0x04, + CB_BINARY_INCREMENT = 0x05, + CB_BINARY_DECREMENT = 0x06, + CB_BINARY_QUIT = 0x07, + CB_BINARY_FLUSH = 0x08, + CB_BINARY_GETQ = 0x09, + CB_BINARY_NOOP = 0x0a, + CB_BINARY_VERSION = 0x0b, + CB_BINARY_GETK = 0x0c, + CB_BINARY_GETKQ = 0x0d, + CB_BINARY_APPEND = 0x0e, + CB_BINARY_PREPEND = 0x0f, + CB_BINARY_STAT = 0x10, + CB_BINARY_SETQ = 0x11, + CB_BINARY_ADDQ = 0x12, + CB_BINARY_REPLACEQ = 0x13, + CB_BINARY_DELETEQ = 0x14, + CB_BINARY_INCREMENTQ = 0x15, + CB_BINARY_DECREMENTQ = 0x16, + CB_BINARY_QUITQ = 0x17, + CB_BINARY_FLUSHQ = 0x18, + CB_BINARY_APPENDQ = 0x19, + CB_BINARY_PREPENDQ = 0x1a, + CB_BINARY_TOUCH = 0x1c, + CB_BINARY_GAT = 0x1d, + CB_BINARY_GATQ = 0x1e, + CB_BINARY_GATK = 0x23, + CB_BINARY_GATKQ = 0x24, + + CB_BINARY_SASL_LIST_MECHS = 0x20, + CB_BINARY_SASL_AUTH = 0x21, + CB_BINARY_SASL_STEP = 0x22, + + // Collection Management Commands (Couchbase 7.0+) + CB_GET_CLUSTER_CONFIG = 0xb5, + CB_GET_COLLECTIONS_MANIFEST = 0xba, + CB_COLLECTIONS_GET_CID = 0xbb, + CB_COLLECTIONS_GET_SCOPE_ID = 0xbc, + +}; + +struct CouchbaseRequestHeader { + // Magic number identifying the package (See Couchbase Binary + // Protocol#Magic_Byte) + uint8_t magic; + + // Command code (See Couchbase Binary Protocol#Command_opcodes) + uint8_t command; + + // Length in bytes of the text key that follows the command extras + uint16_t key_length; + + // Length in bytes of the command extras + uint8_t extras_length; + + // Reserved for future use (See Couchbase Binary Protocol#Data_Type) + uint8_t data_type; + + // The virtual bucket for this command + uint16_t vbucket_id; + + // Length in bytes of extra + key + value + uint32_t total_body_length; + + // Will be copied back to you in the response + uint32_t opaque; + + // Data version check + uint64_t cas_value; +}; + +struct CouchbaseResponseHeader { + // Magic number identifying the package (See Couchbase Binary + // Protocol#Magic_Byte) + uint8_t magic; + + // Command code (See Couchbase Binary Protocol#Command_opcodes) + uint8_t command; + + // Length in bytes of the text key that follows the command extras + uint16_t key_length; + + // Length in bytes of the command extras + uint8_t extras_length; + + // Reserved for future use (See Couchbase Binary Protocol#Data_Type) + uint8_t data_type; + + // Status of the response (non-zero on error) + uint16_t status; + + // Length in bytes of extra + key + value + uint32_t total_body_length; + + // Will be copied back to you in the response + uint32_t opaque; + + // Data version check + uint64_t cas_value; +}; + +// Parse couchbase messages. +ParseResult ParseCouchbaseMessage(butil::IOBuf* source, Socket* socket, + bool read_eof, const void* arg); + +// Actions to a couchbase response. +void ProcessCouchbaseResponse(InputMessageBase* msg); + +// Serialize a couchbase request. +void SerializeCouchbaseRequest(butil::IOBuf* buf, Controller* cntl, + const google::protobuf::Message* request); + +// Pack `request' to `method' into `buf'. +void PackCouchbaseRequest(butil::IOBuf* buf, SocketMessage**, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* controller, const butil::IOBuf& request, + const Authenticator* auth); +// process couchbase request. +// since, there is no server side instance running, this function is not +// implemented. void ProcessCouchbaseRequest(InputMessageBase* msg); + +const std::string& GetCouchbaseMethodName( + const google::protobuf::MethodDescriptor*, const Controller*); + +} // namespace policy +} // namespace brpc #endif // BRPC_POLICY_COUCHBASE_BINARY_PROTOCOL_H From d45e5f3b37316ef4d8fbc5bf7af8416749937ac9 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 10 Nov 2025 16:41:27 +0530 Subject: [PATCH 46/49] updated couchbase_example.md --- docs/en/couchbase_example.md | 236 ++++++++++++++++++++++++++++++++--- 1 file changed, 218 insertions(+), 18 deletions(-) diff --git a/docs/en/couchbase_example.md b/docs/en/couchbase_example.md index 8ad2728b4a..6748580827 100644 --- a/docs/en/couchbase_example.md +++ b/docs/en/couchbase_example.md @@ -12,6 +12,7 @@ The core pieces are: * `src/brpc/couchbase.[h|cpp]` – high-level `CouchbaseOperations` class with request (`CouchbaseRequest`) and response (`CouchbaseResponse`) builders, parsers and error-handlers. * `example/couchbase_c++/couchbase_client.cpp` – an end‑to‑end example using the high-level API for authentication, bucket selection, CRUD operations, and collection‑scoped operations. * `example/couchbase_c++/multithreaded_couchbase_client.cpp` – a multithreaded example where an instance of `CouchbaseOperations` is shared across the threads operating on same bucket. An another block of code where multiple threads have their own `CouchbaseOperations` instance as the threads operate on different buckets. +* `example/couchbase_c++/traditional_brpc_couchbase_client.cpp` – demonstrates the traditional bRPC approach with manual channel, controller, and request/response management for advanced users who need fine-grained control. Design goals: * **SSL Support**: Built-in SSL/TLS support for secure connections to Couchbase Capella. @@ -26,6 +27,7 @@ Design goals: | Category | Supported Operations | Notes | |----------|----------------------|-------| | **High-Level API** | `CouchbaseOperations` class | **Recommended**: Simple methods returning `Result` struct | +| **Traditional API** | Manual channel/controller management | **Advanced**: Direct bRPC access for custom configurations | | **SSL/TLS Support** | Built-in SSL encryption | **Required** for Couchbase Capella, optional for local clusters | | Authentication | SASL `PLAIN` with/without SSL | `authenticate()` for non-SSL, `authenticateSSL()` for SSL connections | | Bucket selection | Integrated with authentication | Bucket specified during authentication; `selectBucket()` also available separately | @@ -220,22 +222,215 @@ if (!result.success) { ``` --- -### 5. Request/Response Classes (`CouchbaseRequest`/`CouchbaseResponse`) +### 5. Traditional bRPC Couchbase Client (`traditional_brpc_couchbase_client.cpp`) -These classes are public in `CouchbaseOperations` and can be used for advanced bRPC programs. The high-level API uses these classes internally. They are responsible for building the request that needs to be sent and received over the channel. +For developers who need fine-grained control over the bRPC framework or want to understand the low-level implementation, we provide a traditional bRPC client example. This approach requires manual management of channels, controllers, and response parsing. -#### Request Building (Advanced Usage): +**When to use Traditional API:** +- Advanced bRPC users who need custom channel configurations +- Fine-grained control over connection pooling and retry logic +- Direct access to underlying bRPC controller for debugging +- Learning the internal workings of the high-level API + +**When to use High-Level API (Recommended):** +- Standard CRUD operations and authentication +- Simpler error handling and cleaner code +- Collection based operations with minimal boilerplate +- Pipeline operations for batch processing while also available in traditional approach it is easier to do using High-Level API. + +#### Traditional Client Example Walkthrough + +The traditional client (`example/couchbase_c++/traditional_brpc_couchbase_client.cpp`) demonstrates the low-level bRPC approach: + +**1. Channel Setup and Configuration** +```cpp +brpc::Channel channel; +brpc::ChannelOptions options; +options.protocol = brpc::PROTOCOL_COUCHBASE; // Set Couchbase protocol +options.connection_type = "single"; // Single persistent connection +options.timeout_ms = 1000; // 1 second timeout +options.max_retry = 3; // Retry up to 3 times + +if (channel.Init("localhost:11210", &options) != 0) { + LOG(ERROR) << "Failed to initialize channel"; + return -1; +} +``` + +**2. Authentication with Manual Request/Response Handling** +```cpp +brpc::Controller cntl; +brpc::CouchbaseOperations::CouchbaseRequest req; +brpc::CouchbaseOperations::CouchbaseResponse res; +uint64_t cas; + +// Build authentication request +req.authenticateRequest("Administrator", "password"); + +// Execute the request +channel.CallMethod(NULL, &cntl, &req, &res, NULL); + +// Check controller status +if (cntl.Failed()) { + LOG(ERROR) << "Unable to authenticate: " << cntl.ErrorText(); + return -1; +} + +// Parse response - must call popHello() and popAuthenticate() in order +if (res.popHello(&cas) && res.popAuthenticate(&cas)) { + std::cout << "Authentication Successful" << std::endl; +} else { + std::cout << "Authentication Failed with status code: " + << std::hex << res._status_code << std::endl; + return -1; +} +``` + +**3. Bucket Selection** ```cpp -CouchbaseRequest req; -req.helloRequest(); // HELLO negotiation -req.authenticateRequest(user, pass); // SASL PLAIN authentication -req.selectBucketRequest("travel-sample"); -req.addRequest("doc::1", json_body, flags, exptime, /*cas*/0); -req.getRequest("doc::1"); // Pipeline GET after ADD +// IMPORTANT: Reset controller and clear request/response before each operation +cntl.Reset(); +req.Clear(); +res.Clear(); + +// Build bucket selection request +req.selectBucketRequest("testing"); + +// Execute the request +channel.CallMethod(NULL, &cntl, &req, &res, NULL); + +if (cntl.Failed()) { + LOG(ERROR) << "Unable to select bucket: " << cntl.ErrorText(); + return -1; +} -channel.CallMethod(nullptr, &cntl, &req, &resp, nullptr); +// Parse response - status_code only updated AFTER calling pop function +if (res.popSelectBucket(&cas)) { + std::cout << "Bucket Selection Successful" << std::endl; +} else { + std::cout << "Bucket Selection Failed with status code: " + << std::hex << res._status_code << std::endl; + std::cout << "Error Message: " << res.lastError() << std::endl; + return -1; +} ``` +**4. ADD Operation (Create Document)** +```cpp +// Reset for new operation +cntl.Reset(); +req.Clear(); +res.Clear(); + +// Build ADD request +req.addRequest( + "sample_key", // key + R"({"name": "John Doe", "age": 30, "email": "john@example.com"})", // value + 0, // flags + 0, // exptime (0 = no expiration) + 0 // cas (0 for new document) +); + +// Execute the request +channel.CallMethod(NULL, &cntl, &req, &res, NULL); + +if (cntl.Failed()) { + LOG(ERROR) << "Unable to add key-value: " << cntl.ErrorText(); + return -1; +} + +// Parse response +if (res.popAdd(&cas)) { + std::cout << "Key-Value Addition Successful" << std::endl; +} else { + std::cout << "Key-Value Addition Failed with status code: " + << std::hex << res._status_code << std::endl; + std::cout << "Error Message: " << res.lastError() << std::endl; + return -1; +} +``` + +**5. GET Operation (Retrieve Document)** +```cpp +// Reset for new operation +cntl.Reset(); +req.Clear(); +res.Clear(); + +// Build GET request +req.getRequest("sample_key"); + +// Execute the request +channel.CallMethod(NULL, &cntl, &req, &res, NULL); + +if (cntl.Failed()) { + LOG(ERROR) << "Unable to get value for key: " << cntl.ErrorText(); + return -1; +} + +// Parse response - GET returns value and flags +std::string value; +uint32_t flags; +if (res.popGet(&value, &flags, &cas)) { + std::cout << "Key-Value Retrieval Successful" << std::endl; + std::cout << "Retrieved Value: " << value << std::endl; +} else { + std::cout << "Key-Value Retrieval Failed with status code: " + << std::hex << res._status_code << std::endl; + std::cout << "Error Message: " << res.lastError() << std::endl; + return -1; +} +``` + +**6. DELETE Operation (Remove Document)** +```cpp +// Reset for new operation +cntl.Reset(); +req.Clear(); +res.Clear(); + +// Build DELETE request +req.deleteRequest("sample_key"); + +// Execute the request +channel.CallMethod(NULL, &cntl, &req, &res, NULL); + +if (cntl.Failed()) { + LOG(ERROR) << "Unable to delete key-value: " << cntl.ErrorText(); + return -1; +} + +// Parse response +if (res.popDelete()) { + std::cout << "Key-Value Deletion Successful" << std::endl; +} else { + std::cout << "Key-Value Deletion Failed with status code: " + << std::hex << res._status_code << std::endl; + std::cout << "Error Message: " << res.lastError() << std::endl; + return -1; +} +``` + +#### Key Differences: Traditional vs High-Level API + +| Aspect | Traditional API | High-Level API | +|--------|----------------|----------------| +| **Setup** | Manual channel, controller, request/response management | Single `CouchbaseOperations` instance | +| **Error Handling** | Check both `cntl.Failed()` and response status | Simple `Result.success` boolean | +| **Resource Management** | Must call `cntl.Reset()`, `req.Clear()`, `res.Clear()` | Automatic | +| **Response Parsing** | Manual `pop*()` calls with CAS handling | Transparent | +| **Code Verbosity** | ~15-20 lines per operation | ~2-3 lines per operation | +| **Collections** | Manual collection ID retrieval and management | Automatic with collection name parameter | +| **Pipeline Operations** | Complex manual request building | Simple `beginPipeline()`, `pipelineRequest()`, `executePipeline()` | +| **SSL Support** | Manual SSL configuration in channel options | Built-in `authenticateSSL()` method | +| **Threading** | Manual connection pooling management | Automatic connection group management | + +--- +### 6. Request/Response Classes (`CouchbaseRequest`/`CouchbaseResponse`) + +These classes are public in `CouchbaseOperations` and can be used for advanced bRPC programs. The high-level API uses these classes internally, and the traditional client example demonstrates their direct usage. They are responsible for building the request that needs to be sent and received over the channel. + + #### Response Parsing: Each `pop*` method consumes the front of the internal response buffer, validating: 1. Header present. @@ -244,7 +439,7 @@ Each `pop*` method consumes the front of the internal response buffer, validatin 4. Body length sufficient. --- -### 6. Example Client Walkthrough +### 7. Example Client Walkthrough #### Single-Threaded Example (`couchbase_client.cpp`) Uses the **high-level `CouchbaseOperations` API**: @@ -796,7 +991,7 @@ Key features: - Both separate instance and shared instance patterns --- -### 7. Building and Running the Examples +### 8. Building and Running the Examples #### Build both examples: ```bash @@ -804,18 +999,23 @@ cd example/couchbase_c++/ make ``` -#### Run Single-Threaded Example: +#### Run Single-Threaded Example (High-Level API): ```bash ./couchbase_client ``` -#### Run Multithreaded Example: +#### Run Multithreaded Example (High-Level API): ```bash ./multithreaded_couchbase_client ``` +#### Run Traditional bRPC Client (Low-Level API): +```bash +./traditional_brpc_couchbase_client +``` + --- -### 8. Setting Up Couchbase +### 9. Setting Up Couchbase #### A. Local Install (Non‑Docker) Download from: https://www.couchbase.com/downloads/ (Community or Enterprise) and Install. @@ -872,7 +1072,7 @@ auto result = couchbase_ops.authenticateSSL( --- -### 9. Error Handling Patterns +### 10. Error Handling Patterns #### High-Level API (Recommended) The `CouchbaseOperations` class uses a simple `Result` struct: @@ -908,7 +1108,7 @@ if (get_result.success) { ``` --- -### 10. Best Practices +### 11. Best Practices #### Threading Patterns > **💡 FLEXIBLE THREADING OPTIONS** @@ -955,7 +1155,7 @@ ops_server2.authenticate(username, password, "server2:11210", bucket_name); ``` --- -### 11. Summary and References +### 12. Summary and References This implementation provides high-level APIs for Couchbase KV operations. Couchbase (the company) contributed to this implementation, but it is not officially supported; it is "[Community Supported](https://docs.couchbase.com/server/current/third-party/integrations.html#support-model)". --- From 3e225dee91bcc6290dcc14622a0f3a723ac31efd Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Mon, 10 Nov 2025 20:16:56 +0530 Subject: [PATCH 47/49] added unit test cases --- test/brpc_couchbase_unittest.cpp | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 test/brpc_couchbase_unittest.cpp diff --git a/test/brpc_couchbase_unittest.cpp b/test/brpc_couchbase_unittest.cpp new file mode 100644 index 0000000000..317d73bea5 --- /dev/null +++ b/test/brpc_couchbase_unittest.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +namespace brpc { +DECLARE_int32(idle_timeout_second); +} + +int main(int argc, char* argv[]) { + brpc::FLAGS_idle_timeout_second = 0; + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +namespace { + +// Unit Tests - No Server Required +class CouchbaseUnitTest : public testing::Test {}; + +TEST_F(CouchbaseUnitTest, RequestBuilders) { + brpc::CouchbaseOperations::CouchbaseRequest req; + req.Clear(); + req.helloRequest(); + req.authenticateRequest("user", "pass"); + req.selectBucketRequest("bucket"); + req.addRequest("key", "value", 0, 0, 0); + req.getRequest("key"); + req.upsertRequest("key", "value", 0, 0, 0); + req.deleteRequest("key"); + EXPECT_TRUE(true); +} + +TEST_F(CouchbaseUnitTest, ResultStruct) { + brpc::CouchbaseOperations::Result result; + + result.success = true; + result.error_message = "Test"; + result.value = R"({"test": "data"})"; + result.status_code = 0x01; + + EXPECT_TRUE(result.success); + EXPECT_EQ("Test", result.error_message); + EXPECT_EQ(R"({"test": "data"})", result.value); + EXPECT_EQ(0x01, result.status_code); + + result.success = false; + result.error_message = ""; + result.value = ""; + result.status_code = 0x00; + + EXPECT_FALSE(result.success); + EXPECT_TRUE(result.error_message.empty()); + EXPECT_TRUE(result.value.empty()); + EXPECT_EQ(0x00, result.status_code); +} + +TEST_F(CouchbaseUnitTest, EdgeCases) { + brpc::CouchbaseOperations::CouchbaseRequest req; + req.addRequest("", "value", 0, 0, 0); + req.addRequest("key", "", 0, 0, 0); + req.addRequest(std::string(1000, 'x'), "val", 0, 0, 0); + req.addRequest("key", std::string(10000, 'x'), 0, 0, 0); + req.addRequest("test::special::!!!","val", 0, 0, 0); + req.addRequest("key", R"({"unicode":"123"})", 0, 0, 0); + EXPECT_TRUE(true); +} + +} // namespace From 7e4560dd3faaa82f98abce53e60d45044a95d245 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Wed, 12 Nov 2025 14:47:07 +0530 Subject: [PATCH 48/49] removed using namespace std from couchbase.h --- .../multithreaded_couchbase_client.cpp | 4 +- src/brpc/couchbase.cpp | 128 ++++++------ src/brpc/couchbase.h | 186 ++++++++---------- test/brpc_couchbase_unittest.cpp | 2 +- 4 files changed, 146 insertions(+), 174 deletions(-) diff --git a/example/couchbase_c++/multithreaded_couchbase_client.cpp b/example/couchbase_c++/multithreaded_couchbase_client.cpp index bc8bcd725d..d6dd9e56be 100644 --- a/example/couchbase_c++/multithreaded_couchbase_client.cpp +++ b/example/couchbase_c++/multithreaded_couchbase_client.cpp @@ -140,7 +140,7 @@ void perform_crud_operations_col1(brpc::CouchbaseOperations& couchbase_ops, stats->operations_successful++; } else { stats->operations_failed++; - cout << "UPSERT failed: " << result.error_message << std::endl; + std::cout << "UPSERT failed: " << result.error_message << std::endl; return; } @@ -152,7 +152,7 @@ void perform_crud_operations_col1(brpc::CouchbaseOperations& couchbase_ops, stats->operations_successful++; } else { stats->operations_failed++; - cout << "GET failed: " << result.error_message << std::endl; + std::cout << "GET failed: " << result.error_message << std::endl; return; } diff --git a/src/brpc/couchbase.cpp b/src/brpc/couchbase.cpp index 3e915ccdad..52e16dc9c7 100644 --- a/src/brpc/couchbase.cpp +++ b/src/brpc/couchbase.cpp @@ -57,7 +57,7 @@ CouchbaseManifestManager* &common_metadata_tracking; bool brpc::CouchbaseManifestManager::setBucketToCollectionManifest( - string server, string bucket, + std::string server, std::string bucket, CouchbaseManifestManager::CollectionManifest manifest) { // Then update the collection manifest with proper locking { @@ -69,7 +69,7 @@ bool brpc::CouchbaseManifestManager::setBucketToCollectionManifest( } bool brpc::CouchbaseManifestManager::getBucketToCollectionManifest( - string server, string bucket, + std::string server, std::string bucket, CouchbaseManifestManager::CollectionManifest* manifest) { SharedLock read_lock(rw_bucket_to_collection_manifest_mutex_); auto it1 = bucket_to_collection_manifest_.find(server); @@ -85,8 +85,8 @@ bool brpc::CouchbaseManifestManager::getBucketToCollectionManifest( } bool brpc::CouchbaseManifestManager::getManifestToCollectionId( - CouchbaseManifestManager::CollectionManifest* manifest, string scope, - string collection, uint8_t* collection_id) { + CouchbaseManifestManager::CollectionManifest* manifest, std::string scope, + std::string collection, uint8_t* collection_id) { if (manifest == nullptr || collection_id == nullptr) { DEBUG_PRINT("Invalid input: manifest or collection_id is null"); return false; @@ -107,7 +107,7 @@ bool brpc::CouchbaseManifestManager::getManifestToCollectionId( } bool CouchbaseManifestManager::jsonToCollectionManifest( - const string& json, + const std::string& json, CouchbaseManifestManager::CollectionManifest* manifest) { if (manifest == nullptr) { DEBUG_PRINT("Invalid input: manifest is null"); @@ -119,7 +119,7 @@ bool CouchbaseManifestManager::jsonToCollectionManifest( manifest->scope_to_collection_id_map.clear(); if (json.empty()) { - DEBUG_PRINT("JSON string is empty"); + DEBUG_PRINT("JSON std::string is empty"); return false; } @@ -165,7 +165,7 @@ bool CouchbaseManifestManager::jsonToCollectionManifest( DEBUG_PRINT("Missing or invalid 'name' field in scope at index " << i); return false; } - string scope_name = scope["name"].GetString(); + std::string scope_name = scope["name"].GetString(); // Extract collections if (!scope.HasMember("collections") || !scope["collections"].IsArray()) { @@ -175,7 +175,7 @@ bool CouchbaseManifestManager::jsonToCollectionManifest( } const BUTIL_RAPIDJSON_NAMESPACE::Value& collections = scope["collections"]; - unordered_map collection_map; + std:: unordered_map collection_map; for (BUTIL_RAPIDJSON_NAMESPACE::SizeType j = 0; j < collections.Size(); ++j) { @@ -193,20 +193,20 @@ bool CouchbaseManifestManager::jsonToCollectionManifest( << j << " in scope '" << scope_name << "'"); return false; } - string collection_name = collection["name"].GetString(); + std::string collection_name = collection["name"].GetString(); - // Extract collection uid (hex string) + // Extract collection uid (hex std::string) if (!collection.HasMember("uid") || !collection["uid"].IsString()) { DEBUG_PRINT("Missing or invalid 'uid' field in collection '" << collection_name << "' in scope '" << scope_name << "'"); return false; } - string collection_uid_str = collection["uid"].GetString(); + std::string collection_uid_str = collection["uid"].GetString(); - // Convert hex string to uint8_t + // Convert hex std::string to uint8_t uint8_t collection_id = 0; try { - // Convert hex string to integer + // Convert hex std::string to integer unsigned long uid_val = std::stoul(collection_uid_str, nullptr, 16); if (uid_val > 255) { DEBUG_PRINT( @@ -237,8 +237,8 @@ bool CouchbaseManifestManager::jsonToCollectionManifest( } bool CouchbaseManifestManager::refreshCollectionManifest( - brpc::Channel* channel, const string& server, const string& bucket, - unordered_map* + brpc::Channel* channel, const std::string& server, const std::string& bucket, + std:: unordered_map* local_collection_manifest_cache) { // first fetch the manifest // then compare the UID with the cached one @@ -265,7 +265,7 @@ bool CouchbaseManifestManager::refreshCollectionManifest( << temp_cntl.ErrorText()); return false; } - string manifest_json; + std::string manifest_json; if (!temp_get_manifest_response.popManifest(&manifest_json)) { DEBUG_PRINT("Failed to parse response for refreshing collection Manifest: " << temp_get_manifest_response.lastError()); @@ -498,7 +498,7 @@ bool CouchbaseOperations::CouchbaseRequest::helloRequest() { #endif agent += ";bssl/0x1010107f)"; - // Generate a random connection ID as hex string + // Generate a random connection ID as hex std::string unsigned char raw_id[CONNECTION_ID_SIZE]; FILE* urandom = fopen("/dev/urandom", "rb"); if (!urandom || @@ -594,7 +594,7 @@ bool CouchbaseOperations::CouchbaseRequest::authenticateRequest( auth_str.append(kPadding, sizeof(kPadding)); auth_str.append(password.data(), password.size()); if (_buf.append(auth_str.data(), auth_str.size())) { - DEBUG_PRINT("Failed to append auth string to buffer"); + DEBUG_PRINT("Failed to append auth std::string to buffer"); return false; } ++_pipelined_count; @@ -817,10 +817,10 @@ bool CouchbaseOperations::CouchbaseRequest::getOrDelete( // collectionID fetching either from the metadata cache or if doesn't exist then // fetch from the server. bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( - string collection_name, uint8_t* coll_id, + std::string collection_name, uint8_t* coll_id, brpc::CouchbaseManifestManager* metadata_tracking, brpc::Channel* channel, - const string& server, const string& selected_bucket, - unordered_map* + const std::string& server, const std::string& selected_bucket, + std:: unordered_map* local_cache) { if (collection_name.empty()) { DEBUG_PRINT("Empty collection name"); @@ -890,8 +890,8 @@ bool CouchbaseOperations::CouchbaseRequest::getCachedOrFetchCollectionId( } bool CouchbaseOperations::CouchbaseRequest::getRequest( - const butil::StringPiece& key, string collection_name, - brpc::Channel* channel, const string& server, const string& bucket) { + const butil::StringPiece& key, std::string collection_name, + brpc::Channel* channel, const std::string& server, const std::string& bucket) { DEBUG_PRINT("getRequest called with key: " << key << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); @@ -930,8 +930,8 @@ bool CouchbaseOperations::CouchbaseRequest::getRequest( } bool CouchbaseOperations::CouchbaseRequest::deleteRequest( - const butil::StringPiece& key, string collection_name, - brpc::Channel* channel, const string& server, const string& bucket) { + const butil::StringPiece& key, std::string collection_name, + brpc::Channel* channel, const std::string& server, const std::string& bucket) { DEBUG_PRINT("deleteRequest called with key: " << key << ", collection_name: " << collection_name << ", server: " << server << ", bucket: " << bucket); @@ -1274,8 +1274,8 @@ CouchbaseOperations::CouchbaseResponse::couchbaseBinaryCommandToString( bool CouchbaseOperations::CouchbaseRequest::upsertRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name, brpc::Channel* channel, const string& server, - const string& bucket) { + std::string collection_name, brpc::Channel* channel, const std::string& server, + const std::string& bucket) { DEBUG_PRINT("upsertRequest called with key: " << key << ", value: " << value << ", collection_name: " << collection_name @@ -1337,8 +1337,8 @@ bool CouchbaseOperations::CouchbaseRequest::getCollectionManifest() { bool CouchbaseOperations::CouchbaseRequest::addRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name, brpc::Channel* channel, const string& server, - const string& bucket) { + std::string collection_name, brpc::Channel* channel, const std::string& server, + const std::string& bucket) { DEBUG_PRINT("addRequest called with key: " << key << ", value: " << value << ", collection_name: " << collection_name @@ -1381,8 +1381,8 @@ bool CouchbaseOperations::CouchbaseRequest::addRequest( bool CouchbaseOperations::CouchbaseRequest::appendRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name, brpc::Channel* channel, const string& server, - const string& bucket) { + std::string collection_name, brpc::Channel* channel, const std::string& server, + const std::string& bucket) { if (value.empty()) { DEBUG_PRINT("value to append must be non-empty"); return false; @@ -1416,8 +1416,8 @@ bool CouchbaseOperations::CouchbaseRequest::appendRequest( bool CouchbaseOperations::CouchbaseRequest::prependRequest( const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name, brpc::Channel* channel, const string& server, - const string& bucket) { + std::string collection_name, brpc::Channel* channel, const std::string& server, + const std::string& bucket) { if (value.empty()) { DEBUG_PRINT("value to prepend must be non-empty"); return false; @@ -1804,10 +1804,10 @@ bool CouchbaseOperations::CouchbaseResponse::popVersion(std::string* version) { return true; } -bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, - const string& value, string collection_name, +bool sendRequest(CouchbaseOperations::operation_type op_type, const std::string& key, + const std::string& value, std::string collection_name, CouchbaseOperations::Result* result, brpc::Channel* channel, - const string& server, const string& bucket, + const std::string& server, const std::string& bucket, CouchbaseOperations::CouchbaseRequest* request, CouchbaseOperations::CouchbaseResponse* response) { if (channel == nullptr) { @@ -1882,7 +1882,7 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, return false; } if (op_type == CouchbaseOperations::GET) { - string value; + std::string value; uint32_t flags = 0; uint64_t cas = 0; if (response->popGet(&value, &flags, &cas) == false) { @@ -2087,15 +2087,15 @@ bool sendRequest(CouchbaseOperations::operation_type op_type, const string& key, } // Successfully performed the operation // Note: For operations other than GET, we don't have a value to return - // so we return empty string for value. + // so we return empty std::string for value. result->success = true; result->value = ""; result->status_code = 0; return true; } } -CouchbaseOperations::Result CouchbaseOperations::get(const string& key, - string collection_name) { +CouchbaseOperations::Result CouchbaseOperations::get(const std::string& key, + std::string collection_name) { // create CouchbaseRequest and CouchbaseResponse objects and then using the // channel which is created for this thread in authenticate() use it to call() CouchbaseRequest request(&local_bucket_to_collection_manifest_); @@ -2108,7 +2108,7 @@ CouchbaseOperations::Result CouchbaseOperations::get(const string& key, } bool CouchbaseOperations::CouchbaseRequest::getLocalCachedCollectionId( - const string& bucket, const string& scope, const string& collection, + const std::string& bucket, const std::string& scope, const std::string& collection, uint8_t* collection_id) { if (bucket.empty() || scope.empty() || collection.empty()) { DEBUG_PRINT("Bucket, scope, and collection names must be non-empty"); @@ -2138,7 +2138,7 @@ bool CouchbaseOperations::CouchbaseRequest::getLocalCachedCollectionId( } CouchbaseOperations::Result CouchbaseOperations::upsert( - const string& key, const string& value, string collection_name) { + const std::string& key, const std::string& value, std::string collection_name) { CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; @@ -2149,7 +2149,7 @@ CouchbaseOperations::Result CouchbaseOperations::upsert( } CouchbaseOperations::Result CouchbaseOperations::delete_( - const string& key, string collection_name) { + const std::string& key, std::string collection_name) { CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; @@ -2162,9 +2162,9 @@ CouchbaseOperations::Result CouchbaseOperations::delete_( return result; } -CouchbaseOperations::Result CouchbaseOperations::add(const string& key, - const string& value, - string collection_name) { +CouchbaseOperations::Result CouchbaseOperations::add(const std::string& key, + const std::string& value, + std::string collection_name) { CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; @@ -2175,24 +2175,24 @@ CouchbaseOperations::Result CouchbaseOperations::add(const string& key, } CouchbaseOperations::Result CouchbaseOperations::authenticate( - const string& username, const string& password, - const string& server_address, const string& bucket_name) { + const std::string& username, const std::string& password, + const std::string& server_address, const std::string& bucket_name) { return authenticateAll(username, password, server_address, bucket_name, false, ""); } CouchbaseOperations::Result CouchbaseOperations::authenticateSSL( - const string& username, const string& password, - const string& server_address, const string& bucket_name, - string path_to_cert) { + const std::string& username, const std::string& password, + const std::string& server_address, const std::string& bucket_name, + std::string path_to_cert) { return authenticateAll(username, password, server_address, bucket_name, true, path_to_cert); } CouchbaseOperations::Result CouchbaseOperations::authenticateAll( - const string& username, const string& password, - const string& server_address, const string& bucket_name, bool enable_ssl, - string path_to_cert) { + const std::string& username, const std::string& password, + const std::string& server_address, const std::string& bucket_name, bool enable_ssl, + std::string path_to_cert) { // Create a channel to the Couchbase server brpc::ChannelOptions options; options.protocol = brpc::PROTOCOL_COUCHBASE; @@ -2280,7 +2280,7 @@ CouchbaseOperations::Result CouchbaseOperations::authenticateAll( } CouchbaseOperations::Result CouchbaseOperations::selectBucket( - const string& bucket_name) { + const std::string& bucket_name) { CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; @@ -2350,7 +2350,7 @@ CouchbaseOperations::Result CouchbaseOperations::selectBucket( } CouchbaseOperations::Result CouchbaseOperations::append( - const string& key, const string& value, string collection_name) { + const std::string& key, const std::string& value, std::string collection_name) { CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; @@ -2361,7 +2361,7 @@ CouchbaseOperations::Result CouchbaseOperations::append( } CouchbaseOperations::Result CouchbaseOperations::prepend( - const string& key, const string& value, string collection_name) { + const std::string& key, const std::string& value, std::string collection_name) { CouchbaseRequest request(&local_bucket_to_collection_manifest_); CouchbaseResponse response; brpc::Controller cntl; @@ -2391,7 +2391,7 @@ CouchbaseOperations::Result CouchbaseOperations::version() { result.error_message = cntl.ErrorText(); return result; } - string version; + std::string version; if (response.popVersion(&version) == false) { result.success = false; result.value = ""; @@ -2423,9 +2423,9 @@ bool CouchbaseOperations::beginPipeline() { } bool CouchbaseOperations::pipelineRequest(operation_type op_type, - const string& key, - const string& value, - string collection_name) { + const std::string& key, + const std::string& value, + std::string collection_name) { if (!pipeline_active) { DEBUG_PRINT("Pipeline not active. Call beginPipeline() first."); return false; @@ -2486,8 +2486,8 @@ bool CouchbaseOperations::pipelineRequest(operation_type op_type, } return true; } -vector CouchbaseOperations::executePipeline() { - vector results; +std::vector CouchbaseOperations::executePipeline() { + std::vector results; if (!pipeline_active || pipeline_operations_queue.empty()) { DEBUG_PRINT("No pipeline active or no operations queued"); @@ -2525,7 +2525,7 @@ vector CouchbaseOperations::executePipeline() { pipeline_operations_queue.pop(); switch (op_type) { case GET: { - string value; + std::string value; uint32_t flags = 0; uint64_t cas = 0; if (response->popGet(&value, &flags, &cas) == false) { diff --git a/src/brpc/couchbase.h b/src/brpc/couchbase.h index f58c3f00d8..3e52f608d3 100644 --- a/src/brpc/couchbase.h +++ b/src/brpc/couchbase.h @@ -33,7 +33,6 @@ #include "brpc/pb_compat.h" #include "butil/iobuf.h" #include "butil/strings/string_piece.h" -using namespace std; namespace brpc { @@ -119,34 +118,34 @@ class UniqueLock { class CouchbaseManifestManager { public: struct CollectionManifest { - string uid; // uid of the manifest, it can be used to track if the manifest + std::string uid; // uid of the manifest, it can be used to track if the manifest // is updated - unordered_map> + std::unordered_map> scope_to_collection_id_map; // scope -> (collection -> collection_id) }; private: - unordered_map> + std::unordered_map> bucket_to_collection_manifest_; ReaderWriterLock rw_bucket_to_collection_manifest_mutex_; public: CouchbaseManifestManager() {} ~CouchbaseManifestManager() { bucket_to_collection_manifest_.clear(); } - bool setBucketToCollectionManifest(string server, string bucket, + bool setBucketToCollectionManifest(std::string server, std::string bucket, CollectionManifest manifest); - bool getBucketToCollectionManifest(string server, string bucket, + bool getBucketToCollectionManifest(std::string server, std::string bucket, CollectionManifest* manifest); - bool getManifestToCollectionId(CollectionManifest* manifest, string scope, - string collection, uint8_t* collection_id); + bool getManifestToCollectionId(CollectionManifest* manifest, std::string scope, + std::string collection, uint8_t* collection_id); - bool jsonToCollectionManifest(const string& json, + bool jsonToCollectionManifest(const std::string& json, CollectionManifest* manifest); bool refreshCollectionManifest( - brpc::Channel* channel, const string& server, const string& bucket, - unordered_map* local_cache = nullptr); + brpc::Channel* channel, const std::string& server, const std::string& bucket, + std::unordered_map* local_cache = nullptr); } static common_metadata_tracking; class CouchbaseOperations { public: @@ -161,23 +160,23 @@ class CouchbaseOperations { }; struct Result { bool success; - string error_message; - string value; + std::string error_message; + std::string value; uint16_t status_code; // 0x00 if success }; - Result get(const string& key, string collection_name = "_default"); - Result upsert(const string& key, const string& value, - string collection_name = "_default"); - Result add(const string& key, const string& value, - string collection_name = "_default"); + Result get(const std::string& key, std::string collection_name = "_default"); + Result upsert(const std::string& key, const std::string& value, + std::string collection_name = "_default"); + Result add(const std::string& key, const std::string& value, + std::string collection_name = "_default"); // Warning: Not tested - // Result replace(const string& key, const string& value, string + // Result replace(const std::string& key, const std::string& value, std::string // collection_name = "_default"); - Result append(const string& key, const string& value, - string collection_name = "_default"); - Result prepend(const string& key, const string& value, - string collection_name = "_default"); - Result delete_(const string& key, string collection_name = "_default"); + Result append(const std::string& key, const std::string& value, + std::string collection_name = "_default"); + Result prepend(const std::string& key, const std::string& value, + std::string collection_name = "_default"); + Result delete_(const std::string& key, std::string collection_name = "_default"); // Warning: Not tested // Result Increment(const string& key, uint64_t delta, uint64_t initial_value, // uint32_t exptime, string collection_name = "_default"); Result @@ -186,19 +185,19 @@ class CouchbaseOperations { // string& key, uint32_t exptime, string collection_name = "_default"); Result // Flush(uint32_t timeout = 0); Result version(); - Result authenticateSSL(const string& username, const string& password, - const string& server_address, - const string& bucket_name, string path_to_cert = ""); - Result authenticate(const string& username, const string& password, - const string& server_address, const string& bucket_name); - Result selectBucket(const string& bucket_name); + Result authenticateSSL(const std::string& username, const std::string& password, + const std::string& server_address, + const std::string& bucket_name, std::string path_to_cert = ""); + Result authenticate(const std::string& username, const std::string& password, + const std::string& server_address, const std::string& bucket_name); + Result selectBucket(const std::string& bucket_name); // Pipeline management bool beginPipeline(); - bool pipelineRequest(operation_type op_type, const string& key, - const string& value = "", - string collection_name = "_default"); - vector executePipeline(); // Return by value instead of pointer + bool pipelineRequest(operation_type op_type, const std::string& key, + const std::string& value = "", + std::string collection_name = "_default"); + std::vector executePipeline(); // Return by value instead of pointer bool clearPipeline(); // Pipeline status @@ -209,25 +208,25 @@ class CouchbaseOperations { : pipeline_request_couchbase_req(&local_bucket_to_collection_manifest_), pipeline_active(false) {} ~CouchbaseOperations() {} - bool getLocalCachedCollectionId(const string& bucket, const string& scope, - const string& collection, uint8_t* coll_id); + bool getLocalCachedCollectionId(const std::string& bucket, const std::string& scope, + const std::string& collection, uint8_t* coll_id); private: - CouchbaseOperations::Result authenticateAll(const string& username, - const string& password, - const string& server_address, - const string& bucket_name, + CouchbaseOperations::Result authenticateAll(const std::string& username, + const std::string& password, + const std::string& server_address, + const std::string& bucket_name, bool enable_ssl, - string path_to_cert); + std::string path_to_cert); friend void policy::ProcessCouchbaseResponse(InputMessageBase* msg); friend void policy::SerializeCouchbaseRequest( butil::IOBuf* buf, Controller* cntl, const google::protobuf::Message* request); brpc::Channel* channel_; - string server_address_; - string selected_bucket_; + std::string server_address_; + std::string selected_bucket_; - unordered_map + std::unordered_map local_bucket_to_collection_manifest_; public: @@ -253,12 +252,12 @@ class CouchbaseOperations { uint32_t hashCrc32(const char* key, size_t key_length); public: - unordered_map* local_collection_manifest_cache; CouchbaseRequest( - unordered_map* local_cache_reference) : NonreflectableMessage() { @@ -297,83 +296,56 @@ class CouchbaseOperations { bool getScopeId(const butil::StringPiece& scope_name); bool getCollectionManifest(); - - bool getLocalCachedCollectionId(const string& bucket, const string& scope, - const string& collection, uint8_t* coll_id); + + bool getLocalCachedCollectionId(const std::string& bucket, const std::string& scope, + const std::string& collection, uint8_t* coll_id); bool getCachedOrFetchCollectionId( - string collection_name, uint8_t* coll_id, + std::string collection_name, uint8_t* coll_id, brpc::CouchbaseManifestManager* metadata_tracking, - brpc::Channel* channel, const string& server, - const string& selected_bucket, - unordered_map* + brpc::Channel* channel, const std::string& server, + const std::string& selected_bucket, + std::unordered_map* local_cache); // Collection-aware document operations bool getRequest(const butil::StringPiece& key, - string collection_name = "_default", - brpc::Channel* channel = nullptr, const string& server = "", - const string& bucket = ""); + std::string collection_name = "_default", + brpc::Channel* channel = nullptr, const std::string& server = "", + const std::string& bucket = ""); bool upsertRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default", + std::string collection_name = "_default", brpc::Channel* channel = nullptr, - const string& server = "", const string& bucket = ""); + const std::string& server = "", const std::string& bucket = ""); bool addRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default", - brpc::Channel* channel = nullptr, const string& server = "", - const string& bucket = ""); - - // Warning: Not tested - // bool ReplaceRequest(const butil::StringPiece& key, const - // butil::StringPiece& value, - // uint32_t flags, uint32_t exptime, uint64_t cas_value, - // string collection_name = "_default", - // brpc::Channel* channel = nullptr, const string& server = "", - // const string& bucket = ""); + std::string collection_name = "_default", + brpc::Channel* channel = nullptr, const std::string& server = "", + const std::string& bucket = ""); bool appendRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default", + std::string collection_name = "_default", brpc::Channel* channel = nullptr, - const string& server = "", const string& bucket = ""); + const std::string& server = "", const std::string& bucket = ""); bool prependRequest(const butil::StringPiece& key, const butil::StringPiece& value, uint32_t flags, uint32_t exptime, uint64_t cas_value, - string collection_name = "_default", + std::string collection_name = "_default", brpc::Channel* channel = nullptr, - const string& server = "", const string& bucket = ""); + const std::string& server = "", const std::string& bucket = ""); bool deleteRequest(const butil::StringPiece& key, - string collection_name = "_default", + std::string collection_name = "_default", brpc::Channel* channel = nullptr, - const string& server = "", const string& bucket = ""); - - // Warning: Not tested - // bool FlushRequest(uint32_t timeout); - - // bool IncrementRequest(const butil::StringPiece& key, uint64_t delta, - // uint64_t initial_value, uint32_t exptime, string - // collection_name = "_default", brpc::Channel* channel = - // nullptr, const string& server = "", const string& bucket = - // ""); - // bool DecrementRequest(const butil::StringPiece& key, uint64_t delta, - // uint64_t initial_value, uint32_t exptime, string - // collection_name = "_default", brpc::Channel* channel = - // nullptr, const string& server = "", const string& bucket = - // ""); - - // bool TouchRequest(const butil::StringPiece& key, uint32_t exptime, - // string collection_name = "_default", - // brpc::Channel* channel = nullptr, const string& server = "", - // const string& bucket = ""); + const std::string& server = "", const std::string& bucket = ""); bool versionRequest(); @@ -394,7 +366,7 @@ class CouchbaseOperations { static brpc::CouchbaseManifestManager* metadata_tracking; private: - string _err; + std::string _err; butil::IOBuf _buf; mutable int _cached_size_; bool popCounter(uint8_t command, uint64_t* new_value, uint64_t* cas_value); @@ -494,15 +466,15 @@ class CouchbaseOperations { static const char* statusStr(Status); // Helper method to format error messages with status codes - static string formatErrorMessage(uint16_t status_code, - const string& operation, - const string& error_msg = ""); + static std::string formatErrorMessage(uint16_t status_code, + const std::string& operation, + const std::string& error_msg = ""); // Add methods to handle response parsing void swap(CouchbaseResponse* other); bool popGet(butil::IOBuf* value, uint32_t* flags, uint64_t* cas_value); - bool popGet(string* value, uint32_t* flags, uint64_t* cas_value); - const string& lastError() const { return _err; } + bool popGet(std::string* value, uint32_t* flags, uint64_t* cas_value); + const std::string& lastError() const { return _err; } bool popUpsert(uint64_t* cas_value); bool popAdd(uint64_t* cas_value); // Warning: Not tested @@ -524,19 +496,19 @@ class CouchbaseOperations { // bool popIncrement(uint64_t* new_value, uint64_t* cas_value); // bool popDecrement(uint64_t* new_value, uint64_t* cas_value); // bool popTouch(); - bool popVersion(string* version); + bool popVersion(std::string* version); }; friend bool sendRequest(CouchbaseOperations::operation_type op_type, - const string& key, const string& value, - string collection_name, + const std::string& key, const std::string& value, + std::string collection_name, CouchbaseOperations::Result* result, - brpc::Channel* channel, const string& server, - const string& bucket, CouchbaseRequest* request, + brpc::Channel* channel, const std::string& server, + const std::string& bucket, CouchbaseRequest* request, CouchbaseResponse* response); // Pipeline management - per instance - queue pipeline_operations_queue; + std::queue pipeline_operations_queue; CouchbaseRequest pipeline_request_couchbase_req; CouchbaseResponse pipeline_response_couchbase_resp; bool pipeline_active; diff --git a/test/brpc_couchbase_unittest.cpp b/test/brpc_couchbase_unittest.cpp index 317d73bea5..dacc9a09ce 100644 --- a/test/brpc_couchbase_unittest.cpp +++ b/test/brpc_couchbase_unittest.cpp @@ -82,4 +82,4 @@ TEST_F(CouchbaseUnitTest, EdgeCases) { EXPECT_TRUE(true); } -} // namespace +} // namespace \ No newline at end of file From 9d7f9bef0a1dd270544f462c05fdce00b6c06bd7 Mon Sep 17 00:00:00 2001 From: giriraj-singh-couchbase Date: Thu, 13 Nov 2025 00:20:06 +0530 Subject: [PATCH 49/49] restored original CMakeLists.txt --- CMakeLists.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed3d1129f6..c57da24e8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -219,9 +219,7 @@ if(Protobuf_VERSION GREATER 4.21) absl::variant ) else() - # Use C++17 for shared_mutex support and modern features - set(CMAKE_CXX_STANDARD 17) - set(CMAKE_CXX_STANDARD_REQUIRED ON) + use_cxx11() endif() find_package(Threads REQUIRED) @@ -604,4 +602,4 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/output/include/ # Install pkgconfig configure_file(cmake/brpc.pc.in ${PROJECT_BINARY_DIR}/brpc.pc @ONLY) -install(FILES ${PROJECT_BINARY_DIR}/brpc.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +install(FILES ${PROJECT_BINARY_DIR}/brpc.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) \ No newline at end of file