From 4c2e5862ed339aeabf2600ad194ab75326570553 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Tue, 18 Jun 2024 23:51:49 +0800 Subject: [PATCH 01/16] REFINE --- be/src/vec/common/arena.h | 53 +++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 4ab3ee4c6066e9..91c68eabd03ce2 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -90,6 +90,7 @@ class Arena : private boost::noncopyable { /// Last contiguous chunk of memory. Chunk* head = nullptr; size_t size_in_bytes; + size_t _initial_size = 4096; // The memory used by all chunks, excluding head. size_t _used_size_no_head; @@ -121,6 +122,10 @@ class Arena : private boost::noncopyable { /// Add next contiguous chunk of memory with size not less than specified. void NO_INLINE add_chunk(size_t min_size) { + if (UNLIKELY(head == nullptr)) { + head = new Chunk(min_size, nullptr); + } + _used_size_no_head += head->used(); head = new Chunk(next_size(min_size + pad_right), head); size_in_bytes += head->size(); @@ -135,15 +140,25 @@ class Arena : private boost::noncopyable { size_t linear_growth_threshold_ = 128 * 1024 * 1024) : growth_factor(growth_factor_), linear_growth_threshold(linear_growth_threshold_), - head(new Chunk(initial_size_, nullptr)), size_in_bytes(head->size()), + _initial_size(initial_size_), _used_size_no_head(0) {} - ~Arena() { delete head; } + ~Arena() { + if (head != nullptr) { + delete head; + } + } /// Get piece of memory, without alignment. char* alloc(size_t size) { - if (UNLIKELY(head->pos + size > head->end)) add_chunk(size); + if (UNLIKELY(head == nullptr)) { + head = new Chunk(_initial_size, nullptr); + } + + if (UNLIKELY(head->pos + size > head->end)) { + add_chunk(size); + } char* res = head->pos; head->pos += size; @@ -153,6 +168,10 @@ class Arena : private boost::noncopyable { /// Get piece of memory with alignment char* aligned_alloc(size_t size, size_t alignment) { + if (UNLIKELY(head == nullptr)) { + head = new Chunk(_initial_size, nullptr); + } + do { void* head_pos = head->pos; size_t space = head->end - head->pos; @@ -180,6 +199,10 @@ class Arena : private boost::noncopyable { * the allocation it intended to roll back was indeed the last one. */ void* rollback(size_t size) { + if (UNLIKELY(head == nullptr)) { + head = new Chunk(_initial_size, nullptr); + } + head->pos -= size; ASAN_POISON_MEMORY_REGION(head->pos, size + pad_right); return head->pos; @@ -199,6 +222,10 @@ class Arena : private boost::noncopyable { */ [[nodiscard]] char* alloc_continue(size_t additional_bytes, char const*& range_start, size_t start_alignment = 0) { + if (UNLIKELY(head == nullptr)) { + head = new Chunk(_initial_size, nullptr); + } + if (!range_start) { // Start a new memory range. char* result = start_alignment ? aligned_alloc(additional_bytes, start_alignment) @@ -291,6 +318,10 @@ class Arena : private boost::noncopyable { * and only 128M can be reused when you apply for 4G memory again. */ void clear() { + if (head == nullptr) { + return; + } + if (head->prev) { delete head->prev; head->prev = nullptr; @@ -303,9 +334,21 @@ class Arena : private boost::noncopyable { /// Size of chunks in bytes. size_t size() const { return size_in_bytes; } - size_t used_size() const { return _used_size_no_head + head->used(); } + size_t used_size() const { + if (UNLIKELY(head == nullptr)) { + return 0; + } + + return _used_size_no_head + head->used(); + } - size_t remaining_space_in_current_chunk() const { return head->remaining(); } + size_t remaining_space_in_current_chunk() const { + if (UNLIKELY(head == nullptr)) { + return 0; + } + + return head->remaining(); + } }; using ArenaPtr = std::shared_ptr; From e9a40ad1aa73ddf2889f6e666919b3a88d097d6c Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Wed, 19 Jun 2024 00:05:49 +0800 Subject: [PATCH 02/16] M --- be/src/vec/common/arena.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 91c68eabd03ce2..40be266a783c1d 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -144,11 +144,7 @@ class Arena : private boost::noncopyable { _initial_size(initial_size_), _used_size_no_head(0) {} - ~Arena() { - if (head != nullptr) { - delete head; - } - } + ~Arena() { delete head; } /// Get piece of memory, without alignment. char* alloc(size_t size) { From 5d5aa62d7b72946e7490a3396116504841a363f2 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Wed, 19 Jun 2024 10:29:51 +0800 Subject: [PATCH 03/16] FIX --- be/src/vec/common/arena.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 40be266a783c1d..c8b8a1719543fd 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -89,7 +89,7 @@ class Arena : private boost::noncopyable { /// Last contiguous chunk of memory. Chunk* head = nullptr; - size_t size_in_bytes; + size_t size_in_bytes = 0; size_t _initial_size = 4096; // The memory used by all chunks, excluding head. size_t _used_size_no_head; @@ -140,7 +140,6 @@ class Arena : private boost::noncopyable { size_t linear_growth_threshold_ = 128 * 1024 * 1024) : growth_factor(growth_factor_), linear_growth_threshold(linear_growth_threshold_), - size_in_bytes(head->size()), _initial_size(initial_size_), _used_size_no_head(0) {} From e62f49c68b44c40e25d3bb83ab860cc2e219d32d Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Wed, 19 Jun 2024 10:37:32 +0800 Subject: [PATCH 04/16] FIX --- be/src/vec/common/arena.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index c8b8a1719543fd..beffe7c1dcf679 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -98,7 +98,12 @@ class Arena : private boost::noncopyable { /// If chunks size is less than 'linear_growth_threshold', then use exponential growth, otherwise - linear growth /// (to not allocate too much excessive memory). - size_t next_size(size_t min_next_size) const { + size_t next_size(size_t min_next_size) { + if (UNLIKELY(head == nullptr)) { + head = new Chunk(_initial_size, nullptr); + size_in_bytes += head->size(); + } + size_t size_after_grow = 0; if (head->size() < linear_growth_threshold) { @@ -123,7 +128,8 @@ class Arena : private boost::noncopyable { /// Add next contiguous chunk of memory with size not less than specified. void NO_INLINE add_chunk(size_t min_size) { if (UNLIKELY(head == nullptr)) { - head = new Chunk(min_size, nullptr); + head = new Chunk(_initial_size, nullptr); + size_in_bytes += head->size(); } _used_size_no_head += head->used(); @@ -149,6 +155,7 @@ class Arena : private boost::noncopyable { char* alloc(size_t size) { if (UNLIKELY(head == nullptr)) { head = new Chunk(_initial_size, nullptr); + size_in_bytes += head->size(); } if (UNLIKELY(head->pos + size > head->end)) { @@ -165,6 +172,7 @@ class Arena : private boost::noncopyable { char* aligned_alloc(size_t size, size_t alignment) { if (UNLIKELY(head == nullptr)) { head = new Chunk(_initial_size, nullptr); + size_in_bytes += head->size(); } do { @@ -196,6 +204,7 @@ class Arena : private boost::noncopyable { void* rollback(size_t size) { if (UNLIKELY(head == nullptr)) { head = new Chunk(_initial_size, nullptr); + size_in_bytes += head->size(); } head->pos -= size; @@ -219,6 +228,7 @@ class Arena : private boost::noncopyable { size_t start_alignment = 0) { if (UNLIKELY(head == nullptr)) { head = new Chunk(_initial_size, nullptr); + size_in_bytes += head->size(); } if (!range_start) { From bf5a79c89b1cb1e02406e1b619c6e73ca69e5fc0 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Wed, 19 Jun 2024 10:39:52 +0800 Subject: [PATCH 05/16] FORMAT --- be/src/vec/common/arena.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index beffe7c1dcf679..65d9c5f3893065 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -103,7 +103,7 @@ class Arena : private boost::noncopyable { head = new Chunk(_initial_size, nullptr); size_in_bytes += head->size(); } - + size_t size_after_grow = 0; if (head->size() < linear_growth_threshold) { From 0cfb674d8d170db2fd896ffd294c6dfbd7efa2dc Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 10:41:51 +0800 Subject: [PATCH 06/16] TMP --- be/src/olap/delete_handler.cpp | 1 + be/src/olap/memtable.cpp | 4 ++ be/src/olap/memtable_writer.cpp | 6 ++- .../rowset/segment_v2/binary_dict_page.cpp | 1 + .../rowset/segment_v2/bitmap_index_writer.cpp | 7 ++- .../rowset/segment_v2/bitmap_index_writer.h | 2 + .../olap/rowset/segment_v2/segment_writer.cpp | 3 +- be/src/olap/tablet_reader.cpp | 1 + be/src/pipeline/dependency.cpp | 1 + be/src/pipeline/dependency.h | 1 + be/src/vec/common/arena.h | 51 ++++++------------- tools/tpch-tools/bin/tpch-data | 1 + 12 files changed, 40 insertions(+), 39 deletions(-) create mode 120000 tools/tpch-tools/bin/tpch-data diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index 73a2e3b196772e..e46009dae6ecbe 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -388,6 +388,7 @@ Status DeleteHandler::init(TabletSchemaSPtr tablet_schema, DCHECK(!_is_inited) << "reinitialize delete handler."; DCHECK(version >= 0) << "invalid parameters. version=" << version; _predicate_arena = std::make_unique(); + RETURN_IF_ERROR(_predicate_arena->init()); for (const auto& delete_pred : delete_preds) { // Skip the delete condition with large version diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 750026c289d0b0..eb3b03da67c431 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -28,6 +28,7 @@ #include "bvar/bvar.h" #include "common/config.h" +#include "common/status.h" #include "olap/memtable_memory_limiter.h" #include "olap/olap_define.h" #include "olap/tablet_schema.h" @@ -183,6 +184,7 @@ Status MemTable::insert(const vectorized::Block* input_block, vectorized::Block target_block = *input_block; target_block = input_block->copy_block(_column_offset); if (_is_first_insertion) { + RETURN_IF_ERROR(_arena->init()); _is_first_insertion = false; auto cloneBlock = target_block.clone_without_columns(); _input_mutable_block = vectorized::MutableBlock::build_mutable_block(&cloneBlock); @@ -243,6 +245,8 @@ void MemTable::_aggregate_two_row_in_block(vectorized::MutableBlock& mutable_blo // dst is non-sequence row, or dst sequence is smaller for (uint32_t cid = _tablet_schema->num_key_columns(); cid < _num_columns; ++cid) { auto col_ptr = mutable_block.mutable_columns()[cid].get(); + // What if memory exception happens here? + // Currently, it is safe, see: MemTableWriter::write _agg_functions[cid]->add(dst_row->agg_places(cid), const_cast(&col_ptr), src_row->_row_pos, _arena.get()); diff --git a/be/src/olap/memtable_writer.cpp b/be/src/olap/memtable_writer.cpp index 13bbff325394a3..d0c31f97c5e796 100644 --- a/be/src/olap/memtable_writer.cpp +++ b/be/src/olap/memtable_writer.cpp @@ -26,6 +26,7 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" +#include "common/exception.h" #include "common/logging.h" #include "common/status.h" #include "exec/tablet_info.h" @@ -115,11 +116,12 @@ Status MemTableWriter::write(const vectorized::Block* block, } _total_received_rows += row_idxs.size(); - RETURN_IF_ERROR(_mem_table->insert(block, row_idxs)); + RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_mem_table->insert(block, row_idxs)); if (UNLIKELY(_mem_table->need_agg() && config::enable_shrink_memory)) { - _mem_table->shrink_memtable_by_agg(); + RETURN_IF_CATCH_EXCEPTION(_mem_table->shrink_memtable_by_agg()); } + if (UNLIKELY(_mem_table->need_flush())) { auto s = _flush_memtable_async(); _reset_mem_table(); diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp index dd7ad59ba1df4f..577bd0adaee32b 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp @@ -48,6 +48,7 @@ BinaryDictPageBuilder::BinaryDictPageBuilder(const PageBuilderOptions& options) _encoding_type(DICT_ENCODING) {} Status BinaryDictPageBuilder::init() { + RETURN_IF_ERROR(_arena.init()); // initially use DICT_ENCODING // TODO: the data page builder type can be created by Factory according to user config PageBuilder* data_page_builder_ptr = nullptr; diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp index 227e9140023913..e2837f5e884803 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp @@ -25,6 +25,7 @@ #include #include +#include "common/status.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/encoding_info.h" @@ -64,6 +65,9 @@ struct BitmapIndexTraits { // template class BitmapIndexWriterImpl : public BitmapIndexWriter { +private: + Status init() override { return _arena.init(); } + public: using CppType = typename CppTypeTraits::CppType; using MemoryIndexType = typename BitmapIndexTraits::MemoryIndexType; @@ -257,7 +261,8 @@ Status BitmapIndexWriter::create(const TypeInfo* type_info, return Status::NotSupported("unsupported type for bitmap index: {}", std::to_string(int(type))); } - return Status::OK(); + + return res->get()->init(); } } // namespace segment_v2 diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.h b/be/src/olap/rowset/segment_v2/bitmap_index_writer.h index cf7de5efe96267..707cb553d81338 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.h +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.h @@ -51,6 +51,8 @@ class BitmapIndexWriter { virtual uint64_t size() const = 0; + virtual Status init() = 0; + private: DISALLOW_COPY_AND_ASSIGN(BitmapIndexWriter); }; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 78fd69150c21c6..f03e6d9c715d2c 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -30,6 +30,7 @@ #include "cloud/config.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" +#include "common/exception.h" #include "common/logging.h" // LOG #include "common/status.h" #include "gutil/port.h" @@ -672,7 +673,7 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* // row column should be filled here if (_tablet_schema->store_row_column()) { // convert block to row store format - _serialize_block_to_row_column(full_block); + RETURN_IF_CATCH_EXCEPTION(_serialize_block_to_row_column(full_block)); } // convert missing columns and send to column writer diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index e65a10ac73eccb..f730d25eee3718 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -123,6 +123,7 @@ TabletReader::~TabletReader() { Status TabletReader::init(const ReaderParams& read_params) { _predicate_arena = std::make_unique(); + RETURN_IF_ERROR(_predicate_arena->init()); Status res = _init_params(read_params); if (!res.ok()) { diff --git a/be/src/pipeline/dependency.cpp b/be/src/pipeline/dependency.cpp index 68c00af409dddd..b2ea6f5d3b9e3d 100644 --- a/be/src/pipeline/dependency.cpp +++ b/be/src/pipeline/dependency.cpp @@ -307,6 +307,7 @@ Status AggSharedState::reset_hash_table() { align_aggregate_states)); agg_method.hash_table.reset(new HashTableType()); agg_arena_pool.reset(new vectorized::Arena); + RETURN_IF_ERROR(agg_arena_pool->init()); return Status::OK(); }}, agg_data->method_variant); diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h index 0f9c698a82e601..ad0655e6d0600d 100644 --- a/be/src/pipeline/dependency.h +++ b/be/src/pipeline/dependency.h @@ -623,6 +623,7 @@ struct HashJoinSharedState : public JoinSharedState { std::vector is_null_safe_eq_join; // mark the build hash table whether it needs to store null value std::vector store_null_in_hash_table; + // TODO: need init. std::shared_ptr arena = std::make_shared(); // maybe share hash table with other fragment instances diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 65d9c5f3893065..4596b21ffa9272 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -99,11 +99,7 @@ class Arena : private boost::noncopyable { /// If chunks size is less than 'linear_growth_threshold', then use exponential growth, otherwise - linear growth /// (to not allocate too much excessive memory). size_t next_size(size_t min_next_size) { - if (UNLIKELY(head == nullptr)) { - head = new Chunk(_initial_size, nullptr); - size_in_bytes += head->size(); - } - + DCHECK(head != nullptr); size_t size_after_grow = 0; if (head->size() < linear_growth_threshold) { @@ -127,11 +123,7 @@ class Arena : private boost::noncopyable { /// Add next contiguous chunk of memory with size not less than specified. void NO_INLINE add_chunk(size_t min_size) { - if (UNLIKELY(head == nullptr)) { - head = new Chunk(_initial_size, nullptr); - size_in_bytes += head->size(); - } - + DCHECK(head != nullptr); _used_size_no_head += head->used(); head = new Chunk(next_size(min_size + pad_right), head); size_in_bytes += head->size(); @@ -149,7 +141,10 @@ class Arena : private boost::noncopyable { _initial_size(initial_size_), _used_size_no_head(0) {} - ~Arena() { delete head; } + ~Arena() { + DCHECK(head != nullptr); + delete head; + } /// Get piece of memory, without alignment. char* alloc(size_t size) { @@ -170,10 +165,7 @@ class Arena : private boost::noncopyable { /// Get piece of memory with alignment char* aligned_alloc(size_t size, size_t alignment) { - if (UNLIKELY(head == nullptr)) { - head = new Chunk(_initial_size, nullptr); - size_in_bytes += head->size(); - } + DCHECK(head != nullptr); do { void* head_pos = head->pos; @@ -202,10 +194,7 @@ class Arena : private boost::noncopyable { * the allocation it intended to roll back was indeed the last one. */ void* rollback(size_t size) { - if (UNLIKELY(head == nullptr)) { - head = new Chunk(_initial_size, nullptr); - size_in_bytes += head->size(); - } + DCHECK(head != nullptr); head->pos -= size; ASAN_POISON_MEMORY_REGION(head->pos, size + pad_right); @@ -226,10 +215,7 @@ class Arena : private boost::noncopyable { */ [[nodiscard]] char* alloc_continue(size_t additional_bytes, char const*& range_start, size_t start_alignment = 0) { - if (UNLIKELY(head == nullptr)) { - head = new Chunk(_initial_size, nullptr); - size_in_bytes += head->size(); - } + DCHECK(head != nullptr); if (!range_start) { // Start a new memory range. @@ -323,9 +309,7 @@ class Arena : private boost::noncopyable { * and only 128M can be reused when you apply for 4G memory again. */ void clear() { - if (head == nullptr) { - return; - } + DCHECK(head != nullptr); if (head->prev) { delete head->prev; @@ -337,21 +321,18 @@ class Arena : private boost::noncopyable { } /// Size of chunks in bytes. - size_t size() const { return size_in_bytes; } + size_t size() const { + DCHECK(head != nullptr); + return size_in_bytes; + } size_t used_size() const { - if (UNLIKELY(head == nullptr)) { - return 0; - } - + DCHECK(head != nullptr); return _used_size_no_head + head->used(); } size_t remaining_space_in_current_chunk() const { - if (UNLIKELY(head == nullptr)) { - return 0; - } - + DCHECK(head != nullptr); return head->remaining(); } }; diff --git a/tools/tpch-tools/bin/tpch-data b/tools/tpch-tools/bin/tpch-data new file mode 120000 index 00000000000000..80e8e2db364679 --- /dev/null +++ b/tools/tpch-tools/bin/tpch-data @@ -0,0 +1 @@ +/mnt/disk1/tpch-data \ No newline at end of file From 6c12ef3b9c75fba04164982146ddc78bf2e86040 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 15:20:00 +0800 Subject: [PATCH 07/16] FIX --- be/src/olap/delete_handler.cpp | 1 - be/src/olap/memtable.cpp | 1 - .../rowset/segment_v2/binary_dict_page.cpp | 1 - .../rowset/segment_v2/bitmap_index_writer.cpp | 3 --- .../rowset/segment_v2/bitmap_index_writer.h | 2 -- be/src/olap/tablet_reader.cpp | 1 - be/src/pipeline/dependency.cpp | 1 - be/src/vec/common/arena.h | 24 +++++++++++-------- 8 files changed, 14 insertions(+), 20 deletions(-) diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index e46009dae6ecbe..73a2e3b196772e 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -388,7 +388,6 @@ Status DeleteHandler::init(TabletSchemaSPtr tablet_schema, DCHECK(!_is_inited) << "reinitialize delete handler."; DCHECK(version >= 0) << "invalid parameters. version=" << version; _predicate_arena = std::make_unique(); - RETURN_IF_ERROR(_predicate_arena->init()); for (const auto& delete_pred : delete_preds) { // Skip the delete condition with large version diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index eb3b03da67c431..7bc3bada8df579 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -184,7 +184,6 @@ Status MemTable::insert(const vectorized::Block* input_block, vectorized::Block target_block = *input_block; target_block = input_block->copy_block(_column_offset); if (_is_first_insertion) { - RETURN_IF_ERROR(_arena->init()); _is_first_insertion = false; auto cloneBlock = target_block.clone_without_columns(); _input_mutable_block = vectorized::MutableBlock::build_mutable_block(&cloneBlock); diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp index 577bd0adaee32b..dd7ad59ba1df4f 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp @@ -48,7 +48,6 @@ BinaryDictPageBuilder::BinaryDictPageBuilder(const PageBuilderOptions& options) _encoding_type(DICT_ENCODING) {} Status BinaryDictPageBuilder::init() { - RETURN_IF_ERROR(_arena.init()); // initially use DICT_ENCODING // TODO: the data page builder type can be created by Factory according to user config PageBuilder* data_page_builder_ptr = nullptr; diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp index e2837f5e884803..369f4223735e23 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp @@ -65,9 +65,6 @@ struct BitmapIndexTraits { // template class BitmapIndexWriterImpl : public BitmapIndexWriter { -private: - Status init() override { return _arena.init(); } - public: using CppType = typename CppTypeTraits::CppType; using MemoryIndexType = typename BitmapIndexTraits::MemoryIndexType; diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.h b/be/src/olap/rowset/segment_v2/bitmap_index_writer.h index 707cb553d81338..cf7de5efe96267 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.h +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.h @@ -51,8 +51,6 @@ class BitmapIndexWriter { virtual uint64_t size() const = 0; - virtual Status init() = 0; - private: DISALLOW_COPY_AND_ASSIGN(BitmapIndexWriter); }; diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index f730d25eee3718..e65a10ac73eccb 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -123,7 +123,6 @@ TabletReader::~TabletReader() { Status TabletReader::init(const ReaderParams& read_params) { _predicate_arena = std::make_unique(); - RETURN_IF_ERROR(_predicate_arena->init()); Status res = _init_params(read_params); if (!res.ok()) { diff --git a/be/src/pipeline/dependency.cpp b/be/src/pipeline/dependency.cpp index b2ea6f5d3b9e3d..68c00af409dddd 100644 --- a/be/src/pipeline/dependency.cpp +++ b/be/src/pipeline/dependency.cpp @@ -307,7 +307,6 @@ Status AggSharedState::reset_hash_table() { align_aggregate_states)); agg_method.hash_table.reset(new HashTableType()); agg_arena_pool.reset(new vectorized::Arena); - RETURN_IF_ERROR(agg_arena_pool->init()); return Status::OK(); }}, agg_data->method_variant); diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 4596b21ffa9272..52a3447ce33d8a 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -122,13 +122,20 @@ class Arena : private boost::noncopyable { } /// Add next contiguous chunk of memory with size not less than specified. - void NO_INLINE add_chunk(size_t min_size) { + void NO_INLINE _add_chunk(size_t min_size) { DCHECK(head != nullptr); _used_size_no_head += head->used(); head = new Chunk(next_size(min_size + pad_right), head); size_in_bytes += head->size(); } + void _init_head_if_needed() { + if (UNLIKELY(head == nullptr)) { + head = new Chunk(_initial_size, nullptr); + size_in_bytes += head->size(); + } + } + friend class ArenaAllocator; template friend class AlignedArenaAllocator; @@ -148,13 +155,10 @@ class Arena : private boost::noncopyable { /// Get piece of memory, without alignment. char* alloc(size_t size) { - if (UNLIKELY(head == nullptr)) { - head = new Chunk(_initial_size, nullptr); - size_in_bytes += head->size(); - } + _init_head_if_needed(); if (UNLIKELY(head->pos + size > head->end)) { - add_chunk(size); + _add_chunk(size); } char* res = head->pos; @@ -165,7 +169,7 @@ class Arena : private boost::noncopyable { /// Get piece of memory with alignment char* aligned_alloc(size_t size, size_t alignment) { - DCHECK(head != nullptr); + _init_head_if_needed(); do { void* head_pos = head->pos; @@ -179,7 +183,7 @@ class Arena : private boost::noncopyable { return res; } - add_chunk(size + alignment); + _add_chunk(size + alignment); } while (true); } @@ -215,8 +219,6 @@ class Arena : private boost::noncopyable { */ [[nodiscard]] char* alloc_continue(size_t additional_bytes, char const*& range_start, size_t start_alignment = 0) { - DCHECK(head != nullptr); - if (!range_start) { // Start a new memory range. char* result = start_alignment ? aligned_alloc(additional_bytes, start_alignment) @@ -226,6 +228,8 @@ class Arena : private boost::noncopyable { return result; } + DCHECK(head != nullptr); + // Extend an existing memory range with 'additional_bytes'. // This method only works for extending the last allocation. For lack of From ca5181339f5c44eb9f016e02bc7e01462e9b88da Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 15:22:41 +0800 Subject: [PATCH 08/16] FIX --- be/src/olap/memtable.cpp | 1 - be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp | 3 +-- be/src/pipeline/dependency.h | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 7bc3bada8df579..02c836ef8035fc 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -28,7 +28,6 @@ #include "bvar/bvar.h" #include "common/config.h" -#include "common/status.h" #include "olap/memtable_memory_limiter.h" #include "olap/olap_define.h" #include "olap/tablet_schema.h" diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp index 369f4223735e23..e5af3508ac063b 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp @@ -258,8 +258,7 @@ Status BitmapIndexWriter::create(const TypeInfo* type_info, return Status::NotSupported("unsupported type for bitmap index: {}", std::to_string(int(type))); } - - return res->get()->init(); + return Status::OK(); } } // namespace segment_v2 diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h index ad0655e6d0600d..0f9c698a82e601 100644 --- a/be/src/pipeline/dependency.h +++ b/be/src/pipeline/dependency.h @@ -623,7 +623,6 @@ struct HashJoinSharedState : public JoinSharedState { std::vector is_null_safe_eq_join; // mark the build hash table whether it needs to store null value std::vector store_null_in_hash_table; - // TODO: need init. std::shared_ptr arena = std::make_shared(); // maybe share hash table with other fragment instances From 9fa96d22de009fa023f7d61f656dd8458967e345 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 15:24:43 +0800 Subject: [PATCH 09/16] FIX --- tools/tpch-tools/bin/tpch-data | 1 - 1 file changed, 1 deletion(-) delete mode 120000 tools/tpch-tools/bin/tpch-data diff --git a/tools/tpch-tools/bin/tpch-data b/tools/tpch-tools/bin/tpch-data deleted file mode 120000 index 80e8e2db364679..00000000000000 --- a/tools/tpch-tools/bin/tpch-data +++ /dev/null @@ -1 +0,0 @@ -/mnt/disk1/tpch-data \ No newline at end of file From cc41395bcc2541a53e646fc830d4df1ad4460b3a Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 15:32:02 +0800 Subject: [PATCH 10/16] REFINE --- be/src/olap/memtable.cpp | 2 -- be/src/olap/memtable_writer.cpp | 6 ++---- be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp | 1 - be/src/olap/rowset/segment_v2/segment_writer.cpp | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 02c836ef8035fc..750026c289d0b0 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -243,8 +243,6 @@ void MemTable::_aggregate_two_row_in_block(vectorized::MutableBlock& mutable_blo // dst is non-sequence row, or dst sequence is smaller for (uint32_t cid = _tablet_schema->num_key_columns(); cid < _num_columns; ++cid) { auto col_ptr = mutable_block.mutable_columns()[cid].get(); - // What if memory exception happens here? - // Currently, it is safe, see: MemTableWriter::write _agg_functions[cid]->add(dst_row->agg_places(cid), const_cast(&col_ptr), src_row->_row_pos, _arena.get()); diff --git a/be/src/olap/memtable_writer.cpp b/be/src/olap/memtable_writer.cpp index d0c31f97c5e796..13bbff325394a3 100644 --- a/be/src/olap/memtable_writer.cpp +++ b/be/src/olap/memtable_writer.cpp @@ -26,7 +26,6 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" -#include "common/exception.h" #include "common/logging.h" #include "common/status.h" #include "exec/tablet_info.h" @@ -116,12 +115,11 @@ Status MemTableWriter::write(const vectorized::Block* block, } _total_received_rows += row_idxs.size(); - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_mem_table->insert(block, row_idxs)); + RETURN_IF_ERROR(_mem_table->insert(block, row_idxs)); if (UNLIKELY(_mem_table->need_agg() && config::enable_shrink_memory)) { - RETURN_IF_CATCH_EXCEPTION(_mem_table->shrink_memtable_by_agg()); + _mem_table->shrink_memtable_by_agg(); } - if (UNLIKELY(_mem_table->need_flush())) { auto s = _flush_memtable_async(); _reset_mem_table(); diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp index e5af3508ac063b..227e9140023913 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp @@ -25,7 +25,6 @@ #include #include -#include "common/status.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/encoding_info.h" diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index f03e6d9c715d2c..5d60011660f3f1 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -673,7 +673,7 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* // row column should be filled here if (_tablet_schema->store_row_column()) { // convert block to row store format - RETURN_IF_CATCH_EXCEPTION(_serialize_block_to_row_column(full_block)); + _serialize_block_to_row_column(full_block); } // convert missing columns and send to column writer From 146368dd17bdd9e03b58eb9554b2eddbf6fba09b Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 15:32:47 +0800 Subject: [PATCH 11/16] REFINE --- be/src/olap/rowset/segment_v2/segment_writer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 5d60011660f3f1..78fd69150c21c6 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -30,7 +30,6 @@ #include "cloud/config.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" -#include "common/exception.h" #include "common/logging.h" // LOG #include "common/status.h" #include "gutil/port.h" From ae0e45d00cd7db6909cbada00c6169d56dcc12c1 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 16:58:57 +0800 Subject: [PATCH 12/16] FIX --- be/src/vec/common/arena.h | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 52a3447ce33d8a..4a0eecff720daa 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -149,7 +149,6 @@ class Arena : private boost::noncopyable { _used_size_no_head(0) {} ~Arena() { - DCHECK(head != nullptr); delete head; } From 790df56e58b7a8c3a14fec1b4430e2124062db78 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 17:45:00 +0800 Subject: [PATCH 13/16] FIX --- be/src/vec/common/arena.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 4a0eecff720daa..57c416ac677a41 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -148,9 +148,7 @@ class Arena : private boost::noncopyable { _initial_size(initial_size_), _used_size_no_head(0) {} - ~Arena() { - delete head; - } + ~Arena() { delete head; } /// Get piece of memory, without alignment. char* alloc(size_t size) { @@ -312,7 +310,9 @@ class Arena : private boost::noncopyable { * and only 128M can be reused when you apply for 4G memory again. */ void clear() { - DCHECK(head != nullptr); + if (head == nullptr) { + return; + } if (head->prev) { delete head->prev; From 2ad7bf746cb44cb4505ef8a42de6d4231466179d Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 19:18:40 +0800 Subject: [PATCH 14/16] FIX --- be/src/vec/common/arena.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 57c416ac677a41..916c367f42ebd8 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -325,17 +325,23 @@ class Arena : private boost::noncopyable { /// Size of chunks in bytes. size_t size() const { - DCHECK(head != nullptr); return size_in_bytes; } size_t used_size() const { - DCHECK(head != nullptr); + if (head == nullptr) { + return 0; + } + return _used_size_no_head + head->used(); } size_t remaining_space_in_current_chunk() const { - DCHECK(head != nullptr); + if (head == nullptr) { + return 0; + + } + return head->remaining(); } }; From 8bd3ba1459ec71c037ea85226362c03ae05bae2b Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 19:43:15 +0800 Subject: [PATCH 15/16] FIX --- be/src/vec/common/arena.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index 916c367f42ebd8..e15a5b57d22c86 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -84,15 +84,15 @@ class Arena : private boost::noncopyable { size_t used() const { return pos - begin; } }; - size_t growth_factor; - size_t linear_growth_threshold; + size_t growth_factor = 2; + size_t linear_growth_threshold = 128 * 1024 * 1024; /// Last contiguous chunk of memory. Chunk* head = nullptr; size_t size_in_bytes = 0; size_t _initial_size = 4096; // The memory used by all chunks, excluding head. - size_t _used_size_no_head; + size_t _used_size_no_head = 0; static size_t round_up_to_page_size(size_t s) { return (s + 4096 - 1) / 4096 * 4096; } @@ -330,7 +330,7 @@ class Arena : private boost::noncopyable { size_t used_size() const { if (head == nullptr) { - return 0; + return _used_size_no_head; } return _used_size_no_head + head->used(); From 378028095c275e9a1fe62a0a81ae409048800e9e Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 21 Jun 2024 19:44:33 +0800 Subject: [PATCH 16/16] FORAMT --- be/src/vec/common/arena.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h index e15a5b57d22c86..65e8c1dfabe25b 100644 --- a/be/src/vec/common/arena.h +++ b/be/src/vec/common/arena.h @@ -324,9 +324,7 @@ class Arena : private boost::noncopyable { } /// Size of chunks in bytes. - size_t size() const { - return size_in_bytes; - } + size_t size() const { return size_in_bytes; } size_t used_size() const { if (head == nullptr) { @@ -339,7 +337,6 @@ class Arena : private boost::noncopyable { size_t remaining_space_in_current_chunk() const { if (head == nullptr) { return 0; - } return head->remaining();