branch-2.1.7-rc03 pick: [Opt](merge-on-write) Remove useless delete bitmaps periodically to reduce storage occupation #46696

be/src/common/config.cpp

@@ -1305,6 +1305,8 @@ DEFINE_mInt32(lz4_compression_block_size, "262144");
 
 DEFINE_mBool(enable_pipeline_task_leakage_detect, "false");
 
+DEFINE_mBool(enable_remove_useless_delete_bitmaps, "true");
+
 // clang-format off
 #ifdef BE_TEST
 // test s3

be/src/common/config.h

@@ -1387,6 +1387,8 @@ DECLARE_mInt32(lz4_compression_block_size);
 
 DECLARE_mBool(enable_pipeline_task_leakage_detect);
 
+DECLARE_mBool(enable_remove_useless_delete_bitmaps);
+
 #ifdef BE_TEST
 // test s3
 DECLARE_String(test_s3_resource);

be/src/olap/tablet.cpp

@@ -771,6 +771,30 @@ void Tablet::_delete_stale_rowset_by_version(const Version& version) {
     VLOG_NOTICE << "delete stale rowset. tablet=" << tablet_id() << ", version=" << version;
 }
 
+void Tablet::remove_useless_delete_bitmaps() {
+    std::vector<RowsetId> useless_rowsets;
+    {
+        // hold read lock to find out delete bitmaps to remove
+        std::shared_lock<std::shared_mutex> rlock {_meta_lock};
+        _tablet_meta->delete_bitmap().traverse_rowset_id_prefix([&](const RowsetId& rowset_id) {
+            if (!_contains_rowset(rowset_id)) {
+                // delete bitmaps on these rowsets will never be used
+                useless_rowsets.emplace_back(rowset_id);
+            }
+        });
+    }
+
+    {
+        // hold write lock to remove useless delete bitmaps
+        std::lock_guard<std::shared_mutex> wlock {_meta_lock};
+        for (const auto& rowset_id : useless_rowsets) {
+            _tablet_meta->delete_bitmap().remove_by_rowset_prefix(rowset_id);
+        }
+    }
+    LOG_WARNING("removed useless delete bitmaps on {} rowsets, tablet_id={}",
+                useless_rowsets.size(), tablet_id());
+}
+
 void Tablet::delete_expired_stale_rowset() {
     int64_t now = UnixSeconds();
     // hold write lock while processing stable rowset

be/src/olap/tablet.h

@@ -187,6 +187,8 @@ class Tablet final : public BaseTablet {
     /// need to delete flag.
     void delete_expired_stale_rowset();
 
+    void remove_useless_delete_bitmaps();
+
     // Given spec_version, find a continuous version path and store it in version_path.
     // If quiet is true, then only "does this path exist" is returned.
     // If skip_missing_version is true, return ok even there are missing versions.

be/src/olap/tablet_manager.cpp

@@ -1068,6 +1068,12 @@ Status TabletManager::start_trash_sweep() {
     for_each_tablet([](const TabletSharedPtr& tablet) { tablet->delete_expired_stale_rowset(); },
                     filter_all_tablets);
 
+    if (config::enable_remove_useless_delete_bitmaps) {
+        for_each_tablet(
+                [](const TabletSharedPtr& tablet) { tablet->remove_useless_delete_bitmaps(); },
+                filter_all_tablets);
+    }
+
     std::list<TabletSharedPtr>::iterator last_it;
     {
         std::shared_lock rdlock(_shutdown_tablets_lock);

be/src/olap/tablet_meta.cpp

@@ -27,6 +27,7 @@
 #include <time.h>
 
 #include <cstdint>
+#include <limits>
 #include <set>
 #include <utility>
 
@@ -1124,6 +1125,25 @@ void DeleteBitmap::merge(const DeleteBitmap& other) {
     }
 }
 
+void DeleteBitmap::traverse_rowset_id_prefix(
+        const std::function<void(const RowsetId& rowsetId)>& func) const {
+    auto it = delete_bitmap.cbegin();
+    while (it != delete_bitmap.cend()) {
+        RowsetId rowset_id = std::get<0>(it->first);
+        func(rowset_id);
+        // find next rowset id
+        it = delete_bitmap.upper_bound({rowset_id, std::numeric_limits<SegmentId>::max(),
+                                        std::numeric_limits<Version>::max()});
+    }
+}
+
+void DeleteBitmap::remove_by_rowset_prefix(const RowsetId& rowset_id) {
+    BitmapKey start {rowset_id, 0, 0};
+    BitmapKey end {rowset_id, std::numeric_limits<SegmentId>::max(),
+                   std::numeric_limits<Version>::max()};
+    remove(start, end);
+}
+
 // We cannot just copy the underlying memory to construct a string
 // due to equivalent objects may have different padding bytes.
 // Reading padding bytes is undefined behavior, neither copy nor

be/src/olap/tablet_meta.h

@@ -499,6 +499,10 @@ class DeleteBitmap {
      */
     std::shared_ptr<roaring::Roaring> get_agg(const BitmapKey& bmk) const;
 
+    void traverse_rowset_id_prefix(const std::function<void(const RowsetId& rowsetId)>& func) const;
+
+    void remove_by_rowset_prefix(const RowsetId& rowset_id);
+
     class AggCachePolicy : public LRUCachePolicyTrackingManual {
     public:
         AggCachePolicy(size_t capacity)