Merge pull request #81 from basho/mv-bloom-size-limit

Mv bloom size limit

Author: Matthew Von-Maszewski

db/db_impl.cc

@@ -315,7 +315,7 @@ DBImpl::KeepOrDelete(
           if (type == kTableFile) {
               // temporary hard coding of extra overlapped
               //  levels
-              table_cache_->Evict(number, (Level<3));
+              table_cache_->Evict(number, (Level<config::kNumOverlapLevels));
           }
           Log(options_.info_log, "Delete type=%d #%lld\n",
               int(type),
@@ -1087,8 +1087,9 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
     imm_micros+=PrioritizeWork(is_level0_compaction);
 
     Slice key = input->key();
-    if (compact->compaction->ShouldStopBefore(key) &&
-        compact->builder != NULL) {
+    if (compact->builder != NULL
+        && compact->compaction->ShouldStopBefore(key, compact->builder->NumEntries())) {
+
       status = FinishCompactionOutputFile(compact, input);
       if (!status.ok()) {
         break;
@@ -1427,7 +1428,10 @@ Status DBImpl::Delete(const WriteOptions& options, const Slice& key) {
 Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
   int throttle;
 
+  // protect use of versions_ ... apply lock
+  mutex_.Lock();
   throttle=versions_->WriteThrottleUsec(bg_compaction_scheduled_);
+  mutex_.Unlock();
   if (0!=throttle)
   {
       /// slowing each call down sequentially
@@ -1436,6 +1440,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
       // throttle is per key write, how many in batch?
       //  (batch multiplier killed AAE, removed)
       env_->SleepForMicroseconds(throttle /* * WriteBatchInternal::Count(my_batch)*/);
+      gPerfCounters->Add(ePerfDebug0, throttle);
   }   // if
 
   Writer w(&mutex_);

db/dbformat.h

@@ -22,12 +22,13 @@ class Compaction;
 // parameters set via options.
 namespace config {
 static const int kNumLevels = 7;
+static const int kNumOverlapLevels = 3;
 
 // Level-0 compaction is started when we hit this many files.
-static const int kL0_CompactionTrigger = 4;
+static const size_t kL0_CompactionTrigger = 4;
 
 // Soft limit on number of level-0 files.  We slow down writes at this point.
-static const int kL0_SlowdownWritesTrigger = 8;
+static const size_t kL0_SlowdownWritesTrigger = 8;
 
 // Maximum number of level-0 files.  We stop writes at this point.
 static const int kL0_StopWritesTrigger = 12;

db/table_cache.cc

@@ -82,7 +82,7 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, int level
 
       // temporary hardcoding to match number of levels defined as
       //  overlapped in version_set.cc
-      if (level<3)
+      if (level<config::kNumOverlapLevels)
           cache_->Addref(*handle);
     }
   }

db/version_set.cc

@@ -31,7 +31,11 @@ static struct
     uint64_t m_MaxGrandParentOverlapBytes;       //!< needs tuning, but not essential
                                                  //!<   since moves eliminated
     int64_t  m_ExpandedCompactionByteSizeLimit;  //!< needs tuning
-    uint64_t m_MaxBytesForLevel;                 //!< ignored if m_OverlappedFiles is true
+
+    // next two ignored if m_OverlappedFiles is true
+    uint64_t m_MaxBytesForLevel;                 //!< start write throttle above this
+    uint64_t m_DesiredBytesForLevel;             //!< compact into next level until this
+
     uint64_t m_MaxFileSizeForLevel;              //!< google really applies this
                                                  //!<   to file size of NEXT level
     bool m_OverlappedFiles;                      //!< false means sst files are sorted
@@ -44,15 +48,18 @@ static struct
 //   to one level-1 file and are each slightly larger than 60,000,000.
 // level-1 file size of 1,500,000,000 applies to output file of this level
 //   being written to level-2.  The value is five times the 300,000,000 of level-1.
+// level-3 is the "landing zone" / first sorted level.  Try to keep it clear.
+//   hence the low m_DesiredBytes for level
 
+// WARNING: m_OverlappedFiles flags need to match config::kNumOverlapFiles ... until unified
 {
-    {10485760,  262144000,  57671680,      209715200, 300000000, true},
-    {10485760,  262144000,  57671680,      419430400,1500000000, true},
-    {10485760,  262144000,  57671680,     4194304000, 31457280, true},
-    {10485760,  262144000,  57671680,     2097152000, 41943040, false},
-    {10485760,  262144000,  57671680,    41943040000, 52428800, false},
-    {10485760,  262144000,  57671680,   419430400000, 62914560, false},
-    {10485760,  262144000,  57671680,  4194304000000, 73400320, false}
+    {10485760,  262144000,  57671680,      209715200,             0,  300000000, true},
+    {10485760,  262144000,  57671680,      419430400,             0, 1500000000, true},
+    {10485760,  262144000,  57671680,     4194304000,             0,   31457280, true},
+    {10485760,  125829120,  57671680,     1610612736,      30000000,   41943040, false},
+    {10485760,  147286400,  57671680,    41943040000,   33554432000,   52428800, false},
+    {10485760,  188743680,  57671680,   419430400000,  335544320000,   62914560, false},
+    {10485760,  220200960,  57671680,  4194304000000, 3355443200000,   73400320, false}
 };
 
 
@@ -544,19 +551,17 @@ Version::VerifyLevels(
         if (!gLevelTraits[level].m_OverlappedFiles && 1<files_[level].size())
         {
             const std::vector<FileMetaData*>& files = files_[level];
-            int inner, outer;
+            size_t inner, outer;
 
             for (outer=0; outer<files.size()-1 && !overlap_found; ++outer)
             {
                 FileMetaData* outer_meta = files_[level][outer];
-                const Slice outer_start = outer_meta->smallest.user_key();
                 const Slice outer_limit = outer_meta->largest.user_key();
 
                 for (inner=outer+1; inner<files.size() && !overlap_found; ++inner)
                 {
                     FileMetaData* inner_meta = files_[level][inner];
                     const Slice inner_start = inner_meta->smallest.user_key();
-                    const Slice inner_limit = inner_meta->largest.user_key();
 
                     // do files overlap? assumes vector sorted by "start"
                     if (user_cmp->Compare(inner_start, outer_limit) <= 0)
@@ -1054,29 +1059,67 @@ void VersionSet::Finalize(Version* v) {
       // file size is small (perhaps because of a small write-buffer
       // setting, or very high compression ratios, or lots of
       // overwrites/deletions).
-      score = v->files_[level].size() /
-          static_cast<double>(config::kL0_CompactionTrigger);
 
-      // don't screw around ... get data written to disk!
-      if (0==level
-          && (size_t)config::kL0_SlowdownWritesTrigger <= v->files_[level].size())
-          score*=1000000.0;
+      score=0;
+
+      // score of 1 at compaction trigger, incrementing for each thereafter
+      if ( config::kL0_CompactionTrigger <= v->files_[level].size())
+          score += v->files_[level].size() - config::kL0_CompactionTrigger +1;
+
+      // raise score above slowdown trigger to ensure this out scores
+      //  compactions at config::kNumOverlapLevels level
+      if ( config::kL0_SlowdownWritesTrigger <= v->files_[level].size())
+          score += (v->files_[level].size() - config::kL0_SlowdownWritesTrigger)*3;
 
       // compute penalty for write throttle if too many Level-0 files accumulating
-      if ((size_t)config::kL0_CompactionTrigger < v->files_[level].size())
+      if (config::kL0_CompactionTrigger < v->files_[level].size())
       {
-          penalty+=v->files_[level].size() - config::kL0_CompactionTrigger;
+          // assume each overlapped file represents another pass at same key
+          //   and we are "close" on compaction backlog
+          if ( v->files_[level].size() < config::kL0_SlowdownWritesTrigger)
+          {
+              penalty+= (v->files_[level].size() - config::kL0_CompactionTrigger);
+          }   // if
+
+          // no longer estimating work, now trying to throw on the breaks
+          //  to keep leveldb from stalling
+          else
+          {
+              int loop, count, value;
+
+              count=(v->files_[level].size() - config::kL0_SlowdownWritesTrigger) +1;
+
+              // logarithmic throttle.  8 works against FusionIO, but 7 or 6 should be tested.
+              for (loop=0, value=8; loop<count; ++loop)
+                  value*=8;
+
+              penalty+=value;
+          }   // else
       }   // if
 
+      // don't screw around ... get data written to disk!
+      if (0==level
+          && (size_t)config::kL0_SlowdownWritesTrigger <= v->files_[level].size())
+          score*=1000000.0;
+      else
+          score*=10;  // give weight to overlapped levels over non-overlapped
+
     } else {
       // Compute the ratio of current size to size limit.
+      double penalty_score;
+
       const uint64_t level_bytes = TotalFileSize(v->files_[level]);
-      score = static_cast<double>(level_bytes) / gLevelTraits[level].m_MaxBytesForLevel;
+      score = static_cast<double>(level_bytes) / gLevelTraits[level].m_DesiredBytesForLevel;
 
-      //  riak 1.4:  new overlapped levels remove the requirement for
-      //    aggressive penalties here, hence the retirement of "*2" and previous "*5".
-      if (2.6<score)
-          penalty+=(static_cast<int>(score))-1;// was *2; // was *5;
+      if (config::kNumOverlapLevels!=level)
+          penalty_score = static_cast<double>(level_bytes) / gLevelTraits[level].m_MaxBytesForLevel;
+
+      // first sort layer needs to clear before next dump of overlapped files.
+      else
+          penalty_score = static_cast<double>(level_bytes) / gLevelTraits[level].m_DesiredBytesForLevel;
+
+      if (1.0<penalty_score)
+          penalty+=(static_cast<int>(penalty_score));
     }
 
     if (score > best_score) {
@@ -1088,8 +1131,9 @@ void VersionSet::Finalize(Version* v) {
   v->compaction_level_ = best_level;
   v->compaction_score_ = best_score;
 
-  if (500<penalty)
-      penalty=500;
+  if (100000<penalty)
+      penalty=100000;
+
   v->write_penalty_ = penalty;
 
 }
@@ -1546,11 +1590,10 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
     return ret_flag;
 }
 
-bool Compaction::ShouldStopBefore(const Slice& internal_key) {
-#if 0
-    // 04/11/2013 code seems to create way too many small files
-    //   in lower levels once highers start to populate
-    // this causes max_open_files to blow out too early
+bool Compaction::ShouldStopBefore(const Slice& internal_key, size_t key_count) {
+
+  // This is a look ahead to see how costly this key will make the subsequent compaction
+  //  of this new file to the next higher level.  Start a new file if the cost is high.
   if (!gLevelTraits[level()+1].m_OverlappedFiles)
   {
     // Scan to find earliest grandparent file that contains key.
@@ -1569,18 +1612,21 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
       // Too much overlap for current output; start new output
       overlapped_bytes_ = 0;
       return true;
-    } else {
-      return false;
-    }
+    } // if
+
+    // Second consideration:  sorted files need to keep the bloom filter size controlled
+    //  to meet file open speed goals
+    else
+    {
+      overlapped_bytes_ = 0;
+      return (75000 < key_count);
+    } // else
   }  // if
   else
   {
     // overlapped levels do NOT split their output file
     return false;
   }
-#else
-  return false;
-#endif
 }
 
 void Compaction::ReleaseInputs() {

db/version_set.h

@@ -357,7 +357,7 @@ class Compaction {
 
   // Returns true iff we should stop building the current output
   // before processing "internal_key".
-  bool ShouldStopBefore(const Slice& internal_key);
+  bool ShouldStopBefore(const Slice& internal_key, size_t key_count);
 
   // Release the input version for the compaction, once the compaction
   // is successful.
@@ -382,7 +382,7 @@ class Compaction {
   std::vector<FileMetaData*> grandparents_;
   size_t grandparent_index_;  // Index in grandparent_starts_
   bool seen_key_;             // Some output key has been seen
-  int64_t overlapped_bytes_;  // Bytes of overlap between current output
+  uint64_t overlapped_bytes_;  // Bytes of overlap between current output
                               // and grandparent files
 
   // State for implementing IsBaseLevelForKey

util/env_posix.cc

@@ -105,9 +105,13 @@ void * arg)
 
             // average write time for level 1+ compactions per key
             //   times the average number of tasks waiting
-            new_throttle=(tot_micros / tot_keys)
-                * (tot_backlog / tot_compact);
+            //   ( the *100 stuff is to exploit fractional data in integers )
+            new_throttle=((tot_micros*100) / tot_keys)
+                * ((tot_backlog*100) / tot_compact);
 
+            new_throttle /= 10000;  // remove *100 stuff
+            if (0==new_throttle)
+                new_throttle=1;     // throttle must have an effect
         }   // if
 
 	// attempt to most recent level0
@@ -122,7 +126,7 @@ void * arg)
 	}   // else if
         else
         {
-            new_throttle=0;
+            new_throttle=1;
         }   // else
 
         // change the throttle slowly
@@ -131,6 +135,9 @@ void * arg)
         else
             gThrottleRate-=(gThrottleRate - new_throttle)/THROTTLE_SCALING;
 
+        if (0==gThrottleRate)
+            gThrottleRate=1;   // throttle must always have an effect
+
         gPerfCounters->Set(ePerfThrottleGauge, gThrottleRate);
         gPerfCounters->Add(ePerfThrottleCounter, gThrottleRate*THROTTLE_SECONDS);
 

util/options.cc

@@ -15,7 +15,7 @@ Options::Options()
     : comparator(BytewiseComparator()),
       create_if_missing(false),
       error_if_exists(false),
-      paranoid_checks(false),
+      paranoid_checks(true),
       env(Env::Default()),
       info_log(NULL),
       write_buffer_size(4<<20),