CDPD-78671: HDDS-12193. Provide option to disable RDBStoreMetrics for Snapshotted DB (#7982)

Change-Id: I700aad96cd81c777e5342a82b08baaf6011fd0ed

Author: SaketaChalamchala

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1401,6 +1401,15 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.om.snapshot.rocksdb.metrics.enabled</name>
+    <value>false</value>
+    <tag>OZONE, OM</tag>
+    <description>
+      Skip collecting RocksDBStore metrics for Snapshotted DB.
+    </description>
+  </property>
+
   <property>
     <name>hdds.rest.http-address</name>
     <value>0.0.0.0:9880</value>

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java

@@ -105,6 +105,7 @@ public final class DBStoreBuilder {
   private final DBProfile defaultCfProfile;
   private boolean enableCompactionDag;
   private boolean createCheckpointDirs = true;
+  private boolean enableRocksDbMetrics = true;
   // this is to track the total size of dbUpdates data since sequence
   // number in request to avoid increase in heap memory.
   private long maxDbUpdatesSizeThreshold;
@@ -234,7 +235,7 @@ public DBStore build() throws IOException {
       return new RDBStore(dbFile, rocksDBOption, statistics, writeOptions, tableConfigs,
           registry.build(), openReadOnly, maxFSSnapshots, dbJmxBeanNameName,
           enableCompactionDag, maxDbUpdatesSizeThreshold, createCheckpointDirs,
-          configuration);
+          configuration, enableRocksDbMetrics);
     } finally {
       tableConfigs.forEach(TableConfig::close);
     }
@@ -304,6 +305,11 @@ public DBStoreBuilder setCreateCheckpointDirs(boolean createCheckpointDirs) {
     this.createCheckpointDirs = createCheckpointDirs;
     return this;
   }
+
+  public DBStoreBuilder setEnableRocksDbMetrics(boolean enableRocksDbMetrics) {
+    this.enableRocksDbMetrics = enableRocksDbMetrics;
+    return this;
+  }
   /**
    * Set the {@link ManagedDBOptions} and default
    * {@link ManagedColumnFamilyOptions} based on {@code prof}.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java

@@ -88,7 +88,8 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, ManagedStatistics stati
                   String dbJmxBeanName, boolean enableCompactionDag,
                   long maxDbUpdatesSizeThreshold,
                   boolean createCheckpointDirs,
-                  ConfigurationSource configuration)
+                  ConfigurationSource configuration,
+                  boolean enableRocksDBMetrics)
 
       throws IOException {
     Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
@@ -122,13 +123,18 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, ManagedStatistics stati
         dbJmxBeanName = dbFile.getName();
       }
       // Use statistics instead of dbOptions.statistics() to avoid repeated init.
-      metrics = RocksDBStoreMetrics.create(statistics, db, dbJmxBeanName);
-      if (metrics == null) {
-        LOG.warn("Metrics registration failed during RocksDB init, " +
+      if (!enableRocksDBMetrics) {
+        LOG.debug("Skipped Metrics registration during RocksDB init, " +
             "db path :{}", dbJmxBeanName);
       } else {
-        LOG.debug("Metrics registration succeed during RocksDB init, " +
-            "db path :{}", dbJmxBeanName);
+        metrics = RocksDBStoreMetrics.create(statistics, db, dbJmxBeanName);
+        if (metrics == null) {
+          LOG.warn("Metrics registration failed during RocksDB init, " +
+              "db path :{}", dbJmxBeanName);
+        } else {
+          LOG.debug("Metrics registration succeed during RocksDB init, " +
+              "db path :{}", dbJmxBeanName);
+        }
       }
 
       // Create checkpoints and snapshot directories if not exists.

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java

@@ -60,7 +60,7 @@ public static RDBStore newRDBStore(File dbFile, ManagedDBOptions options,
       throws IOException {
     return new RDBStore(dbFile, options, null, new ManagedWriteOptions(), families,
         CodecRegistry.newBuilder().build(), false, 1000, null, false,
-        maxDbUpdatesSizeThreshold, true, null);
+        maxDbUpdatesSizeThreshold, true, null, true);
   }
 
   public static final int MAX_DB_UPDATES_SIZE_THRESHOLD = 80;

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -137,6 +137,10 @@ private OMConfigKeys() {
   public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
       "ozone.om.save.metrics.interval";
   public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
+  public static final String OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED =
+      "ozone.om.snapshot.rocksdb.metrics.enabled";
+  public static final boolean
+      OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED_DEFAULT = false;
 
   /**
    * OM Ratis related configurations.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -100,6 +100,10 @@
 import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT;
 import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix;
@@ -368,8 +372,10 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
       throws IOException {
     lock = new OmReadOnlyLock();
     omEpoch = 0;
-    setStore(loadDB(conf, dir, name, true,
-        java.util.Optional.of(Boolean.TRUE), Optional.empty()));
+    int maxOpenFiles = conf.getInt(OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES, OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT);
+
+    setStore(loadDB(conf, dir, name, true, Optional.of(Boolean.TRUE),
+        Optional.of(maxOpenFiles), false, false, true));
     initializeOmTables(CacheType.PARTIAL_CACHE, false);
   }
 
@@ -401,8 +407,9 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
         checkSnapshotDirExist(checkpoint);
       }
       setStore(loadDB(conf, metaDir, dbName, false,
-          java.util.Optional.of(Boolean.TRUE),
-          Optional.of(maxOpenFiles), false, false));
+          java.util.Optional.of(Boolean.TRUE), Optional.of(maxOpenFiles), false, false,
+          conf.getBoolean(OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED,
+              OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED_DEFAULT)));
       initializeOmTables(CacheType.PARTIAL_CACHE, false);
     } catch (IOException e) {
       stop();
@@ -551,7 +558,7 @@ public void start(OzoneConfiguration configuration) throws IOException {
   public static DBStore loadDB(OzoneConfiguration configuration, File metaDir)
       throws IOException {
     return loadDB(configuration, metaDir, OM_DB_NAME, false,
-            java.util.Optional.empty(), Optional.empty(), true, true);
+            java.util.Optional.empty(), Optional.empty(), true, true, true);
   }
 
   public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
@@ -561,7 +568,7 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
                                java.util.Optional<Integer> maxOpenFiles)
           throws IOException {
     return loadDB(configuration, metaDir, dbName, readOnly,
-        disableAutoCompaction, maxOpenFiles, true, true);
+        disableAutoCompaction, maxOpenFiles, true, true, true);
   }
 
   @SuppressWarnings("checkstyle:parameternumber")
@@ -571,7 +578,8 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
                                    disableAutoCompaction,
                                java.util.Optional<Integer> maxOpenFiles,
                                boolean enableCompactionDag,
-                               boolean createCheckpointDirs)
+                               boolean createCheckpointDirs,
+                               boolean enableRocksDBMetrics)
       throws IOException {
     final int maxFSSnapshots = configuration.getInt(
         OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT);
@@ -583,7 +591,8 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
         .setPath(Paths.get(metaDir.getPath()))
         .setMaxFSSnapshots(maxFSSnapshots)
         .setEnableCompactionDag(enableCompactionDag)
-        .setCreateCheckpointDirs(createCheckpointDirs);
+        .setCreateCheckpointDirs(createCheckpointDirs)
+        .setEnableRocksDbMetrics(enableRocksDBMetrics);
     disableAutoCompaction.ifPresent(
             dbStoreBuilder::disableDefaultCFAutoCompaction);
     maxOpenFiles.ifPresent(dbStoreBuilder::setMaxNumberOfOpenFiles);

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java

@@ -26,6 +26,7 @@
 import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils;
 import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
@@ -37,6 +38,7 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.event.Level;
 
 import java.io.File;
 import java.io.IOException;
@@ -53,6 +55,7 @@
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
+import java.util.concurrent.TimeoutException;
 
 import static org.apache.commons.io.file.PathUtils.copyDirectory;
 import static org.apache.hadoop.hdds.utils.HAUtils.getExistingSstFiles;
@@ -107,6 +110,8 @@ public void init() throws Exception {
     // Only allow one entry in cache so each new one causes an eviction
     configuration.setInt(
         OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE, 1);
+    configuration.setBoolean(
+        OMConfigKeys.OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED, false);
 
     OmTestManagers omTestManagers = new OmTestManagers(configuration);
     om = omTestManagers.getOzoneManager();
@@ -138,8 +143,12 @@ public void testSnapshotFeatureFlagSafetyCheck() throws IOException {
   }
 
   @Test
-  public void testCloseOnEviction() throws IOException {
+  public void testCloseOnEviction() throws IOException,
+      InterruptedException, TimeoutException {
 
+    GenericTestUtils.setLogLevel(RDBStore.getLogger(), Level.DEBUG);
+    GenericTestUtils.LogCapturer logCapture =
+        GenericTestUtils.LogCapturer.captureLogs(RDBStore.getLogger());
     // set up db tables
     Table<String, OmVolumeArgs> volumeTable = mock(Table.class);
     Table<String, OmBucketInfo> bucketTable = mock(Table.class);
@@ -212,6 +221,12 @@ public void testCloseOnEviction() throws IOException {
 
     // confirm store was closed
     verify(firstSnapshotStore, timeout(3000).times(1)).close();
+
+    // Verify RocksDBStoreMetrics registration is skipped.
+    String msg = "Skipped Metrics registration during RocksDB init";
+    GenericTestUtils.waitFor(() -> {
+      return logCapture.getOutput().contains(msg);
+    }, 100, 30_000);
   }
 
   private void setupData() throws IOException {