From 5f6e25941ba108ffb46d35e889da09c37303f404 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Thu, 6 Jun 2024 13:55:15 -0700 Subject: [PATCH 1/4] Upgrade RocksDB to version 9.2.1 --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 6 +++--- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 6 +++--- .../bookie/storage/ldb/KeyValueStorageRocksDB.java | 7 +++++-- pom.xml | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 64ebd7c5bdc..7d40164a1a6 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -271,7 +271,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar [22] -- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23] +- lib/org.rocksdb-rocksdbjni-9.2.1.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -375,7 +375,7 @@ Apache Software License, Version 2. [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622 -[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2 +[23] Source available at https://github.com/facebook/rocksdb/tree/v9.2.1 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 @@ -635,7 +635,7 @@ This private header is also used by Apple's open source * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-9.2.1.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 0e3f88702a2..3f14f6990f6 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -271,7 +271,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar [22] -- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23] +- lib/org.rocksdb-rocksdbjni-9.2.1.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -371,7 +371,7 @@ Apache Software License, Version 2. [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622 -[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2 +[23] Source available at https://github.com/facebook/rocksdb/tree/v9.2.1 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 @@ -630,7 +630,7 @@ This private header is also used by Apple's open source * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-9.2.1.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index 0c13c1c8bff..916b32cdcc1 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -47,6 +47,7 @@ import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.CompressionType; +import org.rocksdb.ConfigOptions; import org.rocksdb.DBOptions; import org.rocksdb.Env; import org.rocksdb.InfoLogLevel; @@ -154,8 +155,10 @@ private RocksDB initializeRocksDBWithConfFile(String basePath, String subPath, D DBOptions dbOptions = new DBOptions(); final List cfDescs = new ArrayList<>(); final List cfHandles = new ArrayList<>(); - try { - OptionsUtil.loadOptionsFromFile(dbFilePath, Env.getDefault(), dbOptions, cfDescs, false); + try (final ConfigOptions cfgOpts = new ConfigOptions() + .setIgnoreUnknownOptions(false) + .setEnv(Env.getDefault())) { + OptionsUtil.loadOptionsFromFile(cfgOpts, dbFilePath, dbOptions, cfDescs); // Configure file path String logPath = conf.getString(ROCKSDB_LOG_PATH, ""); if (!logPath.isEmpty()) { diff --git a/pom.xml b/pom.xml index 19bfcacf8b1..ae6f7d17d30 100644 --- a/pom.xml +++ b/pom.xml @@ -166,7 +166,7 @@ ${protobuf.version} ${grpc.version} 0.9.11 - 7.10.2 + 9.2.1 3.3.0 2.0.12 2.0 From 1014371cf256f7308f61466fb611779a074a0707 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 18 Jun 2024 12:03:40 -0700 Subject: [PATCH 2/4] trying compatibility setings to support downgrade --- .../TestCompatUpgradeDowngrade.groovy | 10 ++++++++++ .../utils/BookKeeperClusterUtils.java | 18 +++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy index 74e0c9486a4..d27b635046f 100644 --- a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy +++ b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy @@ -102,6 +102,16 @@ class TestCompatUpgradeDowngrade { @Test public void upgradeDowngrade_015() { String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION + BookKeeperClusterUtils.appendToAllBookieConf(docker, currentVersion, + "dbStorage_rocksDB_format_version", + "2") + BookKeeperClusterUtils.appendToAllBookieConf(docker, currentVersion, + "dbStorage_rocksDB_checksum_type", + "kCRC32c") + BookKeeperClusterUtils.appendToAllBookieConf(docker, currentVersion, + "conf/default_rocksdb.conf.default", + "format_version", + "2") upgradeDowngrade("4.17.0", currentVersion) } diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index a6c8e8604f7..d47452447d8 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -143,16 +143,28 @@ public static void updateBookieConf(DockerClient docker, String containerId, DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e", sedProgram, confFile); } - public static void appendToAllBookieConf(DockerClient docker, String version, String key, String value) + public static void appendToAllBookieConf(DockerClient docker, String version, String confFile0, + String key, String value) throws Exception { for (String b : allBookies()) { - appendToBookieConf(docker, b, version, key, value); + appendToBookieConf(docker, b, version, confFile0, key, value); } } + public static void appendToAllBookieConf(DockerClient docker, String version, + String key, String value) + throws Exception { + appendToAllBookieConf(docker, version, "conf/bk_server.conf", key, value); + } + public static void appendToBookieConf(DockerClient docker, String containerId, String version, String key, String value) throws Exception { - String confFile = "/opt/bookkeeper/" + version + "/conf/bk_server.conf"; + appendToBookieConf(docker, containerId, version, "conf/bk_server.conf", key, value); + } + + public static void appendToBookieConf(DockerClient docker, String containerId, + String version, String confFile0, String key, String value) throws Exception { + String confFile = "/opt/bookkeeper/" + version + "/" + confFile0; String sedProgram = String.format("$a%s=%s", key, value); DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e", sedProgram, confFile); } From 3520340fff915e085e05b0fd79f2daa4d9dc68d0 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 18 Jun 2024 15:32:16 -0700 Subject: [PATCH 3/4] another try --- .../TestCompatUpgradeDowngrade.groovy | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy index d27b635046f..fa0a3683636 100644 --- a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy +++ b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy @@ -18,14 +18,12 @@ package org.apache.bookkeeper.tests.backwardcompat import com.github.dockerjava.api.DockerClient -import java.util.concurrent.TimeUnit +import com.google.common.collect.Lists import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader import org.jboss.arquillian.junit.Arquillian import org.jboss.arquillian.test.api.ArquillianResource -import org.junit.AfterClass import org.junit.Assert -import org.junit.BeforeClass import org.junit.FixMethodOrder import org.junit.Test import org.junit.runner.RunWith @@ -61,17 +59,35 @@ class TestCompatUpgradeDowngrade { LOG.info("Setting ledger storage") - for (String version: BookKeeperClusterUtils.OLD_CLIENT_VERSIONS) { + List versions = Lists.newArrayList(BookKeeperClusterUtils.OLD_CLIENT_VERSIONS) + versions.add(BookKeeperClusterUtils.CURRENT_VERSION) + + for (String version: versions) { + if (version.startsWith("4.18.")) { + // rocksDB version 5 or above should be set as default for newer versions + break + } BookKeeperClusterUtils.appendToAllBookieConf(docker, version, "ledgerStorageClass", "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage") + + try { + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + "dbStorage_rocksDB_format_version", + "2") + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + "dbStorage_rocksDB_checksum_type", + "kCRC32c") + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + "conf/default_rocksdb.conf.default", + "format_version", + "2") + } catch (Exception e) { + LOG.warn(version + ": Failed to set rocksdb configs, might be ok for some older version", e) + } } - BookKeeperClusterUtils.appendToAllBookieConf(docker, BookKeeperClusterUtils.CURRENT_VERSION, - "ledgerStorageClass", - "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage") } - // will ignore older non-supported versions @Test @@ -102,16 +118,6 @@ class TestCompatUpgradeDowngrade { @Test public void upgradeDowngrade_015() { String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - BookKeeperClusterUtils.appendToAllBookieConf(docker, currentVersion, - "dbStorage_rocksDB_format_version", - "2") - BookKeeperClusterUtils.appendToAllBookieConf(docker, currentVersion, - "dbStorage_rocksDB_checksum_type", - "kCRC32c") - BookKeeperClusterUtils.appendToAllBookieConf(docker, currentVersion, - "conf/default_rocksdb.conf.default", - "format_version", - "2") upgradeDowngrade("4.17.0", currentVersion) } From f41c27b1c9ba01a8c0509bfab12bd28d615eb926 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 19 Aug 2024 12:11:45 -0700 Subject: [PATCH 4/4] another try --- .../TestCompatUpgradeDowngrade.groovy | 45 ++++++++++++++++--- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy index fa0a3683636..28314eff475 100644 --- a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy +++ b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy @@ -62,26 +62,57 @@ class TestCompatUpgradeDowngrade { List versions = Lists.newArrayList(BookKeeperClusterUtils.OLD_CLIENT_VERSIONS) versions.add(BookKeeperClusterUtils.CURRENT_VERSION) + boolean useRocksDbVersion5 = false + boolean useKxxHash = false for (String version: versions) { - if (version.startsWith("4.18.")) { - // rocksDB version 5 or above should be set as default for newer versions - break - } BookKeeperClusterUtils.appendToAllBookieConf(docker, version, "ledgerStorageClass", "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage") + // versions should be in the increasing order + if (version.startsWith("4.17.")) { + // rocksDB version 5 or above should be set as default for newer versions + useKxxHash = true + } + if (version.startsWith("4.18.")) { + // rocksDB version 5 or above should be set as default for newer versions + useRocksDbVersion5 = true + } + try { BookKeeperClusterUtils.appendToAllBookieConf(docker, version, "dbStorage_rocksDB_format_version", - "2") + useRocksDbVersion5 ? "5" : "2" ) BookKeeperClusterUtils.appendToAllBookieConf(docker, version, "dbStorage_rocksDB_checksum_type", - "kCRC32c") + useKxxHash ? "kxxHash" : "kCRC32c") + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, "conf/default_rocksdb.conf.default", "format_version", - "2") + useRocksDbVersion5 ? "5" : "2") + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + "conf/entry_location_rocksdb.conf.default", + "format_version", + useRocksDbVersion5 ? "5" : "2") + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + "conf/ledger_metadata_rocksdb.conf.default", + "format_version", + useRocksDbVersion5 ? "5" : "2") + + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + "conf/default_rocksdb.conf.default", + "checksum", + useKxxHash ? "kxxHash" : "kCRC32c") + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + "conf/entry_location_rocksdb.conf.default", + "checksum", + useKxxHash ? "kxxHash" : "kCRC32c") + BookKeeperClusterUtils.appendToAllBookieConf(docker, version, + "conf/ledger_metadata_rocksdb.conf.default", + "checksum", + useKxxHash ? "kxxHash" : "kCRC32c") + } catch (Exception e) { LOG.warn(version + ": Failed to set rocksdb configs, might be ok for some older version", e) }