From 2be9f9bc00bfa76b3b794977faeaf4ccdf840671 Mon Sep 17 00:00:00 2001 From: lushiji Date: Fri, 21 Jan 2022 17:34:27 +0800 Subject: [PATCH 01/79] improve rocksDB: 1.rocks all old parameter change to be configurable 2.add RateLimiter feature,default closed --- .../storage/ldb/KeyValueStorageRocksDB.java | 79 ++++++++++++++++--- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index bda82725586..aa056bf7b05 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -21,6 +21,9 @@ package org.apache.bookkeeper.bookie.storage.ldb; import static com.google.common.base.Preconditions.checkState; +import static org.rocksdb.RateLimiter.DEFAULT_FAIRNESS; +import static org.rocksdb.RateLimiter.DEFAULT_MODE; +import static org.rocksdb.RateLimiter.DEFAULT_REFILL_PERIOD_MICROS; //CHECKSTYLE.OFF: IllegalImport import io.netty.util.internal.PlatformDependent; @@ -43,6 +46,8 @@ import org.rocksdb.InfoLogLevel; import org.rocksdb.LRUCache; import org.rocksdb.Options; +import org.rocksdb.RateLimiter; +import org.rocksdb.RateLimiterMode; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; @@ -83,6 +88,25 @@ public class KeyValueStorageRocksDB implements KeyValueStorage { private static final String ROCKSDB_NUM_LEVELS = "dbStorage_rocksDB_numLevels"; private static final String ROCKSDB_NUM_FILES_IN_LEVEL0 = "dbStorage_rocksDB_numFilesInLevel0"; private static final String ROCKSDB_MAX_SIZE_IN_LEVEL1_MB = "dbStorage_rocksDB_maxSizeInLevel1MB"; + // add new params for rocksDB + private static final String ROCKSDB_MAX_WRITE_BUFFER_NUMBER = "dbStorage_rocksDB_maxWriteBufferNumber"; + private static final String ROCKSDB_MAX_BACK_GROUND_JOBS = "dbStorage_rocksDB_maxBackgroundJobs"; + private static final String ROCKSDB_RATE_BYTES_PER_SECOND = "dbStorage_rocksDB_rateBytesPerSecond"; + private static final String ROCKSDB_RATE_REFILL_PERIOD_MICROS = "dbStorage_rocksDB_rateRefillPeriodMicros"; + private static final String ROCKSDB_RATE_FAIRNESS = "dbStorage_rocksDB_rateFairness"; + private static final String ROCKSDB_RATELIMITERMODE = "dbStorage_rocksDB_rateLimiterMode"; + private static final String ROCKSDB_INCREASE_PARALLELISM = "dbStorage_rocksDB_increaseParallelism"; + private static final String ROCKSDB_MAX_TOTAL_WAL_SIZE = "dbStorage_rocksDB_maxTotalWalSize"; + private static final String ROCKSDB_MAX_OPEN_FILES = "dbStorage_rocksDB_maxOpenFiles"; + private static final String ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICROS = + "dbStorage_rocksDB_deleteObsoleteFilesPeriodMicros"; + private static final String ROCKSDB_FORMAT_VERSION = "dbStorage_rocksDB_formatVersion"; + private static final String ROCKSDB_CHECKSUM_TYPE = "dbStorage_rocksDB_checksumType"; + private static final String ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS = "dbStorage_rocksDB_cacheIndexAndFilterBlocks"; + private static final String ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES = + "dbStorage_rocksDB_levelCompactionDynamicLevelBytes"; + private static final String ROCKSDB_KEEP_LOG_FILE_NUM = "dbStorage_rocksDB_keepLogFileNum"; + private static final String ROCKSDB_LOG_FILE_TIME_TO_ROLL = "dbStorage_rocksDB_logFileTimeToRoll"; public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbConfigType, ServerConfiguration conf) throws IOException { @@ -123,36 +147,65 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10); boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true); + // add new params for rocksDB + int maxWriteBufferNumber = conf.getInt(ROCKSDB_MAX_WRITE_BUFFER_NUMBER, 4); + int maxBackgroundJobs = conf.getInt(ROCKSDB_MAX_BACK_GROUND_JOBS, 32); + int rateBytesPerSecond = conf.getInt(ROCKSDB_RATE_BYTES_PER_SECOND, -1); + int increaseParallelism = conf.getInt(ROCKSDB_INCREASE_PARALLELISM, 32); + int maxTotalWalSize = conf.getInt(ROCKSDB_MAX_TOTAL_WAL_SIZE, 512 * 1024 * 1024); + int maxOpenFiles = conf.getInt(ROCKSDB_MAX_OPEN_FILES, -1); + long deleteObsoleteFilesPeriodMicros = conf.getLong(ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICROS, + TimeUnit.HOURS.toMicros(1)); + int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 2); + ChecksumType checksumType = ChecksumType.valueOf(conf.getString(ROCKSDB_CHECKSUM_TYPE, + ChecksumType.kxxHash.name())); + boolean cacheIndexAndFilterBlocks = conf.getBoolean(ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS, true); + boolean levelCompactionDynamicLevelBytes = conf.getBoolean(ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES, + true); + if (lz4CompressionEnabled) { options.setCompressionType(CompressionType.LZ4_COMPRESSION); } options.setWriteBufferSize(writeBufferSizeMB * 1024 * 1024); - options.setMaxWriteBufferNumber(4); + options.setMaxWriteBufferNumber(maxWriteBufferNumber); if (numLevels > 0) { options.setNumLevels(numLevels); } options.setLevelZeroFileNumCompactionTrigger(numFilesInLevel0); options.setMaxBytesForLevelBase(maxSizeInLevel1MB * 1024 * 1024); - options.setMaxBackgroundJobs(32); - options.setIncreaseParallelism(32); - options.setMaxTotalWalSize(512 * 1024 * 1024); - options.setMaxOpenFiles(-1); + options.setMaxBackgroundJobs(maxBackgroundJobs); + if (rateBytesPerSecond > 0) { + long refillPeriodMicros = conf.getLong(ROCKSDB_RATE_REFILL_PERIOD_MICROS, + DEFAULT_REFILL_PERIOD_MICROS); + int fairness = conf.getInt(ROCKSDB_RATE_FAIRNESS, + DEFAULT_FAIRNESS); + RateLimiterMode rateLimiterMode = RateLimiterMode.getRateLimiterMode( + conf.getByte(ROCKSDB_RATELIMITERMODE, DEFAULT_MODE.getValue())); + final boolean autoTune = true; + RateLimiter rateLimiter = new RateLimiter(rateBytesPerSecond, refillPeriodMicros, fairness, + rateLimiterMode, autoTune); + options.setRateLimiter(rateLimiter); + } + + options.setIncreaseParallelism(increaseParallelism); + options.setMaxTotalWalSize(maxTotalWalSize); + options.setMaxOpenFiles(maxOpenFiles); options.setTargetFileSizeBase(sstSizeMB * 1024 * 1024); - options.setDeleteObsoleteFilesPeriodMicros(TimeUnit.HOURS.toMicros(1)); + options.setDeleteObsoleteFilesPeriodMicros(deleteObsoleteFilesPeriodMicros); this.cache = new LRUCache(blockCacheSize); BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); tableOptions.setBlockSize(blockSize); tableOptions.setBlockCache(cache); - tableOptions.setFormatVersion(2); - tableOptions.setChecksumType(ChecksumType.kxxHash); + tableOptions.setFormatVersion(formatVersion); + tableOptions.setChecksumType(checksumType); if (bloomFilterBitsPerKey > 0) { tableOptions.setFilterPolicy(new BloomFilter(bloomFilterBitsPerKey, false)); } // Options best suited for HDDs - tableOptions.setCacheIndexAndFilterBlocks(true); - options.setLevelCompactionDynamicLevelBytes(true); + tableOptions.setCacheIndexAndFilterBlocks(cacheIndexAndFilterBlocks); + options.setLevelCompactionDynamicLevelBytes(levelCompactionDynamicLevelBytes); options.setTableFormatConfig(tableOptions); } else { @@ -188,9 +241,11 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo log.warn("Unrecognized RockDB log level: {}", logLevel); } + int keepLogFileNum = conf.getInt(ROCKSDB_KEEP_LOG_FILE_NUM, 30); + long logFileTimeToRoll = conf.getLong(ROCKSDB_LOG_FILE_TIME_TO_ROLL, TimeUnit.DAYS.toSeconds(1)); // Keep log files for 1month - options.setKeepLogFileNum(30); - options.setLogFileTimeToRoll(TimeUnit.DAYS.toSeconds(1)); + options.setKeepLogFileNum(keepLogFileNum); + options.setLogFileTimeToRoll(logFileTimeToRoll); try { if (readOnly) { From 268c825ec9a1f0e61389189cc631d783e0c8081f Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 26 Jan 2022 10:48:45 -0800 Subject: [PATCH 02/79] Auditor should get the LegdgerManagerFactory from the client instance (#3011) * Auditor should get the LegdgerManagerFactory from the client instance * Removed unused import --- .../java/org/apache/bookkeeper/replication/Auditor.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java index 8c69812ff94..cd85145c1b3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java @@ -89,7 +89,6 @@ import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; -import org.apache.bookkeeper.meta.AbstractZkLedgerManagerFactory; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerManager.LedgerRange; import org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator; @@ -480,10 +479,7 @@ public Thread newThread(Runnable r) { private void initialize(ServerConfiguration conf, BookKeeper bkc) throws UnavailableException { try { - LedgerManagerFactory ledgerManagerFactory = AbstractZkLedgerManagerFactory - .newLedgerManagerFactory( - conf, - bkc.getMetadataClientDriver().getLayoutManager()); + LedgerManagerFactory ledgerManagerFactory = bkc.getLedgerManagerFactory(); ledgerManager = ledgerManagerFactory.newLedgerManager(); this.bookieLedgerIndexer = new BookieLedgerIndexer(ledgerManager); @@ -503,7 +499,7 @@ private void initialize(ServerConfiguration conf, BookKeeper bkc) } catch (CompatibilityException ce) { throw new UnavailableException( "CompatibilityException while initializing Auditor", ce); - } catch (IOException | KeeperException ioe) { + } catch (KeeperException ioe) { throw new UnavailableException( "Exception while initializing Auditor", ioe); } catch (InterruptedException ie) { From e089b51ab5e1cf5f061c81463d27f33a21198271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 26 Jan 2022 22:18:24 +0100 Subject: [PATCH 03/79] [build] remove Maven POM files (#3009) * Initial commit for dropping maven * fix gh action * fix typo --- .github/workflows/pr-validation.yml | 2 +- bin/bookkeeper_gradle | 182 --- bin/common.sh | 41 +- bin/common_gradle.sh | 346 ----- bookkeeper-benchmark/bin/benchmark | 14 +- bookkeeper-benchmark/build.gradle | 1 + bookkeeper-benchmark/pom.xml | 85 -- bookkeeper-common-allocator/pom.xml | 60 - bookkeeper-common/pom.xml | 129 -- bookkeeper-dist/all/pom.xml | 159 --- bookkeeper-dist/bkctl/pom.xml | 96 -- bookkeeper-dist/build.gradle | 1 - bookkeeper-dist/pom.xml | 70 - bookkeeper-dist/server/pom.xml | 146 --- bookkeeper-dist/src/assemble/src.xml | 1 - bookkeeper-http/http-server/pom.xml | 48 - bookkeeper-http/pom.xml | 34 - bookkeeper-http/servlet-http-server/pom.xml | 57 - bookkeeper-http/vertx-http-server/pom.xml | 49 - bookkeeper-proto/pom.xml | 66 - bookkeeper-server/build.gradle | 8 +- bookkeeper-server/pom.xml | 336 ----- .../codahale-metrics-provider/pom.xml | 53 - bookkeeper-stats-providers/pom.xml | 33 - .../prometheus-metrics-provider/pom.xml | 72 - bookkeeper-stats/pom.xml | 63 - build.gradle | 8 + buildtools/pom.xml | 28 - circe-checksum/pom.xml | 236 ---- cpu-affinity/pom.xml | 226 ---- dev/check-all-licenses | 6 +- dev/check-all-licenses-gradle | 33 - dev/common.sh | 6 +- dev/docker/ci.sh | 64 - dev/update-snapshot-version.sh | 38 - metadata-drivers/etcd/pom.xml | 147 --- metadata-drivers/pom.xml | 32 - microbenchmarks/pom.xml | 129 -- pom.xml | 1164 ----------------- settings.gradle | 2 + shaded/bookkeeper-server-shaded/pom.xml | 128 -- shaded/bookkeeper-server-tests-shaded/pom.xml | 147 --- shaded/distributedlog-core-shaded/pom.xml | 251 ---- shaded/pom.xml | 34 - stats/pom.xml | 38 - stats/utils/pom.xml | 54 - stream/api/pom.xml | 49 - stream/bin/streamstorage | 11 +- stream/bin/streamstorage-cli | 10 +- stream/bk-grpc-name-resolver/pom.xml | 80 -- stream/clients/java/all/pom.xml | 109 -- stream/clients/java/base/pom.xml | 56 - stream/clients/java/kv/pom.xml | 44 - stream/clients/java/pom.xml | 34 - stream/clients/pom.xml | 32 - stream/common/pom.xml | 77 -- stream/distributedlog/common/pom.xml | 109 -- stream/distributedlog/core/build.gradle | 1 + stream/distributedlog/core/pom.xml | 135 -- stream/distributedlog/io/dlfs/pom.xml | 91 -- stream/distributedlog/io/pom.xml | 31 - stream/distributedlog/pom.xml | 91 -- stream/distributedlog/protocol/pom.xml | 58 - stream/pom.xml | 92 -- stream/proto/pom.xml | 115 -- stream/server/build.gradle | 6 +- stream/server/pom.xml | 79 -- stream/statelib/pom.xml | 169 --- stream/storage/api/pom.xml | 58 - stream/storage/impl/pom.xml | 160 --- stream/storage/pom.xml | 33 - stream/tests-common/pom.xml | 118 -- tests/backward-compat/bc-non-fips/pom.xml | 79 -- .../current-server-old-clients/pom.xml | 32 - .../hierarchical-ledger-manager/pom.xml | 32 - .../backward-compat/hostname-bookieid/pom.xml | 32 - .../old-cookie-new-cluster/pom.xml | 32 - tests/backward-compat/pom.xml | 41 - .../recovery-no-password/pom.xml | 39 - tests/backward-compat/upgrade-direct/pom.xml | 32 - tests/backward-compat/upgrade/pom.xml | 32 - .../yahoo-custom-version/pom.xml | 32 - .../all-released-versions-image/pom.xml | 71 - .../docker-images/all-versions-image/pom.xml | 111 -- .../current-version-image/pom.xml | 147 --- tests/docker-images/pom.xml | 34 - tests/integration-tests-base-groovy/pom.xml | 118 -- tests/integration-tests-base/pom.xml | 98 -- tests/integration-tests-topologies/pom.xml | 49 - tests/integration-tests-utils/pom.xml | 93 -- .../utils/BookKeeperClusterUtils.java | 15 +- tests/integration/cluster/pom.xml | 79 -- tests/integration/pom.xml | 73 -- tests/integration/smoke/pom.xml | 75 -- tests/integration/standalone/pom.xml | 64 - tests/pom.xml | 57 - tests/scripts/pom.xml | 75 -- .../test/bash/gradle/bk_test_bin_common.sh | 16 +- .../bookkeeper-server-shaded-test/pom.xml | 62 - .../pom.xml | 78 -- .../distributedlog-core-shaded-test/pom.xml | 82 -- tests/shaded/pom.xml | 35 - tools/all/build.gradle | 1 + tools/all/pom.xml | 89 -- tools/framework/pom.xml | 44 - tools/ledger/pom.xml | 67 - tools/perf/build.gradle | 1 + tools/perf/pom.xml | 52 - tools/pom.xml | 35 - tools/stream/pom.xml | 52 - 110 files changed, 55 insertions(+), 8872 deletions(-) delete mode 100755 bin/bookkeeper_gradle delete mode 100755 bin/common_gradle.sh delete mode 100644 bookkeeper-benchmark/pom.xml delete mode 100644 bookkeeper-common-allocator/pom.xml delete mode 100644 bookkeeper-common/pom.xml delete mode 100644 bookkeeper-dist/all/pom.xml delete mode 100644 bookkeeper-dist/bkctl/pom.xml delete mode 100644 bookkeeper-dist/pom.xml delete mode 100644 bookkeeper-dist/server/pom.xml delete mode 100644 bookkeeper-http/http-server/pom.xml delete mode 100644 bookkeeper-http/pom.xml delete mode 100644 bookkeeper-http/servlet-http-server/pom.xml delete mode 100644 bookkeeper-http/vertx-http-server/pom.xml delete mode 100644 bookkeeper-proto/pom.xml delete mode 100644 bookkeeper-server/pom.xml delete mode 100644 bookkeeper-stats-providers/codahale-metrics-provider/pom.xml delete mode 100644 bookkeeper-stats-providers/pom.xml delete mode 100644 bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml delete mode 100644 bookkeeper-stats/pom.xml delete mode 100644 buildtools/pom.xml delete mode 100644 circe-checksum/pom.xml delete mode 100644 cpu-affinity/pom.xml delete mode 100755 dev/check-all-licenses-gradle delete mode 100755 dev/docker/ci.sh delete mode 100755 dev/update-snapshot-version.sh delete mode 100644 metadata-drivers/etcd/pom.xml delete mode 100644 metadata-drivers/pom.xml delete mode 100644 microbenchmarks/pom.xml delete mode 100644 pom.xml delete mode 100644 shaded/bookkeeper-server-shaded/pom.xml delete mode 100644 shaded/bookkeeper-server-tests-shaded/pom.xml delete mode 100644 shaded/distributedlog-core-shaded/pom.xml delete mode 100644 shaded/pom.xml delete mode 100644 stats/pom.xml delete mode 100644 stats/utils/pom.xml delete mode 100644 stream/api/pom.xml delete mode 100644 stream/bk-grpc-name-resolver/pom.xml delete mode 100644 stream/clients/java/all/pom.xml delete mode 100644 stream/clients/java/base/pom.xml delete mode 100644 stream/clients/java/kv/pom.xml delete mode 100644 stream/clients/java/pom.xml delete mode 100644 stream/clients/pom.xml delete mode 100644 stream/common/pom.xml delete mode 100644 stream/distributedlog/common/pom.xml delete mode 100644 stream/distributedlog/core/pom.xml delete mode 100644 stream/distributedlog/io/dlfs/pom.xml delete mode 100644 stream/distributedlog/io/pom.xml delete mode 100644 stream/distributedlog/pom.xml delete mode 100644 stream/distributedlog/protocol/pom.xml delete mode 100644 stream/pom.xml delete mode 100644 stream/proto/pom.xml delete mode 100644 stream/server/pom.xml delete mode 100644 stream/statelib/pom.xml delete mode 100644 stream/storage/api/pom.xml delete mode 100644 stream/storage/impl/pom.xml delete mode 100644 stream/storage/pom.xml delete mode 100644 stream/tests-common/pom.xml delete mode 100644 tests/backward-compat/bc-non-fips/pom.xml delete mode 100644 tests/backward-compat/current-server-old-clients/pom.xml delete mode 100644 tests/backward-compat/hierarchical-ledger-manager/pom.xml delete mode 100644 tests/backward-compat/hostname-bookieid/pom.xml delete mode 100644 tests/backward-compat/old-cookie-new-cluster/pom.xml delete mode 100644 tests/backward-compat/pom.xml delete mode 100644 tests/backward-compat/recovery-no-password/pom.xml delete mode 100644 tests/backward-compat/upgrade-direct/pom.xml delete mode 100644 tests/backward-compat/upgrade/pom.xml delete mode 100644 tests/backward-compat/yahoo-custom-version/pom.xml delete mode 100644 tests/docker-images/all-released-versions-image/pom.xml delete mode 100644 tests/docker-images/all-versions-image/pom.xml delete mode 100644 tests/docker-images/current-version-image/pom.xml delete mode 100644 tests/docker-images/pom.xml delete mode 100644 tests/integration-tests-base-groovy/pom.xml delete mode 100644 tests/integration-tests-base/pom.xml delete mode 100644 tests/integration-tests-topologies/pom.xml delete mode 100644 tests/integration-tests-utils/pom.xml delete mode 100644 tests/integration/cluster/pom.xml delete mode 100644 tests/integration/pom.xml delete mode 100644 tests/integration/smoke/pom.xml delete mode 100644 tests/integration/standalone/pom.xml delete mode 100644 tests/pom.xml delete mode 100644 tests/scripts/pom.xml delete mode 100644 tests/shaded/bookkeeper-server-shaded-test/pom.xml delete mode 100644 tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml delete mode 100644 tests/shaded/distributedlog-core-shaded-test/pom.xml delete mode 100644 tests/shaded/pom.xml delete mode 100644 tools/all/pom.xml delete mode 100644 tools/framework/pom.xml delete mode 100644 tools/ledger/pom.xml delete mode 100644 tools/perf/pom.xml delete mode 100644 tools/pom.xml delete mode 100644 tools/stream/pom.xml diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 12ba979c254..f96a401dca3 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -49,4 +49,4 @@ jobs: - name: Validate pull request run: ./gradlew build -x spotbugsTest -x signDistTar -x test - name: Check license files - run: dev/check-all-licenses-gradle + run: dev/check-all-licenses diff --git a/bin/bookkeeper_gradle b/bin/bookkeeper_gradle deleted file mode 100755 index 22545b29cbc..00000000000 --- a/bin/bookkeeper_gradle +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -set -e - -BINDIR=`dirname "$0"` -BK_HOME=`cd ${BINDIR}/..;pwd` - -source ${BK_HOME}/bin/common_gradle.sh - -# default variables -DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf -DEFAULT_ZK_CONF=${BK_HOME}/conf/zookeeper.conf - -if [ -z "$BOOKIE_CONF" ]; then - BOOKIE_CONF_TO_CHECK=${DEFAULT_CONF} -else - BOOKIE_CONF_TO_CHECK=${BOOKIE_CONF} -fi - -FIND_TABLE_SERVICE_RESULT=$(find_table_service ${BOOKIE_CONF_TO_CHECK} $1) - -if [ "x${FIND_TABLE_SERVICE_RESULT}" == "xtrue" ]; then - BOOKIE_MODULE_PATH=stream/server - BOOKIE_MODULE_NAME=${TABLE_SERVICE_MODULE_NAME} -elif [ "x${FIND_TABLE_SERVICE_RESULT}" == "xfalse" ]; then - BOOKIE_MODULE_PATH=bookkeeper-server - BOOKIE_MODULE_NAME=${BOOKIE_SERVER_MODULE_NAME} -else - echo ${FIND_TABLE_SERVICE_RESULT} - exit 1 -fi - -# find the module jar -BOOKIE_JAR=$(find_module_jar ${BOOKIE_MODULE_PATH} ${BOOKIE_MODULE_NAME}) - -# set up the classpath -BOOKIE_CLASSPATH=$(set_module_classpath ${BOOKIE_MODULE_PATH}) - -bookkeeper_help() { - cat < -where command is one of: - -[service commands] - - bookie Run a bookie server - autorecovery Run AutoRecovery service daemon - zookeeper Run zookeeper server - -[development commands] - - localbookie Run a test ensemble of bookies locally - standalone Run a standalone cluster (with all service components) locally - -[tooling commands] - - upgrade Upgrade bookie filesystem - shell Run shell for admin commands - -[other commands] - - help This help message - -or command is the full name of a class with a defined main() method. - -Environment variables: - BOOKIE_LOG_CONF Log4j configuration file (default ${DEFAULT_LOG_CONF}) - BOOKIE_CONF Configuration file (default: ${DEFAULT_CONF}) - BOOKIE_ZK_CONF Configuration file for zookeeper (default: $DEFAULT_ZK_CONF) - BOOKIE_EXTRA_OPTS Extra options to be passed to the jvm - BOOKIE_EXTRA_CLASSPATH Add extra paths to the bookkeeper classpath - ENTRY_FORMATTER_CLASS Entry formatter class to format entries. - BOOKIE_PID_DIR Folder where the Bookie server PID file should be stored - BOOKIE_STOP_TIMEOUT Wait time before forcefully kill the Bookie server instance, if the stop is not successful - -These variable can also be set in conf/bkenv.sh -EOF -} - -# if no args specified, show usage -if [ $# = 0 ]; then - bookkeeper_help; - exit 1; -fi - -# get arguments -COMMAND=$1 -shift - -LOCALBOOKIES_CONFIG_DIR="${LOCALBOOKIES_CONFIG_DIR:-/tmp/localbookies-config}" -if [ ${COMMAND} == "shell" ]; then - DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j.shell.properties - if [[ $1 == "-localbookie" ]]; then - if [[ $2 == *:* ]]; - then - BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/$2.conf - shift 2 - else - BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/baseconf.conf - shift - fi - fi -fi - -if [ -z "$BOOKIE_ZK_CONF" ]; then - BOOKIE_ZK_CONF=$DEFAULT_ZK_CONF -fi - -if [ -z "$BOOKIE_CONF" ]; then - BOOKIE_CONF=${DEFAULT_CONF} -fi - -# Configure logging -if [ -z "$BOOKIE_LOG_CONF" ]; then - BOOKIE_LOG_CONF=${DEFAULT_LOG_CONF} -fi -BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"$BK_HOME/logs"} -BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"bookkeeper-server.log"} -BOOKIE_ROOT_LOGGER=${BOOKIE_ROOT_LOGGER:-"INFO,CONSOLE"} - -# Configure the classpath -BOOKIE_CLASSPATH="$BOOKIE_JAR:$BOOKIE_CLASSPATH:$BOOKIE_EXTRA_CLASSPATH" -BOOKIE_CLASSPATH="`dirname $BOOKIE_LOG_CONF`:$BOOKIE_CLASSPATH" - -# Build the OPTS -BOOKIE_OPTS=$(build_bookie_opts) -GC_OPTS=$(build_bookie_jvm_opts ${BOOKIE_LOG_DIR} "gc_%p.log") -NETTY_OPTS=$(build_netty_opts) -LOGGING_OPTS=$(build_logging_opts ${BOOKIE_LOG_CONF} ${BOOKIE_LOG_DIR} ${BOOKIE_LOG_FILE} ${BOOKIE_ROOT_LOGGER}) - -BOOKIE_EXTRA_OPTS="${BOOKIE_EXTRA_OPTS} -Dorg.bouncycastle.fips.approved_only=true" -OPTS="${OPTS} -cp ${BOOKIE_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${BOOKIE_EXTRA_OPTS}" - -# Create log dir if it doesn't exist -if [ ! -d ${BOOKIE_LOG_DIR} ]; then - mkdir ${BOOKIE_LOG_DIR} -fi - -#Change to BK_HOME to support relative paths -cd "$BK_HOME" -if [ ${COMMAND} == "bookie" ]; then - exec "${JAVA}" ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.server.Main --conf ${BOOKIE_CONF} $@ -elif [ ${COMMAND} == "autorecovery" ]; then - exec "${JAVA}" ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.replication.AutoRecoveryMain --conf ${BOOKIE_CONF} $@ -elif [ ${COMMAND} == "localbookie" ]; then - NUMBER=$1 - shift - exec "${JAVA}" ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.util.LocalBookKeeper ${NUMBER} ${BOOKIE_CONF} $@ -elif [ ${COMMAND} == "standalone" ]; then - exec "${JAVA}" ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.stream.cluster.StandaloneStarter --conf ${BK_HOME}/conf/standalone.conf $@ -elif [ ${COMMAND} == "upgrade" ]; then - exec "${JAVA}" ${OPTS} org.apache.bookkeeper.bookie.FileSystemUpgrade --conf ${BOOKIE_CONF} $@ -elif [ $COMMAND == "zookeeper" ]; then - BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"zookeeper.log"} - exec "${JAVA}" $OPTS -Dbookkeeper.log.file=$BOOKIE_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $BOOKIE_ZK_CONF $@ -elif [ ${COMMAND} == "shell" ]; then - ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}" - exec "${JAVA}" ${OPTS} ${ENTRY_FORMATTER_ARG} org.apache.bookkeeper.bookie.BookieShell -conf ${BOOKIE_CONF} $@ -elif [ ${COMMAND} == "help" ]; then - bookkeeper_help; -else - exec "${JAVA}" ${OPTS} ${COMMAND} $@ -fi - diff --git a/bin/common.sh b/bin/common.sh index a61edcfb064..1fbcd885332 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -155,7 +155,7 @@ is_released_binary() { find_module_jar_at() { DIR=$1 MODULE=$2 - REGEX="^${MODULE}-[0-9\\.]*((-[a-zA-Z]*(-[0-9]*)?)|(-SNAPSHOT))?.jar$" + REGEX="^${MODULE}[-0-9\\.]*((-[a-zA-Z]*(-[0-9]*)?)|(-SNAPSHOT))?.jar$" if [ -d ${DIR} ]; then cd ${DIR} for f in *.jar; do @@ -196,23 +196,13 @@ find_module_jar() { fi if [ -z "${MODULE_JAR}" ]; then - BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/target ${MODULE_NAME}) + BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/build/libs ${MODULE_NAME}) if [ -z "${BUILT_JAR}" ]; then + echo "${BK_HOME}/${MODULE_PATH}/build/libs" >&2; echo "Couldn't find module '${MODULE_NAME}' jar." >&2 - read -p "Do you want me to run \`mvn package -DskipTests\` for you ? (y|n) " answer - case "${answer:0:1}" in - y|Y ) - mkdir -p ${BK_HOME}/logs - output="${BK_HOME}/logs/build.out" - echo "see output at ${output} for the progress ..." >&2 - mvn package -DskipTests &> ${output} - ;; - * ) - exit 1 - ;; - esac - - BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/target ${MODULE_NAME}) + ## read -p "Do you want me to run \`mvn package -DskipTests\` for you ? (y|n) " answer + + BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/build/libs ${MODULE_NAME}) fi if [ -n "${BUILT_JAR}" ]; then MODULE_JAR=${BUILT_JAR} @@ -223,28 +213,19 @@ find_module_jar() { echo "Could not find module '${MODULE_JAR}' jar." >&2 exit 1 fi - echo ${MODULE_JAR} + echo "${MODULE_JAR}" return } add_maven_deps_to_classpath() { MODULE_PATH=$1 - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${BK_HOME}/${MODULE_PATH}/target/cached_classpath.txt" - output="${BK_HOME}/${MODULE_PATH}/target/build_classpath.out" - + f="${BK_HOME}/${MODULE_PATH}/build/classpath.txt" if [ ! -f ${f} ]; then - echo "the classpath of module '${MODULE_PATH}' is not found, generating it ..." >&2 - echo "see output at ${output} for the progress ..." >&2 - ${MVN} -f "${BK_HOME}/${MODULE_PATH}/pom.xml" dependency:build-classpath -Dmdep.outputFile="target/cached_classpath.txt" &> ${output} - echo "the classpath of module '${MODULE_PATH}' is generated at '${f}'." >&2 + echo "no classpath.txt found at ${BK_HOME}/${MODULE_PATH}/build" + exit 1 fi } @@ -258,7 +239,7 @@ set_module_classpath() { echo ${BK_CLASSPATH} else add_maven_deps_to_classpath ${MODULE_PATH} >&2 - cat ${BK_HOME}/${MODULE_PATH}/target/cached_classpath.txt + cat ${BK_HOME}/${MODULE_PATH}/build/classpath.txt fi return } diff --git a/bin/common_gradle.sh b/bin/common_gradle.sh deleted file mode 100755 index ec704499468..00000000000 --- a/bin/common_gradle.sh +++ /dev/null @@ -1,346 +0,0 @@ -#!/usr/bin/env bash -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Check net.ipv6.bindv6only -if [ -f /sbin/sysctl ] && [ -f /proc/sys/net/ipv6/bindv6only ]; then - # check if net.ipv6.bindv6only is set to 1 - bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) - if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] - then - echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" - echo "For more info (the following page also applies to bookkeeper): http://wiki.apache.org/hadoop/HadoopIPv6" - exit 1 - fi -fi - -# See the following page for extensive details on setting -# up the JVM to accept JMX remote management: -# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# by default we allow local JMX connections -if [ "x$JMXLOCALONLY" = "x" ] -then - JMXLOCALONLY=false -fi - -if [ "x$JMXDISABLE" = "x" ] -then - # for some reason these two options are necessary on jdk6 on Ubuntu - # accord to the docs they are not necessary, but otw jconsole cannot - # do a local attach - JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" -else - echo "JMX disabled by user request" >&2 -fi - -# Check for the java to use -if [[ -z ${JAVA_HOME} ]]; then - JAVA=$(which java) - if [ $? = 0 ]; then - echo "JAVA_HOME not set, using java from PATH. ($JAVA)" - else - echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 - exit 1 - fi -else - JAVA=${JAVA_HOME}/bin/java -fi - -BINDIR=${BK_BINDIR:-"`dirname "$0"`"} -BK_HOME=${BK_HOME:-"`cd ${BINDIR}/..;pwd`"} -BK_CONFDIR=${BK_HOME}/conf -DEFAULT_LOG_CONF=${BK_CONFDIR}/log4j.properties - -source ${BK_CONFDIR}/nettyenv.sh -source ${BK_CONFDIR}/bkenv.sh -source ${BK_CONFDIR}/bk_cli_env.sh - -detect_jdk8() { - - if [ -f "$JAVA_HOME/lib/modules" ]; then - echo "0" - else - echo "1" - fi - return -} - -# default netty settings -NETTY_LEAK_DETECTION_LEVEL=${NETTY_LEAK_DETECTION_LEVEL:-"disabled"} -NETTY_RECYCLER_MAXCAPACITY=${NETTY_RECYCLER_MAXCAPACITY:-"1000"} -NETTY_RECYCLER_LINKCAPACITY=${NETTY_RECYCLER_LINKCAPACITY:-"1024"} - -USING_JDK8=$(detect_jdk8) - -if [ "$USING_JDK8" -ne "1" ]; then - DEFAULT_BOOKIE_GC_OPTS="-XX:+UseG1GC \ - -XX:MaxGCPauseMillis=10 \ - -XX:+ParallelRefProcEnabled \ - -XX:+DisableExplicitGC" - DEFAULT_BOOKIE_GC_LOGGING_OPTS="" -else - DEFAULT_BOOKIE_GC_OPTS="-XX:+UseG1GC \ - -XX:MaxGCPauseMillis=10 \ - -XX:+ParallelRefProcEnabled \ - -XX:+UnlockExperimentalVMOptions \ - -XX:+AggressiveOpts \ - -XX:+DoEscapeAnalysis \ - -XX:ParallelGCThreads=32 \ - -XX:ConcGCThreads=32 \ - -XX:G1NewSizePercent=50 \ - -XX:+DisableExplicitGC \ - -XX:-ResizePLAB" - DEFAULT_BOOKIE_GC_LOGGING_OPTS="-XX:+PrintGCDetails \ - -XX:+PrintGCApplicationStoppedTime \ - -XX:+UseGCLogFileRotation \ - -XX:NumberOfGCLogFiles=5 \ - -XX:GCLogFileSize=64m" -fi - -BOOKIE_MAX_HEAP_MEMORY=${BOOKIE_MAX_HEAP_MEMORY:-"1g"} -BOOKIE_MIN_HEAP_MEMORY=${BOOKIE_MIN_HEAP_MEMORY:-"1g"} -BOOKIE_MAX_DIRECT_MEMORY=${BOOKIE_MAX_DIRECT_MEMORY:-"2g"} -BOOKIE_MEM_OPTS=${BOOKIE_MEM_OPTS:-"-Xms${BOOKIE_MIN_HEAP_MEMORY} -Xmx${BOOKIE_MAX_HEAP_MEMORY} -XX:MaxDirectMemorySize=${BOOKIE_MAX_DIRECT_MEMORY}"} -BOOKIE_GC_OPTS=${BOOKIE_GC_OPTS:-"${DEFAULT_BOOKIE_GC_OPTS}"} -BOOKIE_GC_LOGGING_OPTS=${BOOKIE_GC_LOGGING_OPTS:-"${DEFAULT_BOOKIE_GC_LOGGING_OPTS}"} - -# default CLI JVM settings -DEFAULT_CLI_GC_OPTS="-XX:+UseG1GC \ - -XX:MaxGCPauseMillis=10" -if [ "$USING_JDK8" -ne "1" ]; then - DEFAULT_CLI_GC_LOGGING_OPTS="" -else - DEFAULT_CLI_GC_LOGGING_OPTS="-XX:+PrintGCDetails \ - -XX:+PrintGCApplicationStoppedTime \ - -XX:+UseGCLogFileRotation \ - -XX:NumberOfGCLogFiles=5 \ - -XX:GCLogFileSize=64m" -fi - -CLI_MAX_HEAP_MEMORY=${CLI_MAX_HEAP_MEMORY:-"512M"} -CLI_MIN_HEAP_MEMORY=${CLI_MIN_HEAP_MEMORY:-"256M"} -CLI_MEM_OPTS=${CLI_MEM_OPTS:-"-Xms${CLI_MIN_HEAP_MEMORY} -Xmx${CLI_MAX_HEAP_MEMORY}"} -CLI_GC_OPTS=${CLI_GC_OPTS:-"${DEFAULT_CLI_GC_OPTS}"} -CLI_GC_LOGGING_OPTS=${CLI_GC_LOGGING_OPTS:-"${DEFAULT_CLI_GC_LOGGING_OPTS}"} - -# module names -BOOKIE_SERVER_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server" -TABLE_SERVICE_MODULE_NAME="(org.apache.bookkeeper-)?stream-storage-server" - -is_released_binary() { - if [ -d ${BK_HOME}/lib ]; then - echo "true" - return - else - echo "false" - return - fi -} - -find_module_jar_at() { - DIR=$1 - MODULE=$2 - REGEX="^${MODULE}[-0-9\\.]*((-[a-zA-Z]*(-[0-9]*)?)|(-SNAPSHOT))?.jar$" - if [ -d ${DIR} ]; then - cd ${DIR} - for f in *.jar; do - if [[ ${f} =~ ${REGEX} ]]; then - echo ${DIR}/${f} - return - fi - done - fi -} - -find_module_release_jar() { - MODULE_NAME=$1 - RELEASE_JAR=$(find_module_jar_at ${BK_HOME} ${MODULE_NAME}) - if [ -n "${RELEASE_JAR}" ]; then - MODULE_JAR=${RELEASE_JAR} - else - RELEASE_JAR=$(find_module_jar_at ${BK_HOME}/lib ${MODULE_NAME}) - if [ -n "${RELEASE_JAR}" ]; then - MODULE_JAR=${RELEASE_JAR} - fi - fi - echo ${RELEASE_JAR} - return -} - -find_module_jar() { - MODULE_PATH=$1 - MODULE_NAME=$2 - RELEASE_JAR=$(find_module_jar_at ${BK_HOME} ${MODULE_NAME}) - if [ -n "${RELEASE_JAR}" ]; then - MODULE_JAR=${RELEASE_JAR} - else - RELEASE_JAR=$(find_module_jar_at ${BK_HOME}/lib ${MODULE_NAME}) - if [ -n "${RELEASE_JAR}" ]; then - MODULE_JAR=${RELEASE_JAR} - fi - fi - - if [ -z "${MODULE_JAR}" ]; then - BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/build/libs ${MODULE_NAME}) - if [ -z "${BUILT_JAR}" ]; then - echo "${BK_HOME}/${MODULE_PATH}/build/libs" >&2; - echo "Couldn't find module '${MODULE_NAME}' jar." >&2 - ## read -p "Do you want me to run \`mvn package -DskipTests\` for you ? (y|n) " answer - - BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/build/libs ${MODULE_NAME}) - fi - if [ -n "${BUILT_JAR}" ]; then - MODULE_JAR=${BUILT_JAR} - fi - fi - - if [ ! -e "${MODULE_JAR}" ]; then - echo "Could not find module '${MODULE_JAR}' jar." >&2 - exit 1 - fi - echo "${MODULE_JAR}" - return -} - -add_maven_deps_to_classpath() { - MODULE_PATH=$1 - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${BK_HOME}/${MODULE_PATH}/build/classpath.txt" - if [ ! -f ${f} ]; then - echo "the classpath of module '${MODULE_PATH}' is not found, generating it ..." >&2 - ${BK_HOME}/gradlew ${MODULE_PATH}:jar - echo "the classpath of module '${MODULE_PATH}' is generated at '${f}'." >&2 - fi -} - -set_module_classpath() { - MODULE_PATH=$1 - if [ -d "${BK_HOME}/lib" ]; then - BK_CLASSPATH="" - for i in ${BK_HOME}/lib/*.jar; do - BK_CLASSPATH=${BK_CLASSPATH}:${i} - done - echo ${BK_CLASSPATH} - else - add_maven_deps_to_classpath ${MODULE_PATH} >&2 - cat ${BK_HOME}/${MODULE_PATH}/build/classpath.txt - fi - return -} - -build_bookie_jvm_opts() { - LOG_DIR=$1 - GC_LOG_FILENAME=$2 - if [ "$USING_JDK8" -eq "1" ]; then - echo "$BOOKIE_MEM_OPTS $BOOKIE_GC_OPTS $BOOKIE_GC_LOGGING_OPTS $BOOKIE_PERF_OPTS -Xloggc:${LOG_DIR}/${GC_LOG_FILENAME}" - else - echo "$BOOKIE_MEM_OPTS $BOOKIE_GC_OPTS $BOOKIE_GC_LOGGING_OPTS $BOOKIE_PERF_OPTS -Xlog:gc=info:file=${LOG_DIR}/${GC_LOG_FILENAME}::filecount=5,filesize=64m" - fi - return -} - -build_cli_jvm_opts() { - LOG_DIR=$1 - GC_LOG_FILENAME=$2 - if [ "$USING_JDK8" -eq "1" ]; then - echo "$CLI_MEM_OPTS $CLI_GC_OPTS $CLI_GC_LOGGING_OPTS -Xloggc:${LOG_DIR}/${GC_LOG_FILENAME}" - else - echo "$CLI_MEM_OPTS $CLI_GC_OPTS $CLI_GC_LOGGING_OPTS -Xlog:gc=info:file=${LOG_DIR}/${GC_LOG_FILENAME}::filecount=5,filesize=64m" - fi - return -} - -build_netty_opts() { - echo "-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL} \ - -Dio.netty.recycler.maxCapacity.default=${NETTY_RECYCLER_MAXCAPACITY} \ - -Dio.netty.recycler.linkCapacity=${NETTY_RECYCLER_LINKCAPACITY}" -} - -build_logging_opts() { - CONF_FILE=$1 - LOG_DIR=$2 - LOG_FILE=$3 - LOGGER=$4 - - echo "-Dlog4j.configuration=`basename ${CONF_FILE}` \ - -Dbookkeeper.root.logger=${LOGGER} \ - -Dbookkeeper.log.dir=${LOG_DIR} \ - -Dbookkeeper.log.file=${LOG_FILE}" -} - -build_cli_logging_opts() { - CONF_FILE=$1 - LOG_DIR=$2 - LOG_FILE=$3 - LOGGER=$4 - - echo "-Dlog4j.configuration=`basename ${CONF_FILE}` \ - -Dbookkeeper.cli.root.logger=${LOGGER} \ - -Dbookkeeper.cli.log.dir=${LOG_DIR} \ - -Dbookkeeper.cli.log.file=${LOG_FILE}" -} - -build_bookie_opts() { - echo "-Djava.net.preferIPv4Stack=true" -} - -find_table_service() { - BOOKIE_CONF_TO_CHECK=$1 - SERVICE_COMMAND=$2 - - # check if it is a released binary - IS_RELEASED_BINARY=$(is_released_binary) - - # check if table service is released - TABLE_SERVICE_RELEASED="true" - if [ "x${IS_RELEASED_BINARY}" == "xtrue" ]; then - TABLE_SERVICE_RELEASE_JAR=$(find_module_release_jar ${TABLE_SERVICE_MODULE_NAME}) - if [ "x${TABLE_SERVICE_RELEASE_JAR}" == "x" ]; then - TABLE_SERVICE_RELEASED="false" - fi - fi - - # check the configuration to see if table service is enabled or not. - if [ -z "${ENABLE_TABLE_SERVICE}" ]; then - # mask exit code if the configuration file doesn't contain `StreamStorageLifecycleComponent` - TABLE_SERVICE_SETTING=$(grep StreamStorageLifecycleComponent ${BOOKIE_CONF_TO_CHECK} | cat) - if [[ "${TABLE_SERVICE_SETTING}" =~ ^extraServerComponents.* ]]; then - if [ "x${TABLE_SERVICE_RELEASED}" == "xfalse" ]; then - echo "The release binary is built without table service. Please disable \`StreamStorageLifecycleComponent\` in your bookie configuration at '${BOOKIE_CONF_TO_CHECK}'." - return - fi - ENABLE_TABLE_SERVICE="true" - fi - fi - - # standalone only run - if [ \( "x${SERVICE_COMMAND}" == "xstandalone" \) -a \( "x${TABLE_SERVICE_RELEASED}" == "xfalse" \) ]; then - echo "The release binary is built without table service. Use \`localbookie \` instead of \`standalone\` for local development." - return - fi - - if [ \( "x${SERVICE_COMMAND}" == "xstandalone" \) -o \( "x${ENABLE_TABLE_SERVICE}" == "xtrue" \) ]; then - echo "true" - return - else - echo "false" - return - fi -} diff --git a/bookkeeper-benchmark/bin/benchmark b/bookkeeper-benchmark/bin/benchmark index 3158aac140f..72753f1931d 100755 --- a/bookkeeper-benchmark/bin/benchmark +++ b/bookkeeper-benchmark/bin/benchmark @@ -31,6 +31,7 @@ fi BINDIR=`dirname "$0"` BENCH_HOME=`cd $BINDIR/..;pwd` +BK_HOME=${BK_HOME:-"`cd ${BINDIR}/../..;pwd`"} RELEASE_JAR=`ls $BENCH_HOME/bookkeeper-benchmark-*.jar 2> /dev/null | tail -1` if [ $? == 0 ]; then @@ -69,18 +70,11 @@ EOF } add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${BENCH_HOME}/target/cached_classpath.txt" + f="${BENCH_HOME}/build/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${BENCH_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + echo "no classpath.txt found at ${BENCH_HOME}/build" + exit 1 fi BENCHMARK_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/bookkeeper-benchmark/build.gradle b/bookkeeper-benchmark/build.gradle index 3dec69e5229..d0089873813 100644 --- a/bookkeeper-benchmark/build.gradle +++ b/bookkeeper-benchmark/build.gradle @@ -74,5 +74,6 @@ test { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-benchmark' } diff --git a/bookkeeper-benchmark/pom.xml b/bookkeeper-benchmark/pom.xml deleted file mode 100644 index fc7ed6f5d47..00000000000 --- a/bookkeeper-benchmark/pom.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper - bookkeeper-benchmark - Apache BookKeeper :: Benchmark - - UTF-8 - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - target/latencyDump.dat - - - - - - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - compile - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - diff --git a/bookkeeper-common-allocator/pom.xml b/bookkeeper-common-allocator/pom.xml deleted file mode 100644 index 31315773665..00000000000 --- a/bookkeeper-common-allocator/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-common-allocator - Apache BookKeeper :: Common :: Allocator - - - io.netty - netty-buffer - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml deleted file mode 100644 index e7709f4597a..00000000000 --- a/bookkeeper-common/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-common - Apache BookKeeper :: Common - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - org.apache.bookkeeper - cpu-affinity - ${project.parent.version} - - - com.google.guava - guava - - - io.netty - netty-common - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - org.jctools - jctools-core - - - com.google.code.findbugs - jsr305 - provided - - - com.google.errorprone - error_prone_annotations - provided - - - org.apache.commons - commons-lang3 - test - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - diff --git a/bookkeeper-dist/all/pom.xml b/bookkeeper-dist/all/pom.xml deleted file mode 100644 index 146dafa3271..00000000000 --- a/bookkeeper-dist/all/pom.xml +++ /dev/null @@ -1,159 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bookkeeper-dist-all - jar - Apache BookKeeper :: Dist (All) - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - - org.apache.bookkeeper.stats - codahale-metrics-provider - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper.http - vertx-http-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-benchmark - ${project.version} - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - org.xerial.snappy - snappy-java - - - - io.dropwizard.metrics - metrics-core - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-all-${project.version} - false - - ../src/assemble/bin-all.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/bookkeeper-dist/bkctl/pom.xml b/bookkeeper-dist/bkctl/pom.xml deleted file mode 100644 index 8ac6545851e..00000000000 --- a/bookkeeper-dist/bkctl/pom.xml +++ /dev/null @@ -1,96 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bkctl - jar - Apache BookKeeper :: Dist (Bkctl) - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - org.rocksdb - rocksdbjni - - - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bkctl-${project.version} - true - - ../src/assemble/bkctl.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/bookkeeper-dist/build.gradle b/bookkeeper-dist/build.gradle index e8c086821da..aeac682345a 100644 --- a/bookkeeper-dist/build.gradle +++ b/bookkeeper-dist/build.gradle @@ -38,7 +38,6 @@ distributions { "**/README.md", "**/bin/**", "**/conf/**", - "**/pom.xml", "**/src/**", "deploy/**", "doc/**", diff --git a/bookkeeper-dist/pom.xml b/bookkeeper-dist/pom.xml deleted file mode 100644 index 5285131dbcb..00000000000 --- a/bookkeeper-dist/pom.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - bookkeeper-dist - pom - Apache BookKeeper :: Dist (Parent) - - all - server - bkctl - - - UTF-8 - UTF-8 - - - - - maven-jar-plugin - ${maven-jar-plugin.version} - - - default-jar - none - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-${project.version} - - src/assemble/src.xml - - posix - - - - package - - single - - - - - - - diff --git a/bookkeeper-dist/server/pom.xml b/bookkeeper-dist/server/pom.xml deleted file mode 100644 index 83777f5f342..00000000000 --- a/bookkeeper-dist/server/pom.xml +++ /dev/null @@ -1,146 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bookkeeper-dist-server - jar - Apache BookKeeper :: Dist (Server) - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper.http - vertx-http-server - ${project.version} - - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - org.xerial.snappy - snappy-java - - - - io.dropwizard.metrics - metrics-core - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-server-${project.version} - true - - ../src/assemble/bin-server.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - - diff --git a/bookkeeper-dist/src/assemble/src.xml b/bookkeeper-dist/src/assemble/src.xml index 146061e28a4..0301c8a9bde 100644 --- a/bookkeeper-dist/src/assemble/src.xml +++ b/bookkeeper-dist/src/assemble/src.xml @@ -30,7 +30,6 @@ **/README.md **/LICENSE **/NOTICE - **/pom.xml **/*gradle* **/src/** **/conf/** diff --git a/bookkeeper-http/http-server/pom.xml b/bookkeeper-http/http-server/pom.xml deleted file mode 100644 index 044a5c02850..00000000000 --- a/bookkeeper-http/http-server/pom.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - http-server - Apache BookKeeper :: Http :: Http Server - http://maven.apache.org - - - commons-configuration - commons-configuration - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - diff --git a/bookkeeper-http/pom.xml b/bookkeeper-http/pom.xml deleted file mode 100644 index bb9307322bf..00000000000 --- a/bookkeeper-http/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.http - bookkeeper-http - pom - Apache BookKeeper :: Http - - http-server - vertx-http-server - servlet-http-server - - diff --git a/bookkeeper-http/servlet-http-server/pom.xml b/bookkeeper-http/servlet-http-server/pom.xml deleted file mode 100644 index 2efa234a518..00000000000 --- a/bookkeeper-http/servlet-http-server/pom.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - servlet-http-server - Apache BookKeeper :: Bookkeeper Http :: Servlet Http Server - http://maven.apache.org - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - javax.servlet - javax.servlet-api - - - commons-io - commons-io - - - org.eclipse.jetty - jetty-server - test - - - org.eclipse.jetty - jetty-webapp - test - - - - diff --git a/bookkeeper-http/vertx-http-server/pom.xml b/bookkeeper-http/vertx-http-server/pom.xml deleted file mode 100644 index c8914241361..00000000000 --- a/bookkeeper-http/vertx-http-server/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - vertx-http-server - Apache BookKeeper :: Bookkeeper Http :: Vertx Http Server - http://maven.apache.org - - - io.vertx - vertx-core - - - io.vertx - vertx-web - - - com.google.guava - guava - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - diff --git a/bookkeeper-proto/pom.xml b/bookkeeper-proto/pom.xml deleted file mode 100644 index 0894fd064ec..00000000000 --- a/bookkeeper-proto/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-proto - Apache BookKeeper :: Protocols - - - com.google.protobuf - protobuf-java - - - - - - org.apache.rat - apache-rat-plugin - - - - **/DataFormats.java - **/BookkeeperProtocol.java - **/DbLedgerStorageDataFormats.java - **/.checkstyle - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - true - - - - - compile - - - - - - - diff --git a/bookkeeper-server/build.gradle b/bookkeeper-server/build.gradle index 5b0cd26ba5d..d55aa8a9c2a 100644 --- a/bookkeeper-server/build.gradle +++ b/bookkeeper-server/build.gradle @@ -80,11 +80,6 @@ dependencies { testImplementation depLibs.log4jCore } -task writeClasspath { - buildDir.mkdirs() - new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" -} - test { retry { maxFailures = 200 @@ -101,3 +96,6 @@ test.doFirst { jvmArgs("-Djunit.timeout.test=600000", "-Djunit.max.retry=3", "-Djava.net.preferIPv4Stack=true", "-Dio.netty.leakDetection.level=paranoid") } +jar { + dependsOn tasks.named("writeClasspath") +} diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml deleted file mode 100644 index 469aab161d9..00000000000 --- a/bookkeeper-server/pom.xml +++ /dev/null @@ -1,336 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-server - Apache BookKeeper :: Server - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-common-allocator - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-proto - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.parent.version} - - - org.rocksdb - rocksdbjni - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - org.apache.zookeeper - zookeeper - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - linux-x86_64 - - - io.netty - netty-tcnative-boringssl-static - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper - circe-checksum - ${project.version} - - - commons-cli - commons-cli - - - commons-codec - commons-codec - - - commons-io - commons-io - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - org.bouncycastle - bc-fips - - - com.beust - jcommander - - - net.java.dev.jna - jna - - - org.apache.httpcomponents - httpclient - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - test-jar - test - - - org.apache.kerby - kerby-config - ${kerby.version} - test - - - org.slf4j - * - - - - - org.apache.kerby - kerb-simplekdc - ${kerby.version} - test - - - org.slf4j - * - - - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.parent.version} - test - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - test - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.rat - apache-rat-plugin - - - - **/target/**/* - **/.classpath - **/.gitignore - **/.project - **/.checkstyle - **/.settings/* - src/test/resources/server-key.pem - src/test/resources/server-key.p12 - src/test/resources/server-key.jks - src/test/resources/server-cert.pem - src/test/resources/client-key.pem - src/test/resources/client-key.p12 - src/test/resources/client-key.jks - src/test/resources/client-cert.pem - src/test/resources/keyStoreClientPassword.txt - src/test/resources/keyStoreServerPassword.txt - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - false - - - listener - org.apache.bookkeeper.common.testing.util.TimedOutTestsListener - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - none - org.apache.bookkeeper.client:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature - - - Bookkeeper - org.apache.bookkeeper* - - - - - - attach-javadocs - - jar - - - - - - - - - vertx-http-server - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - - - - - tls-certs - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - Generate Self-Signed Certificates - generate-test-resources - - exec - - - ${basedir}/src/test/resources - ${basedir}/src/test/resources/generateKeysAndCerts.sh - - - - - - - - - - skipBookKeeperServerTests - - - skipBookKeeperServerTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - - - diff --git a/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml b/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml deleted file mode 100644 index 7ba68aa96e2..00000000000 --- a/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.bookkeeper.stats - codahale-metrics-provider - Apache BookKeeper :: Stats Providers :: Codahale Metrics - http://maven.apache.org - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - io.dropwizard.metrics - metrics-core - - - io.dropwizard.metrics - metrics-jvm - - - io.dropwizard.metrics - metrics-graphite - - - com.google.guava - guava - - - diff --git a/bookkeeper-stats-providers/pom.xml b/bookkeeper-stats-providers/pom.xml deleted file mode 100644 index faed3ad46de..00000000000 --- a/bookkeeper-stats-providers/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - bookkeeper-stats-providers - pom - Apache BookKeeper :: Stats Providers - - codahale-metrics-provider - prometheus-metrics-provider - - - diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml deleted file mode 100644 index 3dfe19c0b2a..00000000000 --- a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.bookkeeper.stats - prometheus-metrics-provider - Apache BookKeeper :: Stats Providers :: Prometheus - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - - io.prometheus - simpleclient - - - - io.prometheus - simpleclient_hotspot - - - - io.prometheus - simpleclient_servlet - - - - io.netty - netty-common - - - - org.eclipse.jetty - jetty-servlet - - - - com.google.guava - guava - - - - com.yahoo.datasketches - sketches-core - - - - diff --git a/bookkeeper-stats/pom.xml b/bookkeeper-stats/pom.xml deleted file mode 100644 index b89cb4bcc97..00000000000 --- a/bookkeeper-stats/pom.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.stats - bookkeeper-stats-api - Apache BookKeeper :: Stats API - http://maven.apache.org - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - none - org.apache.bookkeeper.stats - - - Bookkeeper Stats API - org.apache.bookkeeper.stats - - - - - - attach-javadocs - - jar - - - - - - - - - commons-configuration - commons-configuration - - - diff --git a/build.gradle b/build.gradle index 881981592f4..b92a70cbf41 100644 --- a/build.gradle +++ b/build.gradle @@ -89,6 +89,8 @@ allprojects { reportLevel = 'high' } + + task testJar(type: Jar, dependsOn: testClasses) { classifier = 'tests' from sourceSets.test.output @@ -234,6 +236,12 @@ allprojects { showStandardStreams = true } } + tasks.register('writeClasspath') { + doLast { + buildDir.mkdirs() + new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" + } + } } repositories { diff --git a/buildtools/pom.xml b/buildtools/pom.xml deleted file mode 100644 index 1dced8306c6..00000000000 --- a/buildtools/pom.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - buildtools - Apache BookKeeper :: Build Tools - 4.15.0-SNAPSHOT - diff --git a/circe-checksum/pom.xml b/circe-checksum/pom.xml deleted file mode 100644 index 75fac546587..00000000000 --- a/circe-checksum/pom.xml +++ /dev/null @@ -1,236 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - - circe-checksum - nar - Apache BookKeeper :: Circe Checksum Library - Circe Checksum Library - - - dynamic - -msse4.2 -mpclmul - - - - - - com.google.guava - guava - - - - io.netty - netty-buffer - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - src/main/assembly/assembly.xml - - false - posix - - - - make-assembly - package - - single - - - - - - org.jacoco - jacoco-maven-plugin - ${jacoco-maven-plugin.version} - - - - com/scurrilous/circe/checksum/NarSystem* - - - - - - - - - - jdk-without-javah - - [10,) - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - - default-nar-javah - none - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - -h - ${project.build.directory}/nar/javah-include - - - - - - - - mac - - - Mac OS X - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - circe-checksum - - - jni - com.scurrilous.circe.checksum - - - - ${nar.cpp.optionSet} - false - false - full - - - - - - - - Linux - - - Linux - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - circe-checksum - - - jni - com.scurrilous.circe.checksum - - - - ${nar.cpp.optionSet} - false - false - full - - - rt - - - - - - - - - diff --git a/cpu-affinity/pom.xml b/cpu-affinity/pom.xml deleted file mode 100644 index c1f8eba9d92..00000000000 --- a/cpu-affinity/pom.xml +++ /dev/null @@ -1,226 +0,0 @@ - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - - cpu-affinity - nar - Apache BookKeeper :: CPU Affinity Library - CPU Affinity Library - - - dynamic - - - - - com.google.guava - guava - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - src/main/assembly/assembly.xml - - false - posix - - - - make-assembly - package - - single - - - - - - org.apache.rat - apache-rat-plugin - - - - **/src/test/resources/proc_cpuinfo.txt - - - - - - - - - - jdk-without-javah - - [10,) - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - - default-nar-javah - none - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - -h - ${project.build.directory}/nar/javah-include - - - - - - - - mac - - - Mac OS X - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - cpu-affinity - - - jni - org.apache.bookkeeper.utils.affinity - - - - ${nar.cpp.optionSet} - false - false - full - - - - - - - - - Linux - - - Linux - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - cpu-affinity - - - jni - org.apache.bookkeeper.utils.affinity - - - - ${nar.cpp.optionSet} - false - false - full - - - rt - - - - - - - - - diff --git a/dev/check-all-licenses b/dev/check-all-licenses index 1f548c3cef1..03eeb51c65b 100755 --- a/dev/check-all-licenses +++ b/dev/check-all-licenses @@ -27,7 +27,7 @@ set -e -x HERE=$(dirname $0) BOOKKEEPER_DIST=$HERE/../bookkeeper-dist -$HERE/check-binary-license $BOOKKEEPER_DIST/server/target/bookkeeper-server-*-bin.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/all/target/bookkeeper-all-*-bin.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/target/bkctl-*-bin.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/server/build/distributions/bookkeeper-server-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/all/build/distributions/bookkeeper-all-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/build/distributions/bkctl-*.tar.gz diff --git a/dev/check-all-licenses-gradle b/dev/check-all-licenses-gradle deleted file mode 100755 index 03eeb51c65b..00000000000 --- a/dev/check-all-licenses-gradle +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Script to check licenses on a binary tarball. -# It extracts the list of bundled jars, the NOTICE, and the LICENSE -# files. It checked that every non-bk jar bundled is mentioned in the -# LICENSE file. It checked that all jar files mentioned in NOTICE and -# LICENSE are actually bundled. - -# all error fatal -set -e -x - -HERE=$(dirname $0) -BOOKKEEPER_DIST=$HERE/../bookkeeper-dist -$HERE/check-binary-license $BOOKKEEPER_DIST/server/build/distributions/bookkeeper-server-*.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/all/build/distributions/bookkeeper-all-*.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/build/distributions/bkctl-*.tar.gz - diff --git a/dev/common.sh b/dev/common.sh index d4f3d3f2efa..49825406a55 100644 --- a/dev/common.sh +++ b/dev/common.sh @@ -21,11 +21,7 @@ # function get_bk_version() { - bk_version=$(mvn -q \ - -Dexec.executable="echo" \ - -Dexec.args='${project.version}' \ - --non-recursive \ - org.codehaus.mojo:exec-maven-plugin:1.3.1:exec) + bk_version=$(${BK_HOME}/gradlew --no-daemon properties -q | grep "version:" | awk '{print $2}') echo ${bk_version} } diff --git a/dev/docker/ci.sh b/dev/docker/ci.sh deleted file mode 100755 index 75588a3d3e2..00000000000 --- a/dev/docker/ci.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -x -u - -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - -export IMAGE_NAME="bookkeeper/dev" - -pushd ${SCRIPT_DIR} - -docker build --rm=true -t ${IMAGE_NAME} . - -popd - -if [ "$(uname -s)" == "Linux" ]; then - USER_NAME=${SUDO_USER:=$USER} - USER_ID=$(id -u "${USER_NAME}") - GROUP_ID=$(id -g "${USER_NAME}") - LOCAL_HOME="/home/${USER_NAME}" -else # boot2docker uid and gid - USER_NAME=$USER - USER_ID=1000 - GROUP_ID=50 - LOCAL_HOME="/Users/${USER_NAME}" -fi - -docker build -t "${IMAGE_NAME}-${USER_NAME}" - < - - - - org.apache.bookkeeper.metadata.drivers - metadata-drivers-parent - 4.15.0-SNAPSHOT - .. - - 4.0.0 - org.apache.bookkeeper.metadata.drivers - metadata-stores-etcd - Apache BookKeeper :: Metadata Drivers:: Etcd - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - io.etcd - jetcd-core - ${etcd.version} - - - io.grpc - * - - - - - - io.grpc - grpc-all - ${grpc.version} - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - - org.arquillian.cube - arquillian-cube-docker - ${arquillian-cube.version} - - - com.github.docker-java - * - - - test - - - org.jboss.arquillian.junit - arquillian-junit-standalone - ${arquillian-junit.version} - - - com.github.docker-java - * - - - test - - - org.testcontainers - testcontainers - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/metadata-drivers/pom.xml b/metadata-drivers/pom.xml deleted file mode 100644 index e44e116edeb..00000000000 --- a/metadata-drivers/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.metadata.drivers - metadata-drivers-parent - pom - Apache BookKeeper :: Metadata Drivers :: Parent - - etcd - - diff --git a/microbenchmarks/pom.xml b/microbenchmarks/pom.xml deleted file mode 100644 index dc0093954db..00000000000 --- a/microbenchmarks/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - microbenchmarks - Apache BookKeeper :: microbenchmarks - http://maven.apache.org - - benchmarks - - - - org.openjdk.jmh - jmh-core - - - org.openjdk.jmh - jmh-generator-annprocess - provided - - - org.slf4j - slf4j-api - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - compile - jar - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - org.apache.bookkeeper.stats - codahale-metrics-provider - ${project.version} - - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - ${javac.target} - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - ${uberjar.name} - - - org.openjdk.jmh.Main - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - - diff --git a/pom.xml b/pom.xml deleted file mode 100644 index ac7658821b0..00000000000 --- a/pom.xml +++ /dev/null @@ -1,1164 +0,0 @@ - - - - - org.apache - apache - 19 - - 4.0.0 - org.apache.bookkeeper - 4.15.0-SNAPSHOT - bookkeeper - pom - Apache BookKeeper :: Parent - http://bookkeeeper.apache.org - 2011 - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - scm:git:https://github.com/apache/bookkeeper.git - scm:git:https://github.com/apache/bookkeeper.git - https://github.com/apache/bookkeeper - branch-4.13 - - - JIRA - https://issues.apache.org/jira/browse/BOOKKEEPER - - - Jenkins - https://builds.apache.org/job/bookkeeper-master - - - buildtools - circe-checksum - bookkeeper-common - bookkeeper-common-allocator - stats - - bookkeeper-stats - bookkeeper-proto - bookkeeper-server - bookkeeper-benchmark - bookkeeper-stats-providers - bookkeeper-http - stream - tools - cpu-affinity - metadata-drivers - bookkeeper-dist - shaded - microbenchmarks - tests - - - - BookKeeper User - user-subscribe@bookkeeper.apache.org - user-unsubscribe@bookkeeper.apache.org - user@bookkeeper.apache.org - http://www.mail-archive.com/user@bookkeeper.apache.org - - - BookKeeper Dev - dev-subscribe@bookkeeper.apache.org - dev-unsubscribe@bookkeeper.apache.org - dev@bookkeeper.apache.org - http://www.mail-archive.com/dev@bookkeeper.apache.org - - - BookKeeper Commits - commits-subscribe@bookkeeper.apache.org - commits-unsubscribe@bookkeeper.apache.org - commits@bookkeeper.apache.org - http://www.mail-archive.com/commits@bookkeeper.apache.org - - - - - The Apache BookKeeper Team - dev@bookkeeper.apache.org - http://bookkeeper.apache.org - Apache Software Foundation - http://www.apache.org - - - - UTF-8 - UTF-8 - 1.8 - true - 2 - - - 1.18.2 - 1.6.0.Final - 3.0.1 - 1.2 - 4.1 - 1.6 - 1.10 - 1.19 - 2.6 - 3.6 - 2.7 - 1.0.2.1 - 5.1.0 - 3.2.5 - 0.5.11 - 2.7.0 - 3.0.2 - 2.4.0 - 1.42.1 - 30.0-jre - 1.1.1 - 2.10.0 - 1.3 - 2.1.10 - 2.11.0 - 1.78 - 9.4.43.v20210629 - 1.19 - 2.8.2 - 3.2.7 - 4.12 - 0.14.2 - 1.18.20 - 2.17.1 - 1.3.0 - 3.0.0 - 4.1.72.Final - 2.0.46.Final - 9.1.3 - 2.0.2 - 0.8.1 - 0.8.3 - 4.5.13 - 3.14.0 - 3.14.0 - ${grpc.version} - 0.9.11 - 6.22.1.1 - 3.0.1 - 1.7.32 - 1.19 - 3.1.8 - 1.3.2 - 1.15.1 - 3.9.8 - 3.6.2 - 1.1.7 - 2.1.2 - - 0.12 - 2.7 - 4.3.0 - 1.4.13 - 1.6.0 - 1.6 - 0.8.0 - 1.8 - 3.1.0 - 3.0.0 - 2.5 - 3.8.1 - 3.0.2 - 2.7 - 2.5.1 - 2.4 - 3.1.1 - 3.2.0 - 2.2.1 - 3.0.0-M5 - 3.5.2 - 1.4.1.Final - 0.6.1 - 6.19 - 3.1.8 - 1 - 4.0.0 - - - - - - - - com.github.spotbugs - spotbugs-annotations - ${spotbugs-annotations.version} - - - javax.annotation - javax.annotation-api - ${javax-annotations-api.version} - - - com.google.code.findbugs - jsr305 - ${google.code.version} - - - com.google.errorprone - error_prone_annotations - ${google.errorprone.version} - - - org.projectlombok - lombok - ${lombok.version} - - - org.inferred - freebuilder - ${freebuilder.version} - true - - - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - - - - commons-cli - commons-cli - ${commons-cli.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-configuration - commons-configuration - ${commons-configuration.version} - - - commons-io - commons-io - ${commons-io.version} - - - commons-lang - commons-lang - ${commons-lang.version} - - - com.google.guava - guava - ${guava.version} - - - org.apache.commons - commons-collections4 - ${commons-collections4.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - - org.bouncycastle - bc-fips - ${bouncycastle.version} - - - - org.reflections - reflections - ${reflections.version} - - - - - net.jpountz.lz4 - lz4 - ${lz4.version} - - - - - net.java.dev.jna - jna - ${jna.version} - - - - - org.yaml - snakeyaml - ${snakeyaml.version} - - - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - javax.servlet - javax.servlet-api - ${servlet-api.version} - - - com.fasterxml.jackson.module - jackson-module-paranamer - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-scala_2.11 - ${jackson.version} - - - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - - - org.apache.thrift - libthrift - ${libthrift.version} - - - org.apache.tomcat.embed - tomcat-embed-core - - - javax.annotation - javax.annotation-api - - - - - - - io.netty - netty-common - ${netty.version} - - - io.netty - netty-buffer - ${netty.version} - - - io.netty - netty-transport - ${netty.version} - - - io.netty - netty-handler - ${netty.version} - - - io.netty - netty-transport-native-epoll - ${netty.version} - - - io.netty - netty-transport-native-epoll - ${netty.version} - linux-x86_64 - - - io.netty - netty-codec-dns - ${netty.version} - - - io.netty - netty-codec-http - ${netty.version} - - - io.netty - netty-codec-http2 - ${netty.version} - - - io.netty - netty-codec-socks - ${netty.version} - - - io.netty - netty-handler-proxy - ${netty.version} - - - io.netty - netty-resolver - ${netty.version} - - - io.netty - netty-resolver-dns - ${netty.version} - - - io.netty - netty-tcnative-boringssl-static - ${netty-boringssl.version} - - - - - io.grpc - grpc-bom - ${grpc.version} - pom - import - - - - - org.rocksdb - rocksdbjni - ${rocksdb.version} - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.zookeeper - zookeeper - - - - - - - io.vertx - vertx-core - ${vertx.version} - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - - - io.vertx - vertx-web - ${vertx.version} - - - org.eclipse.jetty - jetty-server - ${jetty.version} - - - org.eclipse.jetty - jetty-webapp - ${jetty.version} - - - org.eclipse.jetty - jetty-servlet - ${jetty.version} - - - - - org.jctools - jctools-core - ${jctools.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - io.dropwizard.metrics - metrics-jvm - ${dropwizard.version} - - - io.dropwizard.metrics - metrics-graphite - ${dropwizard.version} - - - - io.prometheus - simpleclient - ${prometheus.version} - - - io.prometheus - simpleclient_hotspot - ${prometheus.version} - - - io.prometheus - simpleclient_servlet - ${prometheus.version} - - - - com.yahoo.datasketches - sketches-core - ${datasketches.version} - - - - - org.apache.httpcomponents - httpclient - ${httpclient.version} - - - - - com.beust - jcommander - ${jcommander.version} - - - - - org.hdrhistogram - HdrHistogram - ${hdrhistogram.version} - - - - - junit - junit - ${junit.version} - - - org.hamcrest - hamcrest-all - ${hamcrest.version} - - - org.jmock - jmock - ${jmock.version} - - - org.mockito - mockito-core - ${mockito.version} - - - org.powermock - powermock-api-mockito2 - ${powermock.version} - - - org.powermock - powermock-module-junit4 - ${powermock.version} - - - org.apache.hadoop - hadoop-minikdc - ${hadoop.minikdc.version} - - - org.arquillian.cube - arquillian-cube-docker - ${arquillian-cube.version} - - - com.github.docker-java - * - - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - ${arquillian-junit.version} - - - com.github.docker-java - * - - - - - org.codehaus.groovy - groovy-all - ${groovy.version} - pom - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-impl-maven - ${shrinkwrap.version} - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-api - ${shrinkwrap.version} - - - org.testcontainers - testcontainers - ${testcontainers.version} - - - - - org.openjdk.jmh - jmh-core - ${jmh.version} - - - org.openjdk.jmh - jmh-generator-annprocess - ${jmh.version} - - - - - - - - - org.projectlombok - lombok - provided - - - com.github.spotbugs - spotbugs-annotations - provided - - - - - org.slf4j - slf4j-api - - - commons-configuration - commons-configuration - - - - - junit - junit - test - - - org.hamcrest - hamcrest-all - test - - - org.apache.logging.log4j - log4j-core - test - - - org.apache.logging.log4j - log4j-slf4j-impl - test - - - org.mockito - mockito-core - test - - - org.powermock - powermock-api-mockito2 - test - - - org.powermock - powermock-module-junit4 - test - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - ${maven-checkstyle-plugin.version} - - - com.puppycrawl.tools - checkstyle - ${puppycrawl.checkstyle.version} - - - - buildtools/src/main/resources/bookkeeper/checkstyle.xml - buildtools/src/main/resources/bookkeeper/suppressions.xml - UTF-8 - true - true - false - true - - - - checkstyle - validate - - check - - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${session.executionRootDirectory}/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Werror - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid - ${redirectTestOutputToFile} - ${forkCount.variable} - false - false - 1800 - ${testRetryCount} - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - -notimestamp - - none - org.apache.bookkeeper.client:org.apache.bookkeeper.client.api:org.apache.bookkeeper.common.annotation:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature:org.apache.bookkeeper.stats - - - Bookkeeper Client - org.apache.bookkeeper.client:org.apache.bookkeeper.common.annotation:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature - - - Bookkeeper Client (New Fluent API - Experimental) - org.apache.bookkeeper.client.api - - - Bookkeeper Stats API - - org.apache.bookkeeper.stats - - - Bookkeeper Stats Providers - org.apache.bookkeeper.stats.codahale:org.apache.bookkeeper.stats.prometheus - - - BookKeeper Java API (version ${project.version}) - site/_site/overview/index.html - package - false - - - - aggregate - - aggregate - - site - - - - - org.apache.maven.plugins - maven-source-plugin - ${maven-source-plugin.version} - - - attach-sources - - jar - - - - - - org.apache.rat - apache-rat-plugin - ${apache-rat-plugin.version} - - - - **/.idea/** - - - .git/**/* - .github/**/* - **/.gitignore - - - **/.svn/**/* - - - **/target/**/* - - - **/README.md - **/README.rst - **/apidocs/* - **/src/main/resources/deps/** - **/META-INF/** - - - **/.classpath - **/.project - **/.checkstyle - **/.settings/* - **/*.iml - **/*.iws - **/*.ipr - - - .repository/** - - - site/** - site2/** - - - **/org/apache/distributedlog/thrift/* - - - **/*.log - - - data/** - - - dev/.vagrant/** - - - **/proto/**.py - - **/python/.coverage - **/python/.Python - **/python/bin/** - **/python/include/** - **/python/lib/** - **/**.pyc - **/.nox/** - **/.pytest_cache/** - **/__pycache__/** - **/bookkeeper.egg-info/** - **/pip-selfcheck.json - - - **/test_conf_2.conf - - true - - - - - - - - code-coverage - - - - org.eluder.coveralls - coveralls-maven-plugin - ${coveralls-maven-plugin.version} - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - @{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true - ${redirectTestOutputToFile} - ${forkCount.variable} - false - 1800 - - true - - - - org.jacoco - jacoco-maven-plugin - 0.8.0 - - - - prepare-agent - - - - - - - - - - - dev - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true - false - ${forkCount.variable} - false - 1800 - false - 0 - - - - - - - dev-debug - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true -Dbookkeeper.root.logger=DEBUG,CONSOLE - false - ${forkCount.variable} - false - 1800 - false - - - - - - - - jdk11-no-spotbugs - - [11,) - - - - - com.github.spotbugs - spotbugs-maven-plugin - - true - - - - - - - - apache-release - - - - maven-assembly-plugin - - - source-release-assembly - - - true - - - - - - - - - diff --git a/settings.gradle b/settings.gradle index 3c72c6e5f80..3d192cfd755 100644 --- a/settings.gradle +++ b/settings.gradle @@ -30,6 +30,8 @@ pluginManagement { rootProject.name = 'bookkeeper' + + include(':bookkeeper-benchmark', ':bookkeeper-tools', ':bookkeeper-tools-framework', diff --git a/shaded/bookkeeper-server-shaded/pom.xml b/shaded/bookkeeper-server-shaded/pom.xml deleted file mode 100644 index 0644f91bcf9..00000000000 --- a/shaded/bookkeeper-server-shaded/pom.xml +++ /dev/null @@ -1,128 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-shaded - Apache BookKeeper :: Shaded :: bookkeeper-server-shaded - - UTF-8 - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - com.google.guava:guava - com.google.protobuf:protobuf-java - org.apache.bookkeeper:bookkeeper-common - org.apache.bookkeeper:bookkeeper-common-allocator - org.apache.bookkeeper:cpu-affinity - org.apache.bookkeeper:bookkeeper-tools-framework - org.apache.bookkeeper:bookkeeper-proto - org.apache.bookkeeper:bookkeeper-server - org.apache.bookkeeper:circe-checksum - org.apache.bookkeeper.stats:bookkeeper-stats-api - - - - - com.google - org.apache.bookkeeper.shaded.com.google - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/bookkeeper-server-tests-shaded/pom.xml b/shaded/bookkeeper-server-tests-shaded/pom.xml deleted file mode 100644 index a5cd9f6a177..00000000000 --- a/shaded/bookkeeper-server-tests-shaded/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-tests-shaded - Apache BookKeeper :: Shaded :: bookkeeper-server-tests-shaded - - UTF-8 - - - - org.apache.bookkeeper - bookkeeper-server - test-jar - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - org.apache.bookkeeper - bookkeeper-common - - - org.apache.bookkeeper - bookkeeper-tools-framework - - - org.apache.bookkeeper - bookkeeper-proto - - - org.apache.bookkeeper - bookkeeper-server - - - org.apache.bookkeeper - circe-checksum - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - com.google.guava:guava - com.google.protobuf:protobuf-java - org.apache.bookkeeper:bookkeeper-server:test-jar:tests - - - - - - com.google - org.apache.bookkeeper.shaded.com.google - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml deleted file mode 100644 index c1cf1fef955..00000000000 --- a/shaded/distributedlog-core-shaded/pom.xml +++ /dev/null @@ -1,251 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - org.apache.distributedlog - distributedlog-core-shaded - Apache BookKeeper :: Shaded :: distributedlog-core-shaded - - UTF-8 - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - io.netty - netty-common - - - io.netty - netty-buffer - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - commons-codec:commons-codec - commons-cli:commons-cli - commons-io:commons-io - commons-lang:commons-lang - commons-logging:commons-logging - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.core:jackson-annotations - com.google.guava:guava - com.google.protobuf:protobuf-java - net.java.dev.jna:jna - net.jpountz.lz4:lz4 - org.apache.bookkeeper:bookkeeper-common - org.apache.bookkeeper:bookkeeper-common-allocator - org.apache.bookkeeper:cpu-affinity - org.apache.bookkeeper:bookkeeper-tools-framework - org.apache.bookkeeper:bookkeeper-proto - org.apache.bookkeeper:bookkeeper-server - org.apache.bookkeeper:circe-checksum - org.apache.bookkeeper.http:http-server - org.apache.bookkeeper.stats:bookkeeper-stats-api - org.apache.commons:commons-collections4 - org.apache.commons:commons-lang3 - org.apache.distributedlog:distributedlog-common - org.apache.distributedlog:distributedlog-core - org.apache.distributedlog:distributedlog-protocol - org.apache.httpcomponents:httpclient - org.apache.httpcomponents:httpcore - org.apache.thrift:libthrift - org.apache.zookeeper:zookeeper - org.apache.zookeeper:zookeeper-jute - org.rocksdb:rocksdbjni - - - - - - org.apache.commons.cli - dlshade.org.apache.commons.cli - - - org.apache.commons.codec - dlshade.org.apache.commons.codec - - - org.apache.commons.collections4 - dlshade.org.apache.commons.collections4 - - - org.apache.commons.lang - dlshade.org.apache.commons.lang - - - org.apache.commons.lang3 - dlshade.org.apache.commons.lang3 - - - org.apache.commons.logging - dlshade.org.apache.commons.logging - - - org.apache.commons.io - dlshade.org.apache.commons.io - - - - org.apache.httpcomponents - dlshade.org.apache.httpcomponents - - - org.apache.http - dlshade.org.apache.http - - - - org.apache.thrift - dlshade.org.apache.thrift - - - - org.apache.zookeeper - dlshade.org.apache.zookeeper - - - org.apache.jute - dlshade.org.apache.jute - - - - com.fasterxml.jackson - dlshade.com.fasterxml.jackson - - - - com.sun.jna - dlshade.com.sun.jna - - - - com.google - dlshade.com.google - - - - org.jboss.netty - dlshade.org.jboss.netty - - - - net.jpountz - dlshade.net.jpountz - - - - org.rocksdb - dlshade.org.rocksdb - - - - com.scurrilous.circe - dlshade.com.scurrilous.circe - - - org.apache.bookkeeper - dlshade.org.apache.bookkeeper - - - - org.apache.distributedlog - org.apache.distributedlog - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/pom.xml b/shaded/pom.xml deleted file mode 100644 index 254de71d961..00000000000 --- a/shaded/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper - shaded-parent - Apache BookKeeper :: Shaded :: Parent - - bookkeeper-server-shaded - bookkeeper-server-tests-shaded - distributedlog-core-shaded - - diff --git a/stats/pom.xml b/stats/pom.xml deleted file mode 100644 index d2c89aa671f..00000000000 --- a/stats/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - pom - org.apache.bookkeeper.stats - bookkeeper-stats-parent - Apache BookKeeper :: Stats :: Parent - - - utils - - - - diff --git a/stats/utils/pom.xml b/stats/utils/pom.xml deleted file mode 100644 index cdfebcd08cc..00000000000 --- a/stats/utils/pom.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - 4.0.0 - - bookkeeper-stats-parent - org.apache.bookkeeper.stats - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.stats - bookkeeper-stats-utils - Apache BookKeeper :: Stats :: Utils - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - org.reflections - reflections - - - org.yaml - snakeyaml - - - com.beust - jcommander - - - com.fasterxml.jackson.core - jackson-annotations - - - diff --git a/stream/api/pom.xml b/stream/api/pom.xml deleted file mode 100644 index 82906fdb484..00000000000 --- a/stream/api/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-api - Apache BookKeeper :: Stream Storage :: API - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - io.netty - netty-buffer - - - org.inferred - freebuilder - - - diff --git a/stream/bin/streamstorage b/stream/bin/streamstorage index a85c6e70d32..1f41b2258ac 100755 --- a/stream/bin/streamstorage +++ b/stream/bin/streamstorage @@ -20,6 +20,7 @@ BINDIR=$(dirname "$0") SS_HOME=`cd $BINDIR/..;pwd` +BK_HOME=${BK_HOME:-"`cd ${BINDIR}/../..;pwd`"} DEFAULT_STANDALONE_CONF=$SS_HOME/conf/standalone.conf DEFAULT_LOG_CONF=$SS_HOME/conf/log4j.properties @@ -79,18 +80,14 @@ EOF } add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${SS_HOME}/server/target/classpath.txt" + f="${SS_HOME}/server/build/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${SS_HOME}/server/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + echo "no classpath.txt found at ${SS_HOME}/server/build" + exit 1 fi SS_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/stream/bin/streamstorage-cli b/stream/bin/streamstorage-cli index 6a1df7810d8..365c50dcf16 100755 --- a/stream/bin/streamstorage-cli +++ b/stream/bin/streamstorage-cli @@ -57,18 +57,14 @@ elif [ -e "$BUILT_JAR" ]; then fi add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${SS_HOME}/cli/target/classpath.txt" + f="${SS_HOME}/cli/build/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${SS_HOME}/cli/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + echo "no classpath.txt found at ${SS_HOME}/cli/build" + exit 1 fi SS_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/stream/bk-grpc-name-resolver/pom.xml b/stream/bk-grpc-name-resolver/pom.xml deleted file mode 100644 index 37feb436205..00000000000 --- a/stream/bk-grpc-name-resolver/pom.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - bk-grpc-name-resolver - Apache BookKeeper :: Stream Storage :: Common :: BK Grpc Name Resolver - - - org.apache.bookkeeper - stream-storage-common - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.version} - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - diff --git a/stream/clients/java/all/pom.xml b/stream/clients/java/all/pom.xml deleted file mode 100644 index 1aa66fd526a..00000000000 --- a/stream/clients/java/all/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-client - Apache BookKeeper :: Stream Storage :: Clients :: Java Client - - - - org.apache.bookkeeper - stream-storage-java-kv-client - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - org.apache.bookkeeper:stream-storage-* - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - - ${project.basedir} - - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/stream/clients/java/base/pom.xml b/stream/clients/java/base/pom.xml deleted file mode 100644 index c8d8388420b..00000000000 --- a/stream/clients/java/base/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-client-base - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: Base - - - - org.apache.bookkeeper - stream-storage-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/clients/java/kv/pom.xml b/stream/clients/java/kv/pom.xml deleted file mode 100644 index d9c9e359324..00000000000 --- a/stream/clients/java/kv/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-kv-client - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: KV - - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - diff --git a/stream/clients/java/pom.xml b/stream/clients/java/pom.xml deleted file mode 100644 index 44a462220e9..00000000000 --- a/stream/clients/java/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-clients-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-java-client-parent - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: Parent - - base - kv - all - - diff --git a/stream/clients/pom.xml b/stream/clients/pom.xml deleted file mode 100644 index 3001eeb8d90..00000000000 --- a/stream/clients/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-clients-parent - Apache BookKeeper :: Stream Storage :: Clients :: Parent - - java - - diff --git a/stream/common/pom.xml b/stream/common/pom.xml deleted file mode 100644 index 6f5091ad933..00000000000 --- a/stream/common/pom.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-common - Apache BookKeeper :: Stream Storage :: Common - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - io.netty - netty-buffer - - - io.grpc - grpc-all - - - io.grpc - grpc-netty-shaded - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - javax.annotation - javax.annotation-api - - true - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - test - - - diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml deleted file mode 100644 index f91055fa99d..00000000000 --- a/stream/distributedlog/common/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-common - Apache BookKeeper :: DistributedLog :: Common - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - org.slf4j - slf4j-log4j12 - - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.commons - commons-lang3 - - - com.google.guava - guava - - - commons-lang - commons-lang - - - commons-codec - commons-codec - - - io.netty - netty-buffer - - - net.jpountz.lz4 - lz4 - - - org.jmock - jmock - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G - always - 1800 - - - - - diff --git a/stream/distributedlog/core/build.gradle b/stream/distributedlog/core/build.gradle index ee9a79fc10a..d8f88a366fc 100644 --- a/stream/distributedlog/core/build.gradle +++ b/stream/distributedlog/core/build.gradle @@ -75,6 +75,7 @@ test.doFirst { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'distributedlog-core' } diff --git a/stream/distributedlog/core/pom.xml b/stream/distributedlog/core/pom.xml deleted file mode 100644 index 78c6ff62b28..00000000000 --- a/stream/distributedlog/core/pom.xml +++ /dev/null @@ -1,135 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-core - Apache BookKeeper :: DistributedLog :: Core Library - - - org.apache.distributedlog - distributedlog-protocol - ${project.parent.version} - - - org.apache.zookeeper - zookeeper - - - org.apache.thrift - libthrift - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - - - javax.annotation - javax.annotation-api - - true - - - org.jmock - jmock - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.distributedlog - distributedlog-common - ${project.parent.version} - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - false - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G - always - 1800 - - - listener - org.apache.bookkeeper.common.testing.util.TimedOutTestsListener - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - ../../buildtools/src/main/resources/distributedlog/suppressions.xml - ../../buildtools/src/main/resources/bookkeeper/checkstyle.xml - **/thrift/**/* - - - - - diff --git a/stream/distributedlog/io/dlfs/pom.xml b/stream/distributedlog/io/dlfs/pom.xml deleted file mode 100644 index 31366ed9fc0..00000000000 --- a/stream/distributedlog/io/dlfs/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - 4.0.0 - - distributedlog - org.apache.distributedlog - 4.15.0-SNAPSHOT - ../.. - - org.apache.distributedlog - dlfs - Apache BookKeeper :: DistributedLog :: IO :: FileSystem - http://maven.apache.org - - UTF-8 - ${basedir}/lib - - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/stream/distributedlog/io/pom.xml b/stream/distributedlog/io/pom.xml deleted file mode 100644 index 38572cb2e0b..00000000000 --- a/stream/distributedlog/io/pom.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - 4.0.0 - distributedlog-io - pom - Apache BookKeeper :: DistributedLog :: IO - - dlfs - - diff --git a/stream/distributedlog/pom.xml b/stream/distributedlog/pom.xml deleted file mode 100644 index 2f77d0fc025..00000000000 --- a/stream/distributedlog/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.distributedlog - distributedlog - pom - Apache BookKeeper :: DistributedLog :: Parent - - Apache DistributedLog provides a high performance replicated log service. - - 2016 - - common - protocol - core - io - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - -notimestamp - none - - - Core Library - org.apache.distributedlog:org.apache.distributedlog.annotations:org.apache.distributedlog.callback:org.apache.distributedlog.exceptions:org.apache.distributedlog.feature:org.apache.distributedlog.io:org.apache.distributedlog.lock:org.apache.distributedlog.logsegment:org.apache.distributedlog.metadata:org.apache.distributedlog.namespace:org.apache.distributedlog.net:org.apache.distributedlog.stats:org.apache.distributedlog.api.subscription - - - - org.apache.distributedlog.acl:org.apache.distributedlog.admin:org.apache.distributedlog.auditor:org.apache.distributedlog.basic:org.apache.distributedlog.benchmark*:org.apache.distributedlog.bk:org.apache.distributedlog.ownership:org.apache.distributedlog.proxy:org.apache.distributedlog.resolver:org.apache.distributedlog.service.*:org.apache.distributedlog.config:org.apache.distributedlog.function:org.apache.distributedlog.impl*:org.apache.distributedlog.injector:org.apache.distributedlog.kafka:org.apache.distributedlog.limiter:org.apache.distributedlog.mapreduce:org.apache.distributedlog.messaging:org.apache.distributedlog.common.rate:org.apache.distributedlog.readahead:org.apache.distributedlog.selector:org.apache.distributedlog.stats:org.apache.distributedlog.thrift*:org.apache.distributedlog.tools:org.apache.distributedlog.util:org.apache.distributedlog.zk:org.apache.bookkeeper.client:org.apache.bookkeeper.stats - - - - - aggregate - - aggregate - - site - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID - always - 1800 - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${session.executionRootDirectory}/buildtools/src/main/resources/distributedlog/findbugsExclude.xml - - - - - - diff --git a/stream/distributedlog/protocol/pom.xml b/stream/distributedlog/protocol/pom.xml deleted file mode 100644 index 727a8009c21..00000000000 --- a/stream/distributedlog/protocol/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-protocol - Apache BookKeeper :: DistributedLog :: Protocol - - - org.apache.distributedlog - distributedlog-common - ${project.version} - - - io.netty - netty-buffer - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/stream/pom.xml b/stream/pom.xml deleted file mode 100644 index d1b01cec7e0..00000000000 --- a/stream/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - pom - org.apache.bookkeeper - stream-storage-parent - Apache BookKeeper :: Stream Storage :: Parent - - - distributedlog - common - tests-common - statelib - api - proto - clients - storage - server - bk-grpc-name-resolver - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - true - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - true - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID - always - 1800 - - - - - - - - streamTests - - - streamTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - - - diff --git a/stream/proto/pom.xml b/stream/proto/pom.xml deleted file mode 100644 index 7827096cb49..00000000000 --- a/stream/proto/pom.xml +++ /dev/null @@ -1,115 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-proto - Apache BookKeeper :: Stream Storage :: Proto - - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - org.apache.commons - commons-lang3 - - - com.google.protobuf - protobuf-java - - - javax.annotation - javax.annotation-api - - true - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - test - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Xlint:unchecked - - -Xpkginfo:always - - false - false - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - true - - - - - compile - compile-custom - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/server/build.gradle b/stream/server/build.gradle index 19faa189683..d8cba32767f 100644 --- a/stream/server/build.gradle +++ b/stream/server/build.gradle @@ -74,12 +74,8 @@ application { mainClassName = "org.apache.bookkeeper.stream.cluster.StandaloneStarter" } -task writeClasspath { - buildDir.mkdirs() - new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" -} - jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'stream-storage-server' } diff --git a/stream/server/pom.xml b/stream/server/pom.xml deleted file mode 100644 index 386b611be91..00000000000 --- a/stream/server/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - - stream-storage-server - Apache BookKeeper :: Stream Storage :: Server - - - - org.apache.bookkeeper - stream-storage-service-impl - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client - ${project.parent.version} - - - com.beust - jcommander - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - runtime - - - - org.xerial.snappy - snappy-java - runtime - - - - io.dropwizard.metrics - metrics-core - runtime - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/statelib/pom.xml b/stream/statelib/pom.xml deleted file mode 100644 index d2466182dad..00000000000 --- a/stream/statelib/pom.xml +++ /dev/null @@ -1,169 +0,0 @@ - - - - 4.0.0 - - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - statelib - Apache BookKeeper :: Stream Storage :: State Library - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - com.google.guava - guava - - - org.rocksdb - rocksdbjni - - - com.google.protobuf - protobuf-java - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - - - - - compile - - - - - - - diff --git a/stream/storage/api/pom.xml b/stream/storage/api/pom.xml deleted file mode 100644 index bc415e3e28b..00000000000 --- a/stream/storage/api/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-service-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-api - Apache BookKeeper :: Stream Storage :: Storage :: Api - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/storage/impl/pom.xml b/stream/storage/impl/pom.xml deleted file mode 100644 index 7b2ff7d5672..00000000000 --- a/stream/storage/impl/pom.xml +++ /dev/null @@ -1,160 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-service-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-impl - Apache BookKeeper :: Stream Storage :: Storage :: Impl - - - - org.apache.bookkeeper - stream-storage-service-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - - - org.apache.bookkeeper - statelib - ${project.parent.version} - - - org.apache.curator - curator-recipes - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - tests - test - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/storage/pom.xml b/stream/storage/pom.xml deleted file mode 100644 index 22e3eaf395d..00000000000 --- a/stream/storage/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-parent - Apache BookKeeper :: Stream Storage :: Storage :: Parent - - api - impl - - diff --git a/stream/tests-common/pom.xml b/stream/tests-common/pom.xml deleted file mode 100644 index c14fde018ae..00000000000 --- a/stream/tests-common/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.tests - stream-storage-tests-common - Apache BookKeeper :: Stream Storage :: Common Classes for Tests - - - - io.grpc - grpc-all - - - io.grpc - grpc-netty-shaded - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - com.google.protobuf - protobuf-java - - - javax.annotation - javax.annotation-api - - true - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Xlint:unchecked - - -Xpkginfo:always - - false - false - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - true - - - - - compile - compile-custom - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/tests/backward-compat/bc-non-fips/pom.xml b/tests/backward-compat/bc-non-fips/pom.xml deleted file mode 100644 index eba5b1d7219..00000000000 --- a/tests/backward-compat/bc-non-fips/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - bc-non-fips - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test Bouncy Castle Provider load non FIPS version - - 1.68 - - - - - junit - junit - ${junit.version} - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.bouncycastle - * - - - test - - - - org.bouncycastle - bcpkix-jdk15on - ${bc-non-fips.version} - - - - org.bouncycastle - bcprov-ext-jdk15on - ${bc-non-fips.version} - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - - diff --git a/tests/backward-compat/current-server-old-clients/pom.xml b/tests/backward-compat/current-server-old-clients/pom.xml deleted file mode 100644 index 2379977810d..00000000000 --- a/tests/backward-compat/current-server-old-clients/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - backward-compat-current-server-old-clients - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test old clients working on current server - - diff --git a/tests/backward-compat/hierarchical-ledger-manager/pom.xml b/tests/backward-compat/hierarchical-ledger-manager/pom.xml deleted file mode 100644 index a75736839c2..00000000000 --- a/tests/backward-compat/hierarchical-ledger-manager/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - hierarchical-ledger-manager - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test compat between old version and new version of hierarchical ledger manager - - diff --git a/tests/backward-compat/hostname-bookieid/pom.xml b/tests/backward-compat/hostname-bookieid/pom.xml deleted file mode 100644 index 28d64fce9c7..00000000000 --- a/tests/backward-compat/hostname-bookieid/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - hostname-bookieid - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between 4.1.0 and current version (with hostname bookie ID) - - diff --git a/tests/backward-compat/old-cookie-new-cluster/pom.xml b/tests/backward-compat/old-cookie-new-cluster/pom.xml deleted file mode 100644 index c7705c999cd..00000000000 --- a/tests/backward-compat/old-cookie-new-cluster/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - old-cookie-new-cluster - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade 4.1.0 to current in cluster with cookies - - diff --git a/tests/backward-compat/pom.xml b/tests/backward-compat/pom.xml deleted file mode 100644 index 396840bb8f4..00000000000 --- a/tests/backward-compat/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - integration-tests-base-groovy - 4.15.0-SNAPSHOT - ../integration-tests-base-groovy - - org.apache.bookkeeper.tests - backward-compat - Apache BookKeeper :: Tests :: Backward Compatibility - - upgrade - upgrade-direct - hierarchical-ledger-manager - hostname-bookieid - recovery-no-password - old-cookie-new-cluster - current-server-old-clients - yahoo-custom-version - bc-non-fips - - diff --git a/tests/backward-compat/recovery-no-password/pom.xml b/tests/backward-compat/recovery-no-password/pom.xml deleted file mode 100644 index 8eb53fc7f07..00000000000 --- a/tests/backward-compat/recovery-no-password/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - recovery-no-password - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test recovery does not work when password no in metadata - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - diff --git a/tests/backward-compat/upgrade-direct/pom.xml b/tests/backward-compat/upgrade-direct/pom.xml deleted file mode 100644 index c03d5686c38..00000000000 --- a/tests/backward-compat/upgrade-direct/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - upgrade-direct - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between 4.1.0 and current version - - diff --git a/tests/backward-compat/upgrade/pom.xml b/tests/backward-compat/upgrade/pom.xml deleted file mode 100644 index e8377df4e68..00000000000 --- a/tests/backward-compat/upgrade/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - upgrade - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between all released versions and current version - - diff --git a/tests/backward-compat/yahoo-custom-version/pom.xml b/tests/backward-compat/yahoo-custom-version/pom.xml deleted file mode 100644 index 6061316a0e7..00000000000 --- a/tests/backward-compat/yahoo-custom-version/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - yahoo-custom-version - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between yahoo custom version and current - - diff --git a/tests/docker-images/all-released-versions-image/pom.xml b/tests/docker-images/all-released-versions-image/pom.xml deleted file mode 100644 index 7abcbb03f75..00000000000 --- a/tests/docker-images/all-released-versions-image/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - all-released-versions-image - Apache BookKeeper :: Tests :: Docker Images :: All Released Versions - pom - - - docker - - - integrationTests - - - - - - com.spotify - dockerfile-maven-plugin - 1.4.13 - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-all-released-versions - latest - - - - - apachebookkeeper/bookkeeper-all-released-versions - ${project.version} - false - - - - - - - diff --git a/tests/docker-images/all-versions-image/pom.xml b/tests/docker-images/all-versions-image/pom.xml deleted file mode 100644 index 1eb92bb6d5c..00000000000 --- a/tests/docker-images/all-versions-image/pom.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - all-versions-image - Apache BookKeeper :: Tests :: Docker Images :: All Versions - pom - - - org.apache.bookkeeper.tests - all-released-versions-image - ${project.parent.version} - pom - - - - org.apache.bookkeeper - bookkeeper-dist-server - ${project.parent.version} - bin - tar.gz - provided - - - - - docker - - - integrationTests - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile-maven-plugin.version} - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-all-versions - latest - - - - - apachebookkeeper/bookkeeper-all-versions - ${project.version} - false - true - - target/bookkeeper-dist-server-${project.version}-bin.tar.gz - - - - - org.apache.maven.plugins - maven-dependency-plugin - ${maven-dependency-plugin.version} - - - copy-tarball - - copy-dependencies - - generate-resources - - ${project.build.directory}/ - bookkeeper-dist-server - true - - - - - - - - - diff --git a/tests/docker-images/current-version-image/pom.xml b/tests/docker-images/current-version-image/pom.xml deleted file mode 100644 index 5e221904e3a..00000000000 --- a/tests/docker-images/current-version-image/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - current-version-image - Apache BookKeeper :: Tests :: Docker Images :: Current Version - pom - - - org.apache.bookkeeper - bookkeeper-dist-server - ${project.parent.version} - bin - tar.gz - provided - - - - - docker - - - integrationTests - - - - - - - org.codehaus.mojo - exec-maven-plugin - ${exec-maven-plugin.version} - - - build-python-client - generate-resources - - exec - - - ${project.basedir}/target - ${project.basedir}/../../../stream/clients/python/scripts/docker_build.sh - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - generate-resources - - run - - - - copy python wheel file - - - copying docker scripts - - - - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile-maven-plugin.version} - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-current - latest - - - - - apachebookkeeper/bookkeeper-current - ${project.version} - false - true - - ${project.version} - - - - - org.apache.maven.plugins - maven-dependency-plugin - ${maven-dependency-plugin.version} - - - copy-docker-dependencies - - copy-dependencies - - generate-resources - - ${project.build.directory}/ - bookkeeper-dist-server - true - - - - - - - - - diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml deleted file mode 100644 index a337e91176a..00000000000 --- a/tests/docker-images/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - docker-images - Apache BookKeeper :: Tests :: Docker Images - - all-released-versions-image - all-versions-image - current-version-image - - diff --git a/tests/integration-tests-base-groovy/pom.xml b/tests/integration-tests-base-groovy/pom.xml deleted file mode 100644 index ccaa5ea4308..00000000000 --- a/tests/integration-tests-base-groovy/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration-tests-base - 4.15.0-SNAPSHOT - ../integration-tests-base - - - org.apache.bookkeeper.tests - integration-tests-base-groovy - pom - - Apache BookKeeper :: Tests :: Base module for Arquillian based integration tests using groovy - - - 3.6.0-03 - 3.0.2-02 - - - - - - maven-compiler-plugin - - - ${javac.target} - ${javac.target} - groovy-eclipse-compiler - - - - org.codehaus.groovy - groovy-eclipse-compiler - ${groovy-eclipse-compiler.version} - - - org.codehaus.groovy - groovy-eclipse-batch - ${groovy-eclipse-batch.version} - - - - - org.codehaus.groovy - groovy-eclipse-compiler - ${groovy-eclipse-compiler.version} - true - - - org.codehaus.gmaven - groovy-maven-plugin - 2.0 - - - org.codehaus.groovy - groovy-all - ${groovy.version} - pom - - - - - org.apache.maven.plugins - maven-surefire-plugin - - 2.8.1 - - -Xmx4G -Djava.net.preferIPv4Stack=true - 1 - false - - - System.out - true - - - - - - - - - org.codehaus.groovy - groovy-all - pom - - - - - bintray - - false - - groovy-bintray-plugins - https://dl.bintray.com/groovy/maven - - - diff --git a/tests/integration-tests-base/pom.xml b/tests/integration-tests-base/pom.xml deleted file mode 100644 index b5a0ce218d7..00000000000 --- a/tests/integration-tests-base/pom.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-base - pom - - Apache BookKeeper :: Tests :: Base module for Arquillian based integration tests - - - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - test - - - - junit - junit - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/tests/integration-tests-topologies/pom.xml b/tests/integration-tests-topologies/pom.xml deleted file mode 100644 index b2797b09635..00000000000 --- a/tests/integration-tests-topologies/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-topologies - jar - - Apache BookKeeper :: Tests :: Common topologies for Docker based integration tests - - - - org.testcontainers - testcontainers - - - junit - junit - compile - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - - - - diff --git a/tests/integration-tests-utils/pom.xml b/tests/integration-tests-utils/pom.xml deleted file mode 100644 index 6c43bbc1d41..00000000000 --- a/tests/integration-tests-utils/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-utils - jar - - Apache BookKeeper :: Tests :: Utility module for Arquillian based integration tests - - - - org.apache.commons - commons-compress - - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-impl-maven - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-api - - - - org.apache.zookeeper - zookeeper - - - - org.arquillian.cube - arquillian-cube-docker - - - com.github.docker-java - * - - - - - - org.testcontainers - testcontainers - - - - org.codehaus.groovy - groovy-all - pom - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - 2.8.1 - - 1 - false - - - - - - diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index 56960061def..94cdedfab96 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -47,10 +47,6 @@ public class BookKeeperClusterUtils { private static final List OLD_CLIENT_VERSIONS_WITH_CURRENT_LEDGER_METADATA_FORMAT = Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3"); - private static final List OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME = - Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3", "4.3-yahoo"); - - private static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterUtils.class); public static boolean hasVersionLatestMetadataFormat(String version) { @@ -101,7 +97,7 @@ public static boolean metadataFormatIfNeeded(DockerClient docker, String version @Cleanup ZooKeeper zk = BookKeeperClusterUtils.zookeeperClient(docker); if (zk.exists("/ledgers", false) == null) { - String bookkeeper = "/opt/bookkeeper/" + version + "/bin/" + computeBinFilenameByVersion(version); + String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; runOnAnyBookie(docker, bookkeeper, "shell", "metaformat", "-nonInteractive"); return true; } else { @@ -131,7 +127,7 @@ public static String createDlogNamespaceIfNeeded(DockerClient docker, } public static void formatAllBookies(DockerClient docker, String version) throws Exception { - String bookkeeper = "/opt/bookkeeper/" + version + "/bin/" + computeBinFilenameByVersion(version); + String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; BookKeeperClusterUtils.runOnAllBookies(docker, bookkeeper, "shell", "bookieformat", "-nonInteractive"); } @@ -259,11 +255,4 @@ public static boolean waitAllBookieUp(DockerClient docker) { .map((b) -> waitBookieUp(docker, b, 10, TimeUnit.SECONDS)) .reduce(true, BookKeeperClusterUtils::allTrue); } - - private static String computeBinFilenameByVersion(String version) { - if (OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME.contains(version)) { - return "bookkeeper"; - } - return "bookkeeper_gradle"; - } } diff --git a/tests/integration/cluster/pom.xml b/tests/integration/cluster/pom.xml deleted file mode 100644 index 604aeab3e1f..00000000000 --- a/tests/integration/cluster/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - cluster - jar - Apache BookKeeper :: Tests :: Integration :: Cluster test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml deleted file mode 100644 index 1d7535f59bd..00000000000 --- a/tests/integration/pom.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - integration - Apache BookKeeper :: Tests :: Integration - - smoke - standalone - cluster - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/tests/integration/smoke/pom.xml b/tests/integration/smoke/pom.xml deleted file mode 100644 index 35cc17511b8..00000000000 --- a/tests/integration/smoke/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - smoke - jar - Apache BookKeeper :: Tests :: Integration :: Smoke test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/integration/standalone/pom.xml b/tests/integration/standalone/pom.xml deleted file mode 100644 index 7a9cb5a4272..00000000000 --- a/tests/integration/standalone/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - standalone - jar - Apache BookKeeper :: Tests :: Integration :: Standalone test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/pom.xml b/tests/pom.xml deleted file mode 100644 index ae4f4b8ab3c..00000000000 --- a/tests/pom.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - tests-parent - Apache BookKeeper :: Tests - - - 2.5.8 - - - - shaded - docker-images - integration-tests-base - integration-tests-base-groovy - integration-tests-utils - integration-tests-topologies - backward-compat - integration - scripts - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/tests/scripts/pom.xml b/tests/scripts/pom.xml deleted file mode 100644 index 4bbc9e80021..00000000000 --- a/tests/scripts/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - scripts - jar - Apache BookKeeper :: Tests :: Bash Scripts Test - - - - - - - - com.googlecode.maven-download-plugin - download-maven-plugin - - - install-shunit2 - integration-test - - wget - - - https://github.com/kward/shunit2/archive/v2.1.7.zip - true - ${project.basedir}/target/lib - ${skipTests} - - - - - - org.codehaus.mojo - exec-maven-plugin - - - bash-tests - integration-test - - exec - - - ${skipTests} - ${project.basedir}/src/test/bash - ${project.basedir}/src/test/bash/bk_test.sh - - - - - - - diff --git a/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh b/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh index c20165f6050..9a1f7f3917e 100644 --- a/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh +++ b/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh @@ -27,7 +27,7 @@ # testDefaultVariables() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh assertEquals "BINDIR is not set correctly" "${BK_BINDIR}" "${BINDIR}" assertEquals "BK_HOME is not set correctly" "${BK_HOMEDIR}" "${BK_HOME}" assertEquals "DEFAULT_LOG_CONF is not set correctly" "${BK_CONFDIR}/log4j.properties" "${DEFAULT_LOG_CONF}" @@ -48,7 +48,7 @@ testDefaultVariables() { } testFindModuleJarAt() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh MODULE="test-module" @@ -113,7 +113,7 @@ testFindModuleJar() { echo "" > ${BK_HOME}/conf/bkenv.sh echo "" > ${BK_HOME}/conf/bk_cli_env.sh - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh MODULE="test-module" MODULE_PATH="testmodule" @@ -159,7 +159,7 @@ testLoadEnvfiles() { echo "CLI_MAX_HEAP_MEMORY=2048M" > ${BK_HOME}/conf/bk_cli_env.sh # load the common_gradle.sh - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh assertEquals "NETTY_LEAK_DETECTION_LEVEL is not set correctly" "enabled" "${NETTY_LEAK_DETECTION_LEVEL}" assertEquals "BOOKIE_MAX_HEAP_MEMORY is not set correctly" "2048M" "${BOOKIE_MAX_HEAP_MEMORY}" @@ -172,7 +172,7 @@ testLoadEnvfiles() { } testBuildBookieJVMOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh TEST_LOG_DIR=${BK_TMPDIR}/logdir TEST_GC_LOG_FILENAME="test-gc.log" @@ -187,7 +187,7 @@ testBuildBookieJVMOpts() { } testBuildCLIJVMOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh TEST_LOG_DIR=${BK_TMPDIR}/logdir TEST_GC_LOG_FILENAME="test-gc.log" @@ -202,7 +202,7 @@ testBuildCLIJVMOpts() { } testBuildNettyOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh ACTUAL_NETTY_OPTS=$(build_netty_opts) EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled \ @@ -213,7 +213,7 @@ testBuildNettyOpts() { } testBuildBookieOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh ACTUAL_OPTS=$(build_bookie_opts) EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true" diff --git a/tests/shaded/bookkeeper-server-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-shaded-test/pom.xml deleted file mode 100644 index cc548bc2c6f..00000000000 --- a/tests/shaded/bookkeeper-server-shaded-test/pom.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-shaded-test - Apache BookKeeper :: Tests :: bookkeeper-server-shaded test - - - org.apache.bookkeeper - bookkeeper-server-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - - diff --git a/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml deleted file mode 100644 index 2a8cb5490c3..00000000000 --- a/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-tests-shaded-test - Apache BookKeeper :: Tests :: bookkeeper-server-tests-shaded test - - - org.apache.bookkeeper - bookkeeper-server-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - org.apache.bookkeeper - bookkeeper-server-tests-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/tests/shaded/distributedlog-core-shaded-test/pom.xml b/tests/shaded/distributedlog-core-shaded-test/pom.xml deleted file mode 100644 index 7b2206c4ccf..00000000000 --- a/tests/shaded/distributedlog-core-shaded-test/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - distributedlog-core-shaded-test - Apache BookKeeper :: Tests :: distributedlog-core-shaded test - - - org.apache.distributedlog - distributedlog-core-shaded - ${project.version} - test - - - - org.apache.bookkeeper - bookkeeper-server - - - org.apache.bookkeeper.http - bookkeeper-http - - - org.apache.bookkeeper - circe-checksum - - - org.apache.distributedlog - distributedlog-core - - - org.apache.distributedlog - distributedlog-common - - - org.apache.distributedlog - distributedlog-protocol - - - org.apache.zookeeper - zookeeper - - - org.apache.thrift - libthrift - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/tests/shaded/pom.xml b/tests/shaded/pom.xml deleted file mode 100644 index f86a357115b..00000000000 --- a/tests/shaded/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - Apache BookKeeper :: Tests :: Test Shaded Jars - - bookkeeper-server-shaded-test - bookkeeper-server-tests-shaded-test - distributedlog-core-shaded-test - - diff --git a/tools/all/build.gradle b/tools/all/build.gradle index 7593b6a2c6e..670f2c41a72 100644 --- a/tools/all/build.gradle +++ b/tools/all/build.gradle @@ -36,5 +36,6 @@ dependencies { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-tools' } diff --git a/tools/all/pom.xml b/tools/all/pom.xml deleted file mode 100644 index ac15e6bb12a..00000000000 --- a/tools/all/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools - Apache BookKeeper :: Tools - - - org.apache.bookkeeper - bookkeeper-tools-ledger - ${project.version} - - - org.apache.bookkeeper - stream-storage-cli - ${project.version} - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - append-ledger-commands - generate-resources - - run - - - - - - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - append-stream-commands - generate-resources - - run - - - - - - - - - - - - - - - diff --git a/tools/framework/pom.xml b/tools/framework/pom.xml deleted file mode 100644 index 9f5e86b1ef6..00000000000 --- a/tools/framework/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-framework - Apache BookKeeper :: Tools :: Framework - - - com.beust - jcommander - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - - - org.apache.bookkeeper - buildtools - ${project.parent.version} - test - - - diff --git a/tools/ledger/pom.xml b/tools/ledger/pom.xml deleted file mode 100644 index 8becea1f281..00000000000 --- a/tools/ledger/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-ledger - Apache BookKeeper :: Tools :: Ledger - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - - - org.apache.logging.log4j - log4j-1.2-api - runtime - - - org.apache.logging.log4j - log4j-core - runtime - - - org.apache.logging.log4j - log4j-slf4j-impl - runtime - - - org.apache.bookkeeper - buildtools - ${project.parent.version} - test - - - org.apache.bookkeeper - bookkeeper-server - test-jar - ${project.parent.version} - test - - - diff --git a/tools/perf/build.gradle b/tools/perf/build.gradle index 1223e2000d0..80df1221ae9 100644 --- a/tools/perf/build.gradle +++ b/tools/perf/build.gradle @@ -67,5 +67,6 @@ dependencies { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-tools-perf' } diff --git a/tools/perf/pom.xml b/tools/perf/pom.xml deleted file mode 100644 index a0c39e39687..00000000000 --- a/tools/perf/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper-tools-parent - 4.15.0-SNAPSHOT - - bookkeeper-perf - Apache BookKeeper :: Tools :: Perf - - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - org.hdrhistogram - HdrHistogram - - - - diff --git a/tools/pom.xml b/tools/pom.xml deleted file mode 100644 index 50010001879..00000000000 --- a/tools/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-parent - Apache BookKeeper :: Tools :: Parent - pom - - framework - ledger - stream - perf - all - - diff --git a/tools/stream/pom.xml b/tools/stream/pom.xml deleted file mode 100644 index ba9eea73590..00000000000 --- a/tools/stream/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper-tools-parent - 4.15.0-SNAPSHOT - - stream-storage-cli - Apache BookKeeper :: Tools :: Stream - - - - org.apache.bookkeeper - stream-storage-java-client - ${project.version} - - - org.apache.bookkeeper - stream-storage-service-impl - ${project.version} - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - From 62aca0a237851c28cc68650962924c1b45022446 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 27 Jan 2022 05:20:34 -0800 Subject: [PATCH 04/79] Upgraded snakeyaml, CVE-2017-18640 (#3014) --- dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.gradle b/dependencies.gradle index 075b3bb7091..3c334b3c9f5 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -75,7 +75,7 @@ depVersions = [ rocksDb: "6.22.1.1", rxjava: "3.0.1", slf4j: "1.7.32", - snakeyaml: "1.19", + snakeyaml: "1.30", spotbugsAnnotations: "3.1.8", protocGenGrpcJava: "1.12.0", shrinkwrap:"3.1.4", From de3019e41692343b69dea28cdc9400692473b638 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 27 Jan 2022 05:20:53 -0800 Subject: [PATCH 05/79] Upgrading protobuf, CVE-2021-22569 (#3013) --- dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.gradle b/dependencies.gradle index 3c334b3c9f5..3b955a96609 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -70,7 +70,7 @@ depVersions = [ nettyTcnativeBoringSsl: "2.0.46.Final", powermock: "2.0.2", prometheus: "0.8.1", - protobuf: "3.14.0", + protobuf: "3.16.1", reflections: "0.9.11", rocksDb: "6.22.1.1", rxjava: "3.0.1", From 036fc1f22228a14c7f91d4f119378fc6cf719ed5 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 27 Jan 2022 05:24:44 -0800 Subject: [PATCH 06/79] Added OWASP dependency-check (#3002) * Added OWASP dependency-check * Suppress ETCD-related misdetections --- build.gradle | 43 +++++++++++++ gradle.properties | 1 + settings.gradle | 1 + src/owasp-dependency-check-suppressions.xml | 71 +++++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 src/owasp-dependency-check-suppressions.xml diff --git a/build.gradle b/build.gradle index b92a70cbf41..a3f4416c156 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,7 @@ plugins { id 'checkstyle' id 'org.nosphere.apache.rat' id 'com.github.spotbugs' + id 'org.owasp.dependencycheck' } subprojects { @@ -58,6 +59,13 @@ releaseParams { // ReleaseExtension } } +def skipDepCheck = [] +allprojects { + if (it.path.startsWith(':tests')) { + skipDepCheck << it.path + } +} + allprojects { apply from: "$rootDir/dependencies.gradle" if (it.path != ':circe-checksum:src:main:circe' @@ -70,6 +78,41 @@ allprojects { apply plugin: 'org.nosphere.apache.rat' apply plugin: "checkstyle" apply plugin: 'com.github.spotbugs' + + if (!it.path.startsWith(':tests')) { + apply plugin: 'org.owasp.dependencycheck' + + dependencyCheck { + // see https://jeremylong.github.io/DependencyCheck/dependency-check-gradle/configuration.html + // for configuration properties + suppressionFile = "$rootDir/src/owasp-dependency-check-suppressions.xml" + skipProjects = skipDepCheck + skipConfigurations = ["checkstyle", "spotbugs"] + analyzers { + msbuildEnabled = false + rubygemsEnabled = false + pyDistributionEnabled = false + pyPackageEnabled = false + nuspecEnabled = false + nugetconfEnabled = false + assemblyEnabled = false + cmakeEnabled = false + composerEnabled = false + cpanEnabled = false + nodeEnabled = false + cocoapodsEnabled = false + swiftEnabled = false + swiftPackageResolvedEnabled = false + bundleAuditEnabled = false + golangDepEnabled = false + golangModEnabled = false + + nodeAudit.enabled = false + retirejs.enabled = false + } + } + } + checkstyle { toolVersion "${checkStyleVersion}" configFile file("$rootDir/buildtools/src/main/resources/bookkeeper/checkstyle.xml") diff --git a/gradle.properties b/gradle.properties index b8be5831081..bba0e1dfa0a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -27,3 +27,4 @@ checkStyleVersion=6.19 spotbugsPlugin=4.7.0 testLogger=2.0.0 testRetry=1.0.0 +owaspPlugin=6.5.3 diff --git a/settings.gradle b/settings.gradle index 3d192cfd755..036b10413e0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,6 +25,7 @@ pluginManagement { id "com.github.spotbugs" version "${spotbugsPlugin}" id "com.adarshr.test-logger" version "${testLogger}" id "org.gradle.test-retry" version "${testRetry}" + id "org.owasp.dependencycheck" version "${owaspPlugin}" } } diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml new file mode 100644 index 00000000000..c65b18b1b2e --- /dev/null +++ b/src/owasp-dependency-check-suppressions.xml @@ -0,0 +1,71 @@ + + + + + + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15106 + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15112 + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15113 + + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15106 + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15112 + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15113 + + + + From 568fea837c8c793da0020abd9b7c0dd3f525c433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 27 Jan 2022 16:08:59 +0100 Subject: [PATCH 07/79] [build] do not run spotbugsTest by default (#3017) --- .github/workflows/pr-validation.yml | 2 +- build.gradle | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index f96a401dca3..816315b3707 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -47,6 +47,6 @@ jobs: with: java-version: 1.8 - name: Validate pull request - run: ./gradlew build -x spotbugsTest -x signDistTar -x test + run: ./gradlew build -x signDistTar -x test - name: Check license files run: dev/check-all-licenses diff --git a/build.gradle b/build.gradle index a3f4416c156..86495b445fa 100644 --- a/build.gradle +++ b/build.gradle @@ -130,6 +130,7 @@ allprojects { toolVersion = '3.1.8' excludeFilter = file("$rootDir/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml") reportLevel = 'high' + spotbugsTest.enabled = false } From 317080d44334b1751d70de61f27e60505ae834cb Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 2 Feb 2022 09:12:38 -0800 Subject: [PATCH 08/79] Forcing the same version of netty across the projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation While experimenting with OWASP dependency checker I noticed that we have 3 versions of netty mixed in: 4.1.72 (current one, expected) plus 4.1.63 and 4.1.50 (brought with ZK and some other dependencies). ### Changes Made gradle enforce the same version of netty in subprojects. Reviewers: Nicolò Boschi , Enrico Olivelli This closes #3008 from dlg99/gradle-netty --- build.gradle | 3 +++ dependencies.gradle | 1 + 2 files changed, 4 insertions(+) diff --git a/build.gradle b/build.gradle index 86495b445fa..03df876f96a 100644 --- a/build.gradle +++ b/build.gradle @@ -280,6 +280,9 @@ allprojects { showStandardStreams = true } } + dependencies { + implementation(enforcedPlatform(depLibs.nettyBom)) + } tasks.register('writeClasspath') { doLast { buildDir.mkdirs() diff --git a/dependencies.gradle b/dependencies.gradle index 3b955a96609..ff760320805 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -176,6 +176,7 @@ depLibs = [ metricsJvm: "io.dropwizard.metrics:metrics-jvm:${depVersions.dropwizard}", metricsGraphite: "io.dropwizard.metrics:metrics-graphite:${depVersions.dropwizard}", mockito: "org.mockito:mockito-core:${depVersions.mockito}", + nettyBom: "io.netty:netty-bom:${depVersions.netty}", nettyBuffer: "io.netty:netty-buffer:${depVersions.netty}", nettyCommon: "io.netty:netty-common:${depVersions.netty}", nettyHandler: "io.netty:netty-handler:${depVersions.netty}", From f02e9e0218b466a4fdf4122a0e0d64c713d96b47 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 2 Feb 2022 09:14:26 -0800 Subject: [PATCH 09/79] Forced the same version of guava (plus upgraded it); fixed deprecations etc. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation Older versions of guava (w/CVEs) used in some subprojects ### Changes Forced the same version of guava; fixed deprecation problems (murmur3_32) and compilation problems (checkArgument). checkArgument cannot be statically imported because there are now overrides of it; checkstyle was not very cooperative so I had to remove the import altogether. Then updated the guava version to match one in Pulsar. Reviewers: Enrico Olivelli , Nicolò Boschi This closes #3010 from dlg99/gradle-guava --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 8 ++++---- bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt | 8 ++++---- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 8 ++++---- .../main/java/org/apache/bookkeeper/metastore/Value.java | 2 +- build.gradle | 5 +++++ .../bookkeeper/common/util/affinity/impl/NativeUtils.java | 4 +--- .../common/util/affinity/impl/ProcessorsInfo.java | 6 ++---- dependencies.gradle | 3 ++- .../apache/bookkeeper/common/router/HashRouterTest.java | 2 +- 9 files changed, 24 insertions(+), 22 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 2721b5b2886..2799377f88f 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -312,7 +312,7 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ @@ -667,10 +667,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 5d5e822d4f2..0183ae58fae 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -289,7 +289,7 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ @@ -594,10 +594,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 23f1c9a265b..23a197ccd64 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -310,7 +310,7 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ @@ -659,10 +659,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java index 43354c344e5..9a3b43e12d6 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java @@ -92,7 +92,7 @@ public Value project(Set fields) { @Override public int hashCode() { - HashFunction hf = Hashing.murmur3_32(); + HashFunction hf = Hashing.murmur3_32_fixed(); Hasher hc = hf.newHasher(); for (String key : fields.keySet()) { hc.putString(key, Charset.defaultCharset()); diff --git a/build.gradle b/build.gradle index 03df876f96a..a06c27fa8e6 100644 --- a/build.gradle +++ b/build.gradle @@ -289,6 +289,11 @@ allprojects { new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" } } + + dependencies { + implementation(enforcedPlatform(depLibs.guavaBom)) + } + } repositories { diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java index 67c8679ab12..218e9f1af5e 100644 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java +++ b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java @@ -20,8 +20,6 @@ */ package org.apache.bookkeeper.common.util.affinity.impl; -import static com.google.common.base.Preconditions.checkArgument; - import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.File; @@ -50,7 +48,7 @@ public class NativeUtils { value = "OBL_UNSATISFIED_OBLIGATION", justification = "work around for java 9: https://github.com/spotbugs/spotbugs/issues/493") public static void loadLibraryFromJar(String path) throws Exception { - checkArgument(path.startsWith("/"), "absolute path must start with /"); + com.google.common.base.Preconditions.checkArgument(path.startsWith("/"), "absolute path must start with /"); String[] parts = path.split("/"); String filename = (parts.length > 0) ? parts[parts.length - 1] : null; diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java index 8a812a84d1a..a1b3963d2c7 100644 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java +++ b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java @@ -20,8 +20,6 @@ */ package org.apache.bookkeeper.common.util.affinity.impl; -import static com.google.common.base.Preconditions.checkArgument; - import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -80,8 +78,8 @@ static ProcessorsInfo parseCpuInfo(String cpuInfoString) { } } - checkArgument(cpuId >= 0); - checkArgument(coreId >= 0); + com.google.common.base.Preconditions.checkArgument(cpuId >= 0); + com.google.common.base.Preconditions.checkArgument(coreId >= 0); pi.cpus.put(cpuId, coreId); } diff --git a/dependencies.gradle b/dependencies.gradle index ff760320805..920288b3b02 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -45,7 +45,7 @@ depVersions = [ gradleTooling: "4.0.1", grpc: "1.42.1", groovy: "2.5.8", - guava: "30.0-jre", + guava: "31.0.1-jre", hamcrest: "1.3", hdrhistogram: "2.1.10", httpclient: "4.5.13", @@ -139,6 +139,7 @@ depLibs = [ exclude group: 'com.google.guava', module: 'guava' }, groovy: "org.codehaus.groovy:groovy-all:${depVersions.groovy}", + guavaBom: "com.google.guava:guava-bom:${depVersions.guava}", guava: dependencies.create("com.google.guava:guava:${depVersions.guava}"){ exclude group: 'com.google.code.findbugs', module: 'jsr305' }, diff --git a/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java b/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java index a36b29123b4..4f46cb05b1a 100644 --- a/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java +++ b/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java @@ -40,7 +40,7 @@ public void testByteBufHashRouter() { int hash32 = Murmur3.hash32( key, key.readerIndex(), key.readableBytes(), (int) AbstractHashRouter.HASH_SEED); int bytesHash32 = Murmur3.hash32(keyBytes, 0, keyBytes.length, (int) AbstractHashRouter.HASH_SEED); - int guavaHash32 = Hashing.murmur3_32((int) AbstractHashRouter.HASH_SEED) + int guavaHash32 = Hashing.murmur3_32_fixed((int) AbstractHashRouter.HASH_SEED) .newHasher() .putString("foo", UTF_8) .hash() From 3f068390a66706b41798d562835278bb37227b97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 2 Feb 2022 18:17:07 +0100 Subject: [PATCH 10/79] Upgrade Netty to 4.1.73.Final ### Motivation Changelog: https://netty.io/news/2022/01/12/4-1-73-Final.html The main reason to upgrade is because of an [intensive I/O disk scheduled task](https://github.com/netty/netty/pull/11943) introduced in 4.1.72.Final which is synchronous and can cause EventLoop to blocked very often. ### Changes * Upgrade Netty from 4.1.72.Final to 4.1.73.Final * [Netty 4.1.73.Final depends on netty-tc-native 2.0.46](https://github.com/netty/netty/blob/b5219aeb4ee62f15d5dfb2b9c29d0c694aca05be/pom.xml#L545) as Netty 4.1.72.Final, so no need to upgrade Reviewers: Andrey Yegorov This closes #3020 from nicoloboschi/upgrade-netty-4.1.73 --- bookkeeper-dist/all/build.gradle | 2 +- bookkeeper-dist/bkctl/build.gradle | 2 +- bookkeeper-dist/server/build.gradle | 2 +- bookkeeper-dist/src/assemble/bin-all.xml | 2 +- bookkeeper-dist/src/assemble/bin-server.xml | 2 +- bookkeeper-dist/src/assemble/bkctl.xml | 2 +- .../src/main/resources/LICENSE-all.bin.txt | 144 +++++++++--------- .../src/main/resources/LICENSE-bkctl.bin.txt | 140 ++++++++--------- .../src/main/resources/LICENSE-server.bin.txt | 144 +++++++++--------- .../src/main/resources/NOTICE-all.bin.txt | 30 ++-- .../src/main/resources/NOTICE-bkctl.bin.txt | 26 ++-- .../src/main/resources/NOTICE-server.bin.txt | 30 ++-- .../LICENSE.aalto-xml.txt | 0 .../LICENSE.base64.txt | 0 .../LICENSE.bouncycastle.txt | 0 .../LICENSE.caliper.txt | 0 .../LICENSE.commons-lang.txt | 0 .../LICENSE.commons-logging.txt | 0 .../LICENSE.compress-lzf.txt | 0 .../LICENSE.dnsinfo.txt | 0 .../LICENSE.harmony.txt | 0 .../LICENSE.hpack.txt | 0 .../LICENSE.hyper-hpack.txt | 0 .../LICENSE.jboss-marshalling.txt | 0 .../LICENSE.jbzip2.txt | 0 .../LICENSE.jctools.txt | 0 .../LICENSE.jfastlz.txt | 0 .../LICENSE.jsr166y.txt | 0 .../LICENSE.jzlib.txt | 0 .../LICENSE.libdivsufsort.txt | 0 .../LICENSE.log4j.txt | 0 .../LICENSE.lz4.txt | 0 .../LICENSE.lzma-java.txt | 0 .../LICENSE.mvn-wrapper.txt | 0 .../LICENSE.nghttp2-hpack.txt | 0 .../LICENSE.protobuf.txt | 0 .../LICENSE.slf4j.txt | 0 .../LICENSE.snappy.txt | 0 .../LICENSE.webbit.txt | 0 .../NOTICE.harmony.txt | 0 dependencies.gradle | 2 +- 41 files changed, 264 insertions(+), 264 deletions(-) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.aalto-xml.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.base64.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.bouncycastle.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.caliper.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.commons-lang.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.commons-logging.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.compress-lzf.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.dnsinfo.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.harmony.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.hpack.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.hyper-hpack.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jboss-marshalling.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jbzip2.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jctools.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jfastlz.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jsr166y.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jzlib.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.libdivsufsort.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.log4j.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.lz4.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.lzma-java.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.mvn-wrapper.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.nghttp2-hpack.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.protobuf.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.slf4j.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.snappy.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.webbit.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/NOTICE.harmony.txt (100%) diff --git a/bookkeeper-dist/all/build.gradle b/bookkeeper-dist/all/build.gradle index 68f1043eff8..3657337a0a9 100644 --- a/bookkeeper-dist/all/build.gradle +++ b/bookkeeper-dist/all/build.gradle @@ -60,7 +60,7 @@ def depLicences = [ "javax.servlet-api-4.0.0/CDDL+GPL-1.1", "bouncycastle-1.0.2.1/LICENSE.html", "jsr-305/LICENSE", - "netty-4.1.72.Final/*", + "netty-4.1.73.Final/*", "paranamer-2.8/LICENSE.txt", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/bkctl/build.gradle b/bookkeeper-dist/bkctl/build.gradle index 2495b7b0148..34acdf98a26 100644 --- a/bookkeeper-dist/bkctl/build.gradle +++ b/bookkeeper-dist/bkctl/build.gradle @@ -49,7 +49,7 @@ releaseArtifacts { def depLicences = [ "checker-qual-3.5.0/LICENSE", "google-auth-library-credentials-0.20.0/LICENSE", - "netty-4.1.72.Final/*", + "netty-4.1.73.Final/*", "bouncycastle-1.0.2.1/LICENSE.html", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/server/build.gradle b/bookkeeper-dist/server/build.gradle index ba2d50da268..5c3c3fb97d6 100644 --- a/bookkeeper-dist/server/build.gradle +++ b/bookkeeper-dist/server/build.gradle @@ -53,7 +53,7 @@ def depLicences = [ "checker-qual-3.5.0/LICENSE", "google-auth-library-credentials-0.20.0/LICENSE", "javax.servlet-api-4.0.0/CDDL+GPL-1.1", - "netty-4.1.72.Final/*", + "netty-4.1.73.Final/*", "bouncycastle-1.0.2.1/LICENSE.html", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/src/assemble/bin-all.xml b/bookkeeper-dist/src/assemble/bin-all.xml index 4ec9dea8221..3d700895169 100644 --- a/bookkeeper-dist/src/assemble/bin-all.xml +++ b/bookkeeper-dist/src/assemble/bin-all.xml @@ -59,7 +59,7 @@ javax.servlet-api-4.0.0/CDDL+GPL-1.1 bouncycastle-1.0.2.1/LICENSE.html jsr-305/LICENSE - netty-4.1.72.Final/* + netty-4.1.73.Final/* paranamer-2.8/LICENSE.txt protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/assemble/bin-server.xml b/bookkeeper-dist/src/assemble/bin-server.xml index cfb176d3d41..439491e5e9b 100644 --- a/bookkeeper-dist/src/assemble/bin-server.xml +++ b/bookkeeper-dist/src/assemble/bin-server.xml @@ -52,7 +52,7 @@ checker-qual-3.5.0/LICENSE google-auth-library-credentials-0.20.0/LICENSE javax.servlet-api-4.0.0/CDDL+GPL-1.1 - netty-4.1.72.Final/* + netty-4.1.73.Final/* bouncycastle-1.0.2.1/LICENSE.html protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/assemble/bkctl.xml b/bookkeeper-dist/src/assemble/bkctl.xml index 855f410d962..18e3904c2cc 100644 --- a/bookkeeper-dist/src/assemble/bkctl.xml +++ b/bookkeeper-dist/src/assemble/bkctl.xml @@ -66,7 +66,7 @@ checker-qual-3.5.0/LICENSE google-auth-library-credentials-0.20.0/LICENSE - netty-4.1.72.Final/* + netty-4.1.73.Final/* bouncycastle-1.0.2.1/LICENSE.html protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 2799377f88f..adfb859d517 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -217,23 +217,23 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar [11] +- lib/io.netty-netty-common-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.73.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-transport-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar [11] - lib/io.prometheus-simpleclient-0.8.1.jar [12] - lib/io.prometheus-simpleclient_common-0.8.1.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.8.1.jar [12] @@ -319,7 +319,7 @@ Apache Software License, Version 2. [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.73.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.8.1 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/3.9.8 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/3.9.8 @@ -359,229 +359,229 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.73.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.73.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.73.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.73.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.73.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.73.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.73.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.73.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 0183ae58fae..9a55d42c0f4 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -217,21 +217,21 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar [11] +- lib/io.netty-netty-common-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.73.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-transport-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar [11] - lib/org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar [16] - lib/org.apache.logging.log4j-log4j-api-2.17.1.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.17.1.jar [16] @@ -296,7 +296,7 @@ Apache Software License, Version 2. [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.73.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.17.1 [17] Source available at https://github.com/java-native-access/jna/tree/3.2.7 [18] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-collections.git;a=tag;h=a3a5ad @@ -328,229 +328,229 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.73.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.73.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.73.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.73.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.73.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.73.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.73.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.73.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 23a197ccd64..a00cd92b9a9 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -217,23 +217,23 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar [11] +- lib/io.netty-netty-common-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.73.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-transport-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar [11] - lib/io.prometheus-simpleclient-0.8.1.jar [12] - lib/io.prometheus-simpleclient_common-0.8.1.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.8.1.jar [12] @@ -317,7 +317,7 @@ Apache Software License, Version 2. [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.73.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.8.1 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/3.9.8 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/3.9.8 @@ -357,229 +357,229 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.73.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.73.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.73.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.73.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.73.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.73.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.73.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.73.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index 0bff137ce70..f145e9e067b 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -22,23 +22,23 @@ LongAdder), which was released with the following comments: http://creativecommons.org/publicdomain/zero/1.0/ ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.73.Final.jar +- lib/io.netty-netty-codec-4.1.73.Final.jar +- lib/io.netty-netty-codec-dns-4.1.73.Final.jar +- lib/io.netty-netty-codec-http-4.1.73.Final.jar +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar +- lib/io.netty-netty-common-4.1.73.Final.jar +- lib/io.netty-netty-handler-4.1.73.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar +- lib/io.netty-netty-resolver-4.1.73.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.73.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-transport-4.1.73.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 8ee2bd437e0..bffec61c02b 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -5,21 +5,21 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.73.Final.jar +- lib/io.netty-netty-codec-4.1.73.Final.jar +- lib/io.netty-netty-codec-http-4.1.73.Final.jar +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar +- lib/io.netty-netty-common-4.1.73.Final.jar +- lib/io.netty-netty-handler-4.1.73.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar +- lib/io.netty-netty-resolver-4.1.73.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-transport-4.1.73.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index a991cd6ba09..f9dbca9ddf7 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -5,23 +5,23 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.73.Final.jar +- lib/io.netty-netty-codec-4.1.73.Final.jar +- lib/io.netty-netty-codec-dns-4.1.73.Final.jar +- lib/io.netty-netty-codec-http-4.1.73.Final.jar +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar +- lib/io.netty-netty-common-4.1.73.Final.jar +- lib/io.netty-netty-handler-4.1.73.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar +- lib/io.netty-netty-resolver-4.1.73.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.73.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-transport-4.1.73.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.base64.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.base64.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.base64.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.base64.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.caliper.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.caliper.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.caliper.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.caliper.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-lang.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.commons-lang.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-lang.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.commons-lang.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-logging.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.commons-logging.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-logging.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.commons-logging.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.harmony.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.harmony.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.harmony.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.harmony.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jbzip2.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jbzip2.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jbzip2.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jbzip2.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jctools.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jctools.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jctools.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jctools.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jfastlz.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jfastlz.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jfastlz.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jfastlz.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jsr166y.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jsr166y.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jsr166y.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jsr166y.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jzlib.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jzlib.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jzlib.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jzlib.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.log4j.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.log4j.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.log4j.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.log4j.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lz4.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.lz4.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lz4.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.lz4.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lzma-java.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.lzma-java.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lzma-java.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.lzma-java.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.protobuf.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.protobuf.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.protobuf.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.protobuf.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.slf4j.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.slf4j.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.slf4j.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.slf4j.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.snappy.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.snappy.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.snappy.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.snappy.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.webbit.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.webbit.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.webbit.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.webbit.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/NOTICE.harmony.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/NOTICE.harmony.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/NOTICE.harmony.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/NOTICE.harmony.txt diff --git a/dependencies.gradle b/dependencies.gradle index 920288b3b02..b822b63af62 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -66,7 +66,7 @@ depVersions = [ lombok: "1.18.20", lz4: "1.3.0", mockito: "3.0.0", - netty: "4.1.72.Final", + netty: "4.1.73.Final", nettyTcnativeBoringSsl: "2.0.46.Final", powermock: "2.0.2", prometheus: "0.8.1", From d7e5c51aaa5e1faa781cf07d261a63319a946326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 3 Feb 2022 08:39:33 +0100 Subject: [PATCH 11/79] [website] Remove Maven references and remove dead links from README (#3018) --- README.md | 16 +++---- site/community/contributing.md | 15 ------- .../latest/getting-started/installation.md | 45 +++---------------- 3 files changed, 12 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 53876ee9e3b..29ed1e435f5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ logo -[![Coverage Status](https://coveralls.io/repos/github/apache/bookkeeper/badge.svg?branch=master)](https://coveralls.io/github/apache/bookkeeper?branch=master) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.bookkeeper/bookkeeper/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.bookkeeper/bookkeeper) # Apache BookKeeper @@ -29,7 +28,7 @@ Please visit the [Documentation](https://bookkeeper.apache.org/docs/latest/overv ### Report a Bug -For filing bugs, suggesting improvements, or requesting new features, help us out by [opening a Github issue](https://github.com/apache/bookkeeper/issues) or [opening an Apache jira](https://issues.apache.org/jira/browse/BOOKKEEPER). +For filing bugs, suggesting improvements, or requesting new features, help us out by [opening a Github issue](https://github.com/apache/bookkeeper/issues). ### Need Help? @@ -45,15 +44,10 @@ We feel that a welcoming open community is important and welcome contributions. ### Contributing Code -1. See [Developer Setup](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Developer+Setup) to get your local environment setup. +1. See our [installation guide](https://bookkeeper.apache.org/docs/latest/getting-started/installation/) to get your local environment setup. -2. Take a look at our open issues: [JIRA Issues](https://issues.apache.org/jira/browse/BOOKKEEPER) [Github Issues](https://github.com/apache/bookkeeper/issues). +2. Take a look at our open issues: [Github Issues](https://github.com/apache/bookkeeper/issues). -3. Review our [coding style](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Coding+Guide) and follow our [pull requests](https://github.com/apache/bookkeeper/pulls) to learn about our conventions. - -4. Make your changes according to our [contribution guide](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Contributing+to+BookKeeper). - -### Improving Website and Documentation - -1. See [Building the website and documentation](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Building+the+website+and+documentation) on how to build the website and documentation. +3. Review our [coding style](https://bookkeeper.apache.org/community/coding_guide/) and follow our [pull requests](https://github.com/apache/bookkeeper/pulls) to learn more about our conventions. +4. Make your changes according to our [contributing guide](https://bookkeeper.apache.org/community/contributing/) diff --git a/site/community/contributing.md b/site/community/contributing.md index 7bf513b3c6a..b56dcb742cd 100644 --- a/site/community/contributing.md +++ b/site/community/contributing.md @@ -102,20 +102,6 @@ Depending on your preferred development environment, you may need to prepare it ##### IntelliJ -###### Enable Annotation Processing - -To configure annotation processing in IntelliJ: - -1. Open Annotation Processors Settings dialog box by going to Settings -> Build, Execution, Deployment -> Compiler -> Annotation Processors. -1. Select the following buttons: - 1. "Enable annotation processing" - 1. "Obtain processors from project classpath" - 1. "Store generated sources relative to: Module content root" -1. Set the generated source directories to be equal to the Maven directories: - 1. Set "Production sources directory:" to "target/generated-sources/annotations". - 1. Set "Test sources directory:" to "target/generated-test-sources/test-annotations". -1. Click "OK". - ###### Checkstyle IntelliJ supports checkstyle within the IDE using the Checkstyle-IDEA plugin. @@ -153,7 +139,6 @@ Start Eclipse with a fresh workspace in a separate directory from your checkout. File -> Import... - -> Existing Maven Projects -> Browse to the directory you cloned into and select "bookkeeper" -> make sure all bookkeeper projects are selected -> Finalize diff --git a/site/docs/latest/getting-started/installation.md b/site/docs/latest/getting-started/installation.md index 9f856ec1d21..dcc4a82a24e 100644 --- a/site/docs/latest/getting-started/installation.md +++ b/site/docs/latest/getting-started/installation.md @@ -12,64 +12,33 @@ You can install BookKeeper either by [downloading](#download) a [GZipped](http:/ * [Unix environment](http://www.opengroup.org/unix) * [Java Development Kit 1.8](http://www.oracle.com/technetwork/java/javase/downloads/index.html) or later -* [Maven 3.0](https://maven.apache.org/install.html) or later ## Download -You can download Apache BookKeeper releases from one of many [Apache mirrors](http://www.apache.org/dyn/closer.cgi/bookkeeper). Here's an example for the [apache.claz.org](http://apache.claz.org/bookkeeper) mirror: - -```shell -$ curl -O {{ download_url }} -$ tar xvf bookkeeper-{{ site.latest_release }}-src.tar.gz -$ cd bookkeeper-{{ site.latest_release }} -``` +You can download Apache BookKeeper releases from one of many [Apache mirrors](https://dlcdn.apache.org/bookkeeper/). ## Clone -To build BookKeeper from source, clone the repository, either from the [GitHub mirror]({{ site.github_repo }}) or from the [Apache repository](http://git.apache.org/bookkeeper.git/): +To build BookKeeper from source, clone the repository, either from the [GitHub mirror]({{ site.github_repo }}): ```shell -# From the GitHub mirror $ git clone {{ site.github_repo}} - -# From Apache directly -$ git clone git://git.apache.org/bookkeeper.git/ ``` -## Build using Maven +## Build using Gradle -Once you have the BookKeeper on your local machine, either by [downloading](#download) or [cloning](#clone) it, you can then build BookKeeper from source using Maven: +Once you have the BookKeeper on your local machine, either by [downloading](#download) or [cloning](#clone) it, you can then build BookKeeper from source using Gradle: ```shell -$ mvn package +$ ./gradlew build -x signDistTar -x test ``` -Since 4.8.0, bookkeeper introduces `table service`. If you would like to build and tryout table service, you can build it with `stream` profile. +To run all the tests: ```shell -$ mvn package -Dstream +$ ./gradlew test -x signDistTar ``` -> You can skip tests by adding the `-DskipTests` flag when running `mvn package`. - -### Useful Maven commands - -Some other useful Maven commands beyond `mvn package`: - -Command | Action -:-------|:------ -`mvn clean` | Removes build artifacts -`mvn compile` | Compiles JAR files from Java sources -`mvn compile spotbugs:spotbugs` | Compile using the Maven [SpotBugs](https://github.com/spotbugs/spotbugs-maven-plugin) plugin -`mvn install` | Install the BookKeeper JAR locally in your local Maven cache (usually in the `~/.m2` directory) -`mvn deploy` | Deploy the BookKeeper JAR to the Maven repo (if you have the proper credentials) -`mvn verify` | Performs a wide variety of verification and validation tasks -`mvn apache-rat:check` | Run Maven using the [Apache Rat](http://creadur.apache.org/rat/apache-rat-plugin/) plugin -`mvn compile javadoc:aggregate` | Build Javadocs locally -`mvn -am -pl bookkeeper-dist/server package` | Build a server distribution using the Maven [Assembly](http://maven.apache.org/plugins/maven-assembly-plugin/) plugin - -> You can enable `table service` by adding the `-Dstream` flag when running above commands. - ## Package directory The BookKeeper project contains several subfolders that you should be aware of: From b8351cb8c8994f55fdc6f8c302299408903be304 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 3 Feb 2022 08:24:30 -0800 Subject: [PATCH 12/79] CI workflow to check dependencies with OWASP (#3021) --- .github/workflows/owasp-dep-check.yml | 59 +++++++++++++++++++++++++++ build.gradle | 1 + 2 files changed, 60 insertions(+) create mode 100644 .github/workflows/owasp-dep-check.yml diff --git a/.github/workflows/owasp-dep-check.yml b/.github/workflows/owasp-dep-check.yml new file mode 100644 index 00000000000..a3fbdffe471 --- /dev/null +++ b/.github/workflows/owasp-dep-check.yml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: OWASP Dependency Check + +on: + push: + pull_request: + branches: + - master + - branch-* + paths-ignore: + - 'site/**' + workflow_dispatch: + + +jobs: + check: + + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: run "clean build dependencyCheckAggregate" to trigger dependency check + run: ./gradlew clean build -x signDistTar -x test dependencyCheckAggregate + + - name: Upload report + uses: actions/upload-artifact@v2 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: dependency report + path: build/reports/dependency-check-report.html diff --git a/build.gradle b/build.gradle index a06c27fa8e6..c53719dfaec 100644 --- a/build.gradle +++ b/build.gradle @@ -88,6 +88,7 @@ allprojects { suppressionFile = "$rootDir/src/owasp-dependency-check-suppressions.xml" skipProjects = skipDepCheck skipConfigurations = ["checkstyle", "spotbugs"] + failBuildOnCVSS = 7 analyzers { msbuildEnabled = false rubygemsEnabled = false From ee8c5bc2a32f56e3056dd7e5cc12cf292b1cbed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Sat, 5 Feb 2022 00:10:37 +0100 Subject: [PATCH 13/79] [BACKWARD TESTS] add BookKeeper 4.14.4 ### Motivation BK 4.14.4 has been released and we should test it in the upgrade tests Note: for the sake of test performance, we test the upgrades only for the latest releases of each minor release ### Changes * Replaced BK 4.14.3 with 4.14.4 Reviewers: Enrico Olivelli , Andrey Yegorov This closes #2997 from nicoloboschi/tests/add-bk-4144-backward-compat --- .../bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy | 4 ++-- tests/docker-images/all-released-versions-image/Dockerfile | 2 +- .../tests/integration/utils/BookKeeperClusterUtils.java | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy index d885e778ff8..e7e9a3170a3 100644 --- a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy +++ b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy @@ -143,11 +143,11 @@ class TestCompatUpgrade { @Test public void test_006_4130to4143() throws Exception { - testUpgrade("4.13.0", "4.14.3") + testUpgrade("4.13.0", "4.14.4") } @Test public void test_007_4143toCurrentMaster() throws Exception { - testUpgrade("4.14.3", BookKeeperClusterUtils.CURRENT_VERSION) + testUpgrade("4.14.4", BookKeeperClusterUtils.CURRENT_VERSION) } } diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index 9c0301fa205..af2d9651de8 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -38,7 +38,7 @@ RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.10.0/bookke RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.11.1/bookkeeper-server-4.11.1-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookkeeper-server-4.12.1-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.3/bookkeeper-server-4.14.3-bin.tar.gz{,.sha512,.asc} +RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.4/bookkeeper-server-4.14.4-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/pulsar-1.21.0-incubating/apache-pulsar-1.21.0-incubating-bin.tar.gz{,.asc} RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index 94cdedfab96..a29331131fe 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -43,9 +43,10 @@ public class BookKeeperClusterUtils { public static final String CURRENT_VERSION = System.getProperty("currentVersion"); public static final List OLD_CLIENT_VERSIONS = - Arrays.asList("4.8.2", "4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3"); + Arrays.asList("4.8.2", "4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.4"); private static final List OLD_CLIENT_VERSIONS_WITH_CURRENT_LEDGER_METADATA_FORMAT = - Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3"); + Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.4"); + private static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterUtils.class); From d9c40219ad4a02c0703f3f5907ffc7a039dcc42a Mon Sep 17 00:00:00 2001 From: mauricebarnum Date: Fri, 4 Feb 2022 16:28:41 -0800 Subject: [PATCH 14/79] Gradle 6.9.2 (#3022) * Remove annoying println * Update gradle to 6.9.2 This includes `Mitigations for log4j vulnerability in Gradle builds` https://github.com/gradle/gradle/issues/19328 Full release notes https://docs.gradle.org/6.9.2/release-notes.html --- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 259 ++++++++++++++--------- microbenchmarks/build.gradle | 3 +- 3 files changed, 156 insertions(+), 108 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 047e8487800..cb38b87d24e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -21,6 +21,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 86c97c2b584..1b6c787337f 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,101 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/microbenchmarks/build.gradle b/microbenchmarks/build.gradle index 526c170753e..a58eea03b75 100644 --- a/microbenchmarks/build.gradle +++ b/microbenchmarks/build.gradle @@ -41,7 +41,6 @@ jar { manifest { attributes 'Main-Class': 'org.openjdk.jmh.Main' } - println configurations.runtimeClasspath.collect().size() from { configurations.compileClasspath.collect {it.isDirectory() ? it : zipTree(it) } } @@ -49,4 +48,4 @@ jar { exclude 'META-INF/*.SF' exclude 'META-INF/*.DSA' archiveBaseName = 'microbenchmarks' -} \ No newline at end of file +} From dcf12813de7438fb5d5e96f7c99b27db8949dff5 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Sun, 6 Feb 2022 01:36:50 +0800 Subject: [PATCH 15/79] Bump gradle to 7.3.3, compat java 17 (#2924) --- build.gradle | 40 +++++++++++++++++-- gradle/wrapper/gradle-wrapper.properties | 2 +- .../backward-compat/bc-non-fips/build.gradle | 9 ++++- tests/integration/cluster/build.gradle | 1 + tests/integration/smoke/build.gradle | 1 + 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index c53719dfaec..6cba809a34f 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,18 @@ releaseArtifacts { fromProject("bookkeeper-dist-all") fromProject("bookkeeper-dist-bkctl") fromProject("bookkeeper-dist-server") + + tasks.withType(Jar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Tar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Zip) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } } releaseParams { // ReleaseExtension @@ -78,7 +90,7 @@ allprojects { apply plugin: 'org.nosphere.apache.rat' apply plugin: "checkstyle" apply plugin: 'com.github.spotbugs' - + if (!it.path.startsWith(':tests')) { apply plugin: 'org.owasp.dependencycheck' @@ -136,12 +148,24 @@ allprojects { + tasks.withType(Jar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Tar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Zip) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + task testJar(type: Jar, dependsOn: testClasses) { - classifier = 'tests' + archiveClassifier = 'tests' from sourceSets.test.output } - // TODO- Seperate rat plugins and other plugins configuration in seperate files + // TODO- Separate rat plugins and other plugins configuration in separate files rat { excludes = [ '**/.*/**', @@ -186,7 +210,9 @@ allprojects { } configurations { - testArtifacts.extendsFrom testRuntime + testArtifacts { + extendsFrom testRuntimeOnly + } } artifacts { @@ -195,6 +221,12 @@ allprojects { plugins.withType(DistributionPlugin) { distTar { + dependsOn(jar) + if (it.path.contains('bookkeeper-dist-all')) { + dependsOn(':bookkeeper-benchmark:jar') + dependsOn(':bookkeeper-common:jar') + dependsOn(':bookkeeper-common-allocator:jar') + } archiveClassifier = "bin" compression = Compression.GZIP archiveExtension = 'tar.gz' diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index cb38b87d24e..f91c4502a57 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -21,6 +21,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/tests/backward-compat/bc-non-fips/build.gradle b/tests/backward-compat/bc-non-fips/build.gradle index b87df1bc510..e321659662f 100644 --- a/tests/backward-compat/bc-non-fips/build.gradle +++ b/tests/backward-compat/bc-non-fips/build.gradle @@ -23,9 +23,14 @@ plugins { } dependencies { - + testImplementation project(':bookkeeper-common') + testImplementation(project(':bookkeeper-server')) { + exclude group: "org.bouncycastle", module: "bc-fips" + } testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') - testImplementation project(path: ':bookkeeper-server', configuration: 'testArtifacts') + testImplementation(project(path: ':bookkeeper-server', configuration: 'testArtifacts')) { + exclude group: "org.bouncycastle", module: "bc-fips" + } testImplementation depLibs.junit testImplementation depLibs.slf4j testImplementation depLibs.bcpkixJdk15on diff --git a/tests/integration/cluster/build.gradle b/tests/integration/cluster/build.gradle index 4cf0dac30d0..71a01ac3881 100644 --- a/tests/integration/cluster/build.gradle +++ b/tests/integration/cluster/build.gradle @@ -23,6 +23,7 @@ plugins { dependencies { implementation depLibs.grpc + testImplementation project(':bookkeeper-common') testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') testImplementation project(':stream:api') testImplementation project(':stream:storage:api') diff --git a/tests/integration/smoke/build.gradle b/tests/integration/smoke/build.gradle index 02ad13b25fb..b154f34a841 100644 --- a/tests/integration/smoke/build.gradle +++ b/tests/integration/smoke/build.gradle @@ -22,6 +22,7 @@ plugins { } dependencies { + testImplementation project(':bookkeeper-common') testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') testImplementation project(':bookkeeper-server') testImplementation project(':tests:integration-tests-utils') From 1eae5e473b7775419760747b5d1f4dadd0c09052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 8 Feb 2022 14:17:10 +0100 Subject: [PATCH 16/79] [tests] remove backward compatibility test against yahoo-version (apache Pulsar 1.21) (#3028) --- .github/workflows/backward-compat-tests.yml | 10 +- .github/workflows/bookie-tests.yml | 5 +- .github/workflows/client-tests.yml | 5 +- .../workflows/compatibility-check-java11.yml | 5 +- .../workflows/compatibility-check-java8.yml | 5 +- .github/workflows/integration-tests.yml | 10 +- .github/workflows/pr-validation.yml | 6 +- .github/workflows/remaining-tests.yml | 6 +- .github/workflows/replication-tests.yml | 4 +- .github/workflows/stream-tests.yml | 19 +- .github/workflows/tls-tests.yml | 5 +- build.gradle | 2 +- gradle.properties | 2 +- settings.gradle | 1 - .../TestCompatUpgradeYahooCustom.groovy | 220 ------------------ .../src/test/resources/arquillian.xml | 28 --- .../all-released-versions-image/Dockerfile | 5 +- .../scripts/install-pulsar-tarball.sh | 41 ---- 18 files changed, 56 insertions(+), 323 deletions(-) delete mode 100644 tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy delete mode 100644 tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml delete mode 100644 tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh diff --git a/.github/workflows/backward-compat-tests.yml b/.github/workflows/backward-compat-tests.yml index 03f4e66e2da..8efa06f68cc 100644 --- a/.github/workflows/backward-compat-tests.yml +++ b/.github/workflows/backward-compat-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,10 +49,10 @@ jobs: with: java-version: 1.8 - name: Build - run: ./gradlew stream:server:build -x test + run: ./gradlew stream:server:build -x test ${GRADLE_ARGS} - name: Test current server with old clients - run: ./gradlew :tests:backward-compat:current-server-old-clients:test || (tail -n +1 tests/backward-compat/current-server-old-clients/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/current-server-old-clients/build/container-logs/**/* && exit 1) + run: ./gradlew :tests:backward-compat:current-server-old-clients:test ${GRADLE_ARGS} || (tail -n +1 tests/backward-compat/current-server-old-clients/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/current-server-old-clients/build/container-logs/**/* && exit 1) - name: Test progressive upgrade - run: ./gradlew :tests:backward-compat:upgrade:test || (tail -n +1 tests/backward-compat/upgrade/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/upgrade/build/container-logs/**/* && exit 1) + run: ./gradlew :tests:backward-compat:upgrade:test ${GRADLE_ARGS} || (tail -n +1 tests/backward-compat/upgrade/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/upgrade/build/container-logs/**/* && exit 1) - name: Other tests - run: ./gradlew :tests:backward-compat:test -x tests:backward-compat:upgrade:test -x :tests:backward-compat:current-server-old-clients:test + run: ./gradlew :tests:backward-compat:test -x tests:backward-compat:upgrade:test -x :tests:backward-compat:current-server-old-clients:test ${GRADLE_ARGS} diff --git a/.github/workflows/bookie-tests.yml b/.github/workflows/bookie-tests.yml index 34a70bfa85b..8eec299c5a0 100644 --- a/.github/workflows/bookie-tests.yml +++ b/.github/workflows/bookie-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -47,4 +50,4 @@ jobs: java-version: 1.8 - name: Run bookie test - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" -Dtestlogger.theme=plain \ No newline at end of file + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" ${GRADLE_ARGS} \ No newline at end of file diff --git a/.github/workflows/client-tests.yml b/.github/workflows/client-tests.yml index f8c2ada72a7..398278fc937 100644 --- a/.github/workflows/client-tests.yml +++ b/.github/workflows/client-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -46,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run client tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" ${GRADLE_ARGS} diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index e83ca17cbf5..7d93c50b90b 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: @@ -47,4 +50,4 @@ jobs: java-version: 1.11 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS} diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml index 82094fddf95..629fed6dac0 100644 --- a/.github/workflows/compatibility-check-java8.yml +++ b/.github/workflows/compatibility-check-java8.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: @@ -47,4 +50,4 @@ jobs: java-version: 1.8 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 229baeb9762..59c5128c5bc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -50,10 +50,10 @@ jobs: java-version: 1.8 - name: Build tar - run: ./gradlew stream:server:build -x test + run: ./gradlew stream:server:build -x test ${GRADLE_ARGS} - name: run cluster integration test - run: ./gradlew :tests:integration:cluster:test -Dtestlogger.theme=plain + run: ./gradlew :tests:integration:cluster:test ${GRADLE_ARGS} - name: run smoke test - run: ./gradlew tests:integration:smoke:test -Dtestlogger.theme=plain + run: ./gradlew tests:integration:smoke:test ${GRADLE_ARGS} - name: run standalone test - run: ./gradlew tests:integration:standalone:test -Dtestlogger.theme=plain + run: ./gradlew tests:integration:standalone:test ${GRADLE_ARGS} diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 816315b3707..0f4cba2c6b0 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: @@ -47,6 +50,7 @@ jobs: with: java-version: 1.8 - name: Validate pull request - run: ./gradlew build -x signDistTar -x test + run: ./gradlew build -x signDistTar -x test ${GRADLE_ARGS} + - name: Check license files run: dev/check-all-licenses diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml index 835d39ae3d9..a3f53f44297 100644 --- a/.github/workflows/remaining-tests.yml +++ b/.github/workflows/remaining-tests.yml @@ -29,6 +29,10 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + + jobs: test: @@ -46,4 +50,4 @@ jobs: with: java-version: 1.8 - name: Run remaining tests - run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain \ No newline at end of file + run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" ${GRADLE_ARGS} \ No newline at end of file diff --git a/.github/workflows/replication-tests.yml b/.github/workflows/replication-tests.yml index b1c0aeaeb7f..9664d8b822c 100644 --- a/.github/workflows/replication-tests.yml +++ b/.github/workflows/replication-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run replication tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" ${GRADLE_ARGS} diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml index 1cdb6461766..612b47bf1a8 100644 --- a/.github/workflows/stream-tests.yml +++ b/.github/workflows/stream-tests.yml @@ -28,6 +28,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -45,19 +48,19 @@ jobs: with: java-version: 1.8 - name: Run stream:distributedlog:core tests - run: ./gradlew stream:distributedlog:core:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:core:test ${GRADLE_ARGS} - name: Run stream:distributedlog:common tests - run: ./gradlew stream:distributedlog:common:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:common:test ${GRADLE_ARGS} - name: Run stream:distributedlog:protocol tests - run: ./gradlew stream:distributedlog:protocol:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:protocol:test ${GRADLE_ARGS} - name: Run stream:proto tests - run: ./gradlew stream:proto:test -Dtestlogger.theme=plain + run: ./gradlew stream:proto:test ${GRADLE_ARGS} - name: Run stream:serve tests - run: ./gradlew stream:server:test -Dtestlogger.theme=plain + run: ./gradlew stream:server:test ${GRADLE_ARGS} - name: Run stream:statelib tests - run: ./gradlew stream:statelib:test -Dtestlogger.theme=plain + run: ./gradlew stream:statelib:test ${GRADLE_ARGS} - name: Run stream:storage:api:test tests - run: ./gradlew stream:storage:api:test -Dtestlogger.theme=plain + run: ./gradlew stream:storage:api:test ${GRADLE_ARGS} - name: Run stream:storage:impl tests - run: ./gradlew stream:storage:impl:test -Dtestlogger.theme=plain + run: ./gradlew stream:storage:impl:test ${GRADLE_ARGS} diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml index 3aeab4ac269..48f7f74d4d2 100644 --- a/.github/workflows/tls-tests.yml +++ b/.github/workflows/tls-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -46,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run tls tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" ${GRADLE_ARGS} diff --git a/build.gradle b/build.gradle index 6cba809a34f..ad5a9e11715 100644 --- a/build.gradle +++ b/build.gradle @@ -310,7 +310,7 @@ allprojects { systemProperty "currentVersion", project.rootProject.version testLogging { outputs.upToDateWhen {false} - showStandardStreams = true + showStandardStreams = !Boolean.getBoolean("testHideStandardOut") } } dependencies { diff --git a/gradle.properties b/gradle.properties index bba0e1dfa0a..58ef9bbd17f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -25,6 +25,6 @@ shadowPluginVersion=6.1.0 licenseGradlePluginVersion=0.15.0 checkStyleVersion=6.19 spotbugsPlugin=4.7.0 -testLogger=2.0.0 +testLogger=3.1.0 testRetry=1.0.0 owaspPlugin=6.5.3 diff --git a/settings.gradle b/settings.gradle index 036b10413e0..0e2455a9607 100644 --- a/settings.gradle +++ b/settings.gradle @@ -91,7 +91,6 @@ include(':bookkeeper-benchmark', 'tests:backward-compat:recovery-no-password', 'tests:backward-compat:upgrade', 'tests:backward-compat:upgrade-direct', - 'tests:backward-compat:yahoo-custom-version', 'tests:integration-tests-base', 'tests:integration-tests-topologies', 'tests:integration-tests-utils', diff --git a/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy b/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy deleted file mode 100644 index f1d024a61dc..00000000000 --- a/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy +++ /dev/null @@ -1,220 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.bookkeeper.tests.backwardcompat - -import static java.nio.charset.StandardCharsets.UTF_8 - -import com.github.dockerjava.api.DockerClient - -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutionException -import java.util.concurrent.TimeUnit -import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils -import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader - -import org.jboss.arquillian.junit.Arquillian -import org.jboss.arquillian.test.api.ArquillianResource - -import org.junit.Assert -import org.junit.Test -import org.junit.runner.RunWith - -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -@RunWith(Arquillian.class) -class TestCompatUpgradeYahooCustom { - private static final Logger LOG = LoggerFactory.getLogger(TestCompatUpgradeYahooCustom.class) - private static byte[] PASSWD = "foobar".getBytes() - - def yahooRepo = "https://raw.githubusercontent.com/yahoo/bookkeeper/mvn-repo" - def yahooArtifact = "org.apache.bookkeeper:bookkeeper-server:4.3.1.85-yahoo" - - @ArquillianResource - DockerClient docker - - int numEntries = 1 - - def yahooConfiguredBookKeeper(def classLoader, def zookeeper) { - // bookkeeper client configured in same way as pulsar configures it - def bkConf = classLoader.newInstance("org.apache.bookkeeper.conf.ClientConfiguration") - - bkConf.setThrottleValue(0) - bkConf.setAddEntryTimeout(30) - bkConf.setReadEntryTimeout(30) - bkConf.setSpeculativeReadTimeout(0) - bkConf.setNumChannelsPerBookie(16) - bkConf.setUseV2WireProtocol(true) - // we can't specify the class name, because it would try to use the thread - // context classloader to load, which doesn't have the required classes loaded. - bkConf.setLedgerManagerType("hierarchical") - - bkConf.enableBookieHealthCheck() - bkConf.setBookieHealthCheckInterval(60, TimeUnit.SECONDS) - bkConf.setBookieErrorThresholdPerInterval(5) - bkConf.setBookieQuarantineTime(1800, TimeUnit.SECONDS) - - bkConf.setZkServers(zookeeper) - return classLoader.newInstance("org.apache.bookkeeper.client.BookKeeper", bkConf) - } - - def addEntry(def classLoader, def ledger, def entryData) { - def promise = new CompletableFuture() - def buffer = classLoader.callStaticMethod("io.netty.buffer.Unpooled", "copiedBuffer", - [entryData.getBytes(UTF_8)]) - def callback = classLoader.createCallback( - "org.apache.bookkeeper.client.AsyncCallback\$AddCallback", - { rc, _ledger, entryId, ctx -> - if (rc != 0) { - promise.completeExceptionally( - classLoader.callStaticMethod("org.apache.bookkeeper.client.BKException", - "create", [rc])) - } else { - promise.complete(entryId) - } - }) - ledger.asyncAddEntry(buffer, callback, null) - return promise - } - - def assertCantWrite(def cl, def ledger) throws Exception { - try { - addEntry(cl, ledger, "Shouldn't write").get() - fail("Shouldn't be able to write to ledger") - } catch (ExecutionException e) { - // correct behaviour - // TODO add more - } - } - def createAndWrite(def cl, def bk) throws Exception { - def ledger = bk.createLedger(3, 2, cl.digestType("CRC32"), PASSWD) - LOG.info("Created ledger ${ledger.getId()}") - for (int i = 0; i < numEntries; i++) { - addEntry(cl, ledger, "foobar" + i).get() - } - return ledger - } - - def openAndVerifyEntries(def cl, def bk, def ledgerId) throws Exception { - LOG.info("Opening ledger $ledgerId") - def ledger = bk.openLedger(ledgerId, cl.digestType("CRC32"), PASSWD) - def entries = ledger.readEntries(0, ledger.getLastAddConfirmed()) - int j = 0 - while (entries.hasMoreElements()) { - def e = entries.nextElement() - Assert.assertEquals(new String(e.getEntry()), "foobar"+ j) - j++ - } - ledger.close() - } - - def List exerciseClients(List toVerify) { - String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker) - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - - def ledgers = [] - def yahooCL = MavenClassLoader.forArtifact(yahooRepo, yahooArtifact) - def yahooBK = yahooConfiguredBookKeeper(yahooCL, zookeeper) - def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion) - def currentBK = yahooConfiguredBookKeeper(currentCL, zookeeper) - try { - // verify we can read ledgers from previous run - for (Long id : toVerify) { - LOG.info("Verifying $id with yahoo client") - openAndVerifyEntries(yahooCL, yahooBK, id) - LOG.info("Verifying $id with current client") - openAndVerifyEntries(currentCL, currentBK, id) - } - // yahoo client and create open and read - def ledger0 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger0.getId()) - ledger0.close() - openAndVerifyEntries(yahooCL, yahooBK, ledger0.getId()) - - // yahoo client can fence on yahoo bookies - def ledger1 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger1.getId()) - openAndVerifyEntries(yahooCL, yahooBK, ledger1.getId()) - assertCantWrite(yahooCL, ledger1) - - // current client can create open and read - def ledger2 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger2.getId()) - ledger2.close() - openAndVerifyEntries(currentCL, currentBK, ledger2.getId()) - - // current client can fence on yahoo bookies - def ledger3 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger3.getId()) - openAndVerifyEntries(currentCL, currentBK, ledger3.getId()) - assertCantWrite(currentCL, ledger3) - - // current client can fence a bookie created by yahoo client - def ledger4 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger4.getId()) - openAndVerifyEntries(currentCL, currentBK, ledger4.getId()) - assertCantWrite(yahooCL, ledger4) - - // Since METADATA_VERSION is upgraded and it is using binary format, the older - // clients which are expecting text format would fail to read ledger metadata. - def ledger5 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger5.getId()) - try { - openAndVerifyEntries(yahooCL, yahooBK, ledger5.getId()) - } catch (Exception exc) { - Assert.assertEquals(exc.getClass().getName(), - "org.apache.bookkeeper.client.BKException\$ZKException") - } - } finally { - currentBK.close() - currentCL.close() - yahooBK.close() - yahooCL.close() - } - return ledgers - } - - @Test - public void testUpgradeYahooCustom() throws Exception { - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - String yahooVersion = "4.3-yahoo" - BookKeeperClusterUtils.metadataFormatIfNeeded(docker, yahooVersion) - - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, yahooVersion)) - def preUpgradeLedgers = exerciseClients([]) - Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)) - BookKeeperClusterUtils.runOnAllBookies( - docker, "cp", "/opt/bookkeeper/${yahooVersion}/conf/bookkeeper.conf", - "/opt/bookkeeper/${currentVersion}/conf/bk_server.conf") - BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, - "logSizeLimit", "1073741824") - BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, - "statsProviderClass", - "org.apache.bookkeeper.stats.NullStatsProvider") - - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion)) - // Since METADATA_VERSION is upgraded and it is using binary format, the older - // clients which are expecting text format would fail to read ledger metadata. - try { - exerciseClients(preUpgradeLedgers) - } catch (Exception exc) { - Assert.assertEquals(exc.getClass().getName(), - "org.apache.bookkeeper.client.BKException\$ZKException") - } - } -} diff --git a/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml b/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml deleted file mode 100644 index f914ff2c258..00000000000 --- a/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - CUBE - cube-definitions/3-node-all-version-unstarted.yaml - - - diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index af2d9651de8..aee3227ecd1 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -40,17 +40,14 @@ RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookke RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.4/bookkeeper-server-4.14.4-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/pulsar-1.21.0-incubating/apache-pulsar-1.21.0-incubating-bin.tar.gz{,.asc} RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS RUN wget -nv http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552 -O KEYS.old -RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/KEYS -O KEYS.pulsar RUN mkdir -p /etc/supervisord/conf.d && mkdir -p /var/run/supervisor && mkdir -p /var/log/bookkeeper ADD conf/supervisord.conf /etc/supervisord.conf ADD scripts/install-all-tarballs.sh /install-all-tarballs.sh ADD scripts/install-tarball.sh /install-tarball.sh -ADD scripts/install-pulsar-tarball.sh /install-pulsar-tarball.sh -RUN bash /install-all-tarballs.sh && bash /install-pulsar-tarball.sh && rm -rf /tarballs +RUN bash /install-all-tarballs.sh && rm -rf /tarballs WORKDIR / ADD scripts/update-conf-and-boot.sh /update-conf-and-boot.sh diff --git a/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh b/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh deleted file mode 100644 index 68fdd0437aa..00000000000 --- a/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -set -e - -gpg --import KEYS.pulsar - -TARBALL=$(ls apache-pulsar-*-incubating-bin.tar.gz | tail -1) -gpg --verify $TARBALL.asc - -tar -zxf $TARBALL -rm $TARBALL -VERSION=4.3-yahoo - -mv apache-pulsar-*-incubating /opt/bookkeeper/$VERSION - -cat > /etc/supervisord/conf.d/bookkeeper-$VERSION.conf < Date: Wed, 9 Feb 2022 09:54:54 +0800 Subject: [PATCH 17/79] Support specifying bookie http port as a command argument (#2769) ### Motivation I was trying to start multiple bookies locally and found it's a bit inconvenient to specify different http ports for different bookies. ### Changes Add a command-line argument `httpport` to the bookie command to support specifying bookie http port from the command line. --- .../src/main/java/org/apache/bookkeeper/server/Main.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java index 3c7d7fe7cfd..ddc8613acd2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java @@ -116,6 +116,7 @@ public class Main { BK_OPTS.addOption("z", "zkserver", true, "Zookeeper Server"); BK_OPTS.addOption("m", "zkledgerpath", true, "Zookeeper ledgers root path"); BK_OPTS.addOption("p", "bookieport", true, "bookie port exported"); + BK_OPTS.addOption("hp", "httpport", true, "bookie http port exported"); BK_OPTS.addOption("j", "journal", true, "bookie journal directory"); Option indexDirs = new Option ("i", "indexdirs", true, "bookie index directories"); indexDirs.setArgs(10); @@ -212,6 +213,13 @@ private static ServerConfiguration parseArgs(String[] args) conf.setBookiePort(Integer.parseInt(sPort)); } + if (cmdLine.hasOption("httpport")) { + String sPort = cmdLine.getOptionValue("httpport"); + log.info("Get cmdline http port: {}", sPort); + Integer iPort = Integer.parseInt(sPort); + conf.setHttpServerPort(iPort.intValue()); + } + if (cmdLine.hasOption('j')) { String sJournalDir = cmdLine.getOptionValue('j'); log.info("Get cmdline journal dir: {}", sJournalDir); From 5405ee98e1d7e7c155fad55f634c20ed424d1e2e Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 10 Feb 2022 03:24:21 +0800 Subject: [PATCH 18/79] Fix performance issue to avoid unnessary loop. (#3030) --- .../main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java index c047977c383..be55ca18f3f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java @@ -95,6 +95,7 @@ public void onProtocolUpgrade() { for (String allowedRole : allowedRoles) { if (certRole[0].equals(allowedRole)) { authorized = true; + break; } } if (authorized) { From 7d35b67a8d2306a7d92cbc51cacc36cbf29f8899 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 9 Feb 2022 13:13:44 -0800 Subject: [PATCH 19/79] Upgrade RocksDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Descriptions of the changes in this PR: Dependency change ### Motivation I encountered https://github.com/apache/bookkeeper/issues/3024 and noticed that newer version of RocksDB includes multiple fixes for concurrency issues with various side-effects and fixes for a few crashes. I upgraded, ran `org.apache.bookkeeper.bookie.BookieJournalTest` test in a loop and didn't repro the crash so far. It is hard to say 100% if it is fixed given it was not happening all the time. ### Changes Upgraded RocksDB Master Issue: #3024 Reviewers: Enrico Olivelli , Nicolò Boschi This closes #3026 from dlg99/rocksdb-upgrade --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 4 ++-- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 4 ++-- dependencies.gradle | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index adfb859d517..03ee387e733 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -260,7 +260,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.43.v20210629.jar [22] -- lib/org.rocksdb-rocksdbjni-6.22.1.1.jar [23] +- lib/org.rocksdb-rocksdbjni-6.27.3.jar [23] - lib/com.beust-jcommander-1.78.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -586,7 +586,7 @@ This private header is also used by Apple's open source * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-6.22.1.1.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-6.27.3.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index a00cd92b9a9..df8b99e57c6 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -260,7 +260,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.43.v20210629.jar [22] -- lib/org.rocksdb-rocksdbjni-6.22.1.1.jar [23] +- lib/org.rocksdb-rocksdbjni-6.27.3.jar [23] - lib/com.beust-jcommander-1.78.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -584,7 +584,7 @@ This private header is also used by Apple's open source * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-6.22.1.1.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-6.27.3.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/dependencies.gradle b/dependencies.gradle index b822b63af62..e1c297052ea 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -72,7 +72,7 @@ depVersions = [ prometheus: "0.8.1", protobuf: "3.16.1", reflections: "0.9.11", - rocksDb: "6.22.1.1", + rocksDb: "6.27.3", rxjava: "3.0.1", slf4j: "1.7.32", snakeyaml: "1.30", From 098ceaca63e934d4ef9b56e46af7f0c24b5d6a5b Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Fri, 11 Feb 2022 11:10:28 +0800 Subject: [PATCH 20/79] Skip update entryLogMetaMap if not modified (#2964) ### Motivation After we support RocksDB backend entryMetaMap, we should avoid updating the entryMetaMap if unnecessary. In `doGcEntryLogs` method, it iterate through the entryLogMetaMap and update the meta if ledgerNotExists. We should check whether the meta has been modified in `removeIfLedgerNotExists`. If not modified, we can avoid update the entryLogMetaMap. ### Modification 1. Add a flag to represent whether the meta has been modified in `removeIfLedgerNotExists` method. If not, skip update the entryLogMetaMap. --- .../bookie/GarbageCollectorThread.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index bf00566f1b3..8e46a630d4c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -45,6 +45,7 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.util.MathUtils; import org.apache.bookkeeper.util.SafeRunnable; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -438,9 +439,7 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { // Loop through all of the entry logs and remove the non-active ledgers. entryLogMetaMap.forEach((entryLogId, meta) -> { try { - removeIfLedgerNotExists(meta); - // update entryMetadta to persistent-map - entryLogMetaMap.put(meta.getEntryLogId(), meta); + boolean modified = removeIfLedgerNotExists(meta); if (meta.isEmpty()) { // This means the entry log is not associated with any active // ledgers anymore. @@ -448,6 +447,9 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { LOG.info("Deleting entryLogId {} as it has no active ledgers!", entryLogId); removeEntryLog(entryLogId); gcStats.getReclaimedSpaceViaDeletes().add(meta.getTotalSize()); + } else if (modified) { + // update entryLogMetaMap only when the meta modified. + entryLogMetaMap.put(meta.getEntryLogId(), meta); } } catch (EntryLogMetadataMapException e) { // Ignore and continue because ledger will not be cleaned up @@ -462,16 +464,23 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { this.numActiveEntryLogs = entryLogMetaMap.size(); } - private void removeIfLedgerNotExists(EntryLogMetadata meta) throws EntryLogMetadataMapException { + private boolean removeIfLedgerNotExists(EntryLogMetadata meta) throws EntryLogMetadataMapException { + MutableBoolean modified = new MutableBoolean(false); meta.removeLedgerIf((entryLogLedger) -> { // Remove the entry log ledger from the set if it isn't active. try { - return !ledgerStorage.ledgerExists(entryLogLedger); + boolean exist = ledgerStorage.ledgerExists(entryLogLedger); + if (!exist) { + modified.setTrue(); + } + return !exist; } catch (IOException e) { LOG.error("Error reading from ledger storage", e); return false; } }); + + return modified.getValue(); } /** From eabf7b1a88120ef3263ffdb729f0c6e4171d6fbe Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Fri, 11 Feb 2022 11:11:03 +0800 Subject: [PATCH 21/79] Add rack name invalid check (#2980) ### Motivation When we set region or rack placement policy, but the region or rack name set to `/` or empty string, it will throw the following exception on handling bookies join. ``` java.lang.StringIndexOutOfBoundsException: String index out of range: -1 at java.lang.String.substring(String.java:1841) ~[?:?] at org.apache.bookkeeper.net.NetworkTopologyImpl$InnerNode.getNextAncestorName(NetworkTopologyImpl.java:144) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.net.NetworkTopologyImpl$InnerNode.add(NetworkTopologyImpl.java:180) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.net.NetworkTopologyImpl.add(NetworkTopologyImpl.java:425) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy.handleBookiesThatJoined(TopologyAwareEnsemblePlacementPolicy.java:717) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.handleBookiesThatJoined(RackawareEnsemblePlacementPolicyImpl.java:80) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy.handleBookiesThatJoined(RackawareEnsemblePlacementPolicy.java:249) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy.onClusterChanged(TopologyAwareEnsemblePlacementPolicy.java:663) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.onClusterChanged(RackawareEnsemblePlacementPolicyImpl.java:80) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy.onClusterChanged(RackawareEnsemblePlacementPolicy.java:92) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.BookieWatcherImpl.processWritableBookiesChanged(BookieWatcherImpl.java:197) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.BookieWatcherImpl.lambda$initialBlockingBookieRead$1(BookieWatcherImpl.java:233) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.discover.ZKRegistrationClient$WatchTask.accept(ZKRegistrationClient.java:147) [io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.discover.ZKRegistrationClient$WatchTask.accept(ZKRegistrationClient.java:70) [io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) [?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) [?:?] at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:478) [?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.72.Final.jar:4.1.72.Final] at java.lang.Thread.run(Thread.java:829) [?:?] ``` The root cause is that the node networkLocation is empty string and then use `substring(1)` operation, which will lead to `StringIndexOutOfBoundsException` ### Modification 1. Add `n.getNetworkLocation()` is empty check on `isAncestor` method to make the exception more clear. --- .../bookkeeper/net/NetworkTopologyImpl.java | 6 ++-- .../net/NetworkTopologyImplTest.java | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java index 123f13ca8a3..19b2c0497f7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java @@ -17,6 +17,7 @@ */ package org.apache.bookkeeper.net; +import com.google.common.base.Strings; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -119,9 +120,10 @@ boolean isRack() { * @return true if this node is an ancestor of n */ boolean isAncestor(Node n) { - return getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) + return !Strings.isNullOrEmpty(n.getNetworkLocation()) + && (getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) || (n.getNetworkLocation() + NodeBase.PATH_SEPARATOR_STR).startsWith(getPath(this) - + NodeBase.PATH_SEPARATOR_STR); + + NodeBase.PATH_SEPARATOR_STR)); } /** diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java index a6cc8c330a3..6122e51f0cd 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java @@ -17,7 +17,9 @@ */ package org.apache.bookkeeper.net; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.Set; import org.junit.Test; @@ -97,4 +99,31 @@ public void getLeavesShouldReturnLeavesThatAreNotInExcludedScope() { assertTrue(leavesExcludingRack2Scope.contains(bookieRack0ScopeNode)); assertTrue(leavesExcludingRack2Scope.contains(bookieRack2ScopeNode)); } + + @Test + public void testInvalidRackName() { + NetworkTopologyImpl networkTopology = new NetworkTopologyImpl(); + String rack0Scope = ""; + BookieId bookieIdScopeRack0 = BookieId.parse("bookieIdScopeRack0"); + BookieNode bookieRack0ScopeNode = new BookieNode(bookieIdScopeRack0, rack0Scope); + + String rack1Scope = "/"; + BookieId bookieIdScopeRack1 = BookieId.parse("bookieIdScopeRack1"); + BookieNode bookieRack1ScopeNode = new BookieNode(bookieIdScopeRack1, rack1Scope); + + try { + networkTopology.add(bookieRack0ScopeNode); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("bookieIdScopeRack0, which is located at , is not a decendent of /", e.getMessage()); + } + + try { + networkTopology.add(bookieRack1ScopeNode); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("bookieIdScopeRack1, which is located at , is not a decendent of /", e.getMessage()); + } + + } } \ No newline at end of file From 48a87a88f7d98fb890ed2aef58487f19f32a8296 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Fri, 11 Feb 2022 11:12:27 +0800 Subject: [PATCH 22/79] Support multi ledger directories for rocksdb backend entryMetadataMap (#2965) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation When we use RocksDB backend entryMetadataMap for multi ledger directories configured, the bookie start up failed, and throw the following exception. ``` 12:24:28.530 [main] ERROR org.apache.pulsar.PulsarStandaloneStarter - Failed to start pulsar service. java.io.IOException: Error open RocksDB database at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:202) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:89) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.lambda$static$0(KeyValueStorageRocksDB.java:62) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.PersistentEntryLogMetadataMap.(PersistentEntryLogMetadataMap.java:87) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.GarbageCollectorThread.createEntryLogMetadataMap(GarbageCollectorThread.java:265) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.GarbageCollectorThread.(GarbageCollectorThread.java:154) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.GarbageCollectorThread.(GarbageCollectorThread.java:133) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage.(SingleDirectoryDbLedgerStorage.java:182) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.newSingleDirectoryDbLedgerStorage(DbLedgerStorage.java:190) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.initialize(DbLedgerStorage.java:150) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.BookieResources.createLedgerStorage(BookieResources.java:110) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.buildBookie(LocalBookkeeperEnsemble.java:328) ~[org.apache.pulsar-pulsar-zookeeper-utils-2.8.1.jar:2.8.1] at org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.runBookies(LocalBookkeeperEnsemble.java:391) ~[org.apache.pulsar-pulsar-zookeeper-utils-2.8.1.jar:2.8.1] at org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.startStandalone(LocalBookkeeperEnsemble.java:521) ~[org.apache.pulsar-pulsar-zookeeper-utils-2.8.1.jar:2.8.1] at org.apache.pulsar.PulsarStandalone.start(PulsarStandalone.java:264) ~[org.apache.pulsar-pulsar-broker-2.8.1.jar:2.8.1] at org.apache.pulsar.PulsarStandaloneStarter.main(PulsarStandaloneStarter.java:121) [org.apache.pulsar-pulsar-broker-2.8.1.jar:2.8.1] Caused by: org.rocksdb.RocksDBException: lock hold by current process, acquire time 1640492668 acquiring thread 123145515651072: data/standalone/bookkeeper00/entrylogIndexCache/metadata-cache/LOCK: No locks available at org.rocksdb.RocksDB.open(Native Method) ~[org.rocksdb-rocksdbjni-6.10.2.jar:?] at org.rocksdb.RocksDB.open(RocksDB.java:239) ~[org.rocksdb-rocksdbjni-6.10.2.jar:?] at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:199) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] ... 15 more ``` The reason is multi garbageCollectionThread will open the same RocksDB and own the LOCK, and then throw the above exception. ### Modification 1. Change the default GcEntryLogMetadataCachePath from `getLedgerDirNames()[0] + "/" + ENTRYLOG_INDEX_CACHE` to `null`. If it is `null`, it will use each ledger's directory. 2. Remove the internal directory `entrylogIndexCache`. The data structure looks like: ```    └── current    ├── lastMark    ├── ledgers    │   ├── 000003.log    │   ├── CURRENT    │   ├── IDENTITY    │   ├── LOCK    │   ├── LOG    │   ├── MANIFEST-000001    │   └── OPTIONS-000005    ├── locations    │   ├── 000003.log    │   ├── CURRENT    │   ├── IDENTITY    │   ├── LOCK    │   ├── LOG    │   ├── MANIFEST-000001    │   └── OPTIONS-000005    └── metadata-cache    ├── 000003.log    ├── CURRENT    ├── IDENTITY    ├── LOCK    ├── LOG    ├── MANIFEST-000001    └── OPTIONS-000005 ``` 3. If user configured `GcEntryLogMetadataCachePath` in `bk_server.conf`, it only support one ledger directory configured for `ledgerDirectories`. Otherwise, the best practice is to keep it default. 4. The PR is better to release with #1949 --- .../apache/bookkeeper/bookie/BookieImpl.java | 11 +++++++---- .../bookie/GarbageCollectorThread.java | 17 +++++++++++++---- .../bookie/InterleavedLedgerStorage.java | 3 ++- .../ldb/PersistentEntryLogMetadataMap.java | 5 +++-- .../ldb/SingleDirectoryDbLedgerStorage.java | 2 +- .../bookkeeper/conf/ServerConfiguration.java | 12 +++++++----- .../bookkeeper/util/BookKeeperConstants.java | 2 +- 7 files changed, 34 insertions(+), 18 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java index f0a002f4583..94b75254fd0 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java @@ -25,6 +25,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_LEDGER_SCOPE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -1192,10 +1193,12 @@ public static boolean format(ServerConfiguration conf, // Clean up metadata directories if they are separate from the // ledger dirs - File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); - if (!cleanDir(metadataDir)) { - LOG.error("Formatting ledger metadata directory {} failed", metadataDir); - return false; + if (!Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath())) { + File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); + if (!cleanDir(metadataDir)) { + LOG.error("Formatting ledger metadata directory {} failed", metadataDir); + return false; + } } LOG.info("Bookie format completed successfully"); return true; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index 8e46a630d4c..123ab14f17d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -21,7 +21,9 @@ package org.apache.bookkeeper.bookie; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; @@ -121,6 +123,7 @@ public class GarbageCollectorThread extends SafeRunnable { final GarbageCleaner garbageCleaner; final ServerConfiguration conf; + final LedgerDirsManager ledgerDirsManager; /** * Create a garbage collector thread. @@ -130,8 +133,10 @@ public class GarbageCollectorThread extends SafeRunnable { * @throws IOException */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, - final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger) throws IOException { - this(conf, ledgerManager, ledgerStorage, statsLogger, + final LedgerDirsManager ledgerDirsManager, + final CompactableLedgerStorage ledgerStorage, + StatsLogger statsLogger) throws IOException { + this(conf, ledgerManager, ledgerDirsManager, ledgerStorage, statsLogger, Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollectorThread"))); } @@ -144,6 +149,7 @@ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerMana */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, + final LedgerDirsManager ledgerDirsManager, final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger, ScheduledExecutorService gcExecutor) @@ -151,6 +157,7 @@ public GarbageCollectorThread(ServerConfiguration conf, this.gcExecutor = gcExecutor; this.conf = conf; + this.ledgerDirsManager = ledgerDirsManager; this.entryLogger = ledgerStorage.getEntryLogger(); this.entryLogMetaMap = createEntryLogMetadataMap(); this.ledgerStorage = ledgerStorage; @@ -261,11 +268,13 @@ public void removeEntryLog(long logToRemove) { private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { if (conf.isGcEntryLogMetadataCacheEnabled()) { - String baseDir = this.conf.getGcEntryLogMetadataCachePath(); + String baseDir = Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath()) + ? this.ledgerDirsManager.getAllLedgerDirs().get(0).getPath() : conf.getGcEntryLogMetadataCachePath(); try { return new PersistentEntryLogMetadataMap(baseDir, conf); } catch (IOException e) { - LOG.error("Failed to initialize persistent-metadata-map , clean up {}", baseDir, e); + LOG.error("Failed to initialize persistent-metadata-map , clean up {}", + baseDir + "/" + METADATA_CACHE, e); throw e; } } else { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java index dcfeed0bf3c..0960c6658ca 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java @@ -193,7 +193,8 @@ public void initializeWithEntryLogger(ServerConfiguration conf, this.entryLogger.addListener(this); ledgerCache = new LedgerCacheImpl(conf, activeLedgers, null == indexDirsManager ? ledgerDirsManager : indexDirsManager, statsLogger); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger.scope("gc")); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, + this, statsLogger.scope("gc")); pageSize = conf.getPageSize(); ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener()); // Expose Stats diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java index 812ab845385..95b365970f9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java @@ -20,6 +20,7 @@ */ package org.apache.bookkeeper.bookie.storage.ldb; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import io.netty.util.concurrent.FastThreadLocal; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -77,14 +78,14 @@ protected DataInputStream initialValue() { }; public PersistentEntryLogMetadataMap(String metadataPath, ServerConfiguration conf) throws IOException { - LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath); + LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath + "/" + METADATA_CACHE); File dir = new File(metadataPath); if (!dir.mkdirs() && !dir.exists()) { String err = "Unable to create directory " + dir; LOG.error(err); throw new IOException(err); } - metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, "metadata-cache", + metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, METADATA_CACHE, DbConfigType.Small, conf); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 4d2e4adafc4..ce8af36b4e0 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -183,7 +183,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES, TimeUnit.MINUTES); entryLogger = new EntryLogger(conf, ledgerDirsManager, null, statsLogger, allocator); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, this, statsLogger); dbLedgerStorageStats = new DbLedgerStorageStats( ledgerDirStatsLogger, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index a587145fcaf..3f099eb8a27 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -17,7 +17,6 @@ */ package org.apache.bookkeeper.conf; -import static org.apache.bookkeeper.util.BookKeeperConstants.ENTRYLOG_INDEX_CACHE; import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; import com.google.common.annotations.Beta; @@ -515,17 +514,20 @@ public ServerConfiguration setGcEntryLogMetadataCacheEnabled( * gcPersistentEntrylogMetadataMapEnabled is true. * * @return entrylog metadata-map persistent store dir path.(default: it - * creates a sub-directory under a first available base ledger - * directory with name "entrylogIndexCache"). + * creates a sub-directory under each ledger + * directory with name "metadata-cache". If it set, it only works for one ledger directory + * configured for ledgerDirectories). */ public String getGcEntryLogMetadataCachePath() { - return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, getLedgerDirNames()[0] + "/" + ENTRYLOG_INDEX_CACHE); + return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, null); } /** * Set directory to persist Entrylog metadata if gcPersistentEntrylogMetadataMapEnabled is true. + * If it set, it only works for one ledger directory configured for ledgerDirectories. For multi ledgerDirectory + * configured, keep the default value is the best practice. * - * @param gcPersistentEntrylogMetadataMapPath. + * @param gcEntrylogMetadataCachePath * @return server configuration. */ public ServerConfiguration setGcEntryLogMetadataCachePath(String gcEntrylogMetadataCachePath) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java index 8e10a151d55..d928aac2fc5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java @@ -31,7 +31,7 @@ public class BookKeeperConstants { public static final String BOOKIE_STATUS_FILENAME = "BOOKIE_STATUS"; public static final String PASSWD = "passwd"; public static final String CURRENT_DIR = "current"; - public static final String ENTRYLOG_INDEX_CACHE = "entrylogIndexCache"; + public static final String METADATA_CACHE = "metadata-cache"; public static final String READONLY = "readonly"; // ////////////////////////// From 0de75bc1ad6521bb96e358fe69882425afe8c260 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 10 Feb 2022 23:16:30 -0800 Subject: [PATCH 23/79] [ISSUE 3038] Fixed flaky CompactionTest.testMinorCompactionWithMaxTimeMillis (#3039) --- .../bookkeeper/bookie/CompactionTest.java | 71 ++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java index 1c2ee73f11d..aef181f2482 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java @@ -402,7 +402,7 @@ public void testMinorCompaction() throws Exception { } @Test - public void testMinorCompactionWithMaxTimeMillis() throws Exception { + public void testMinorCompactionWithMaxTimeMillisOk() throws Exception { // prepare data LedgerHandle[] lhs = prepareData(6, false); @@ -419,8 +419,10 @@ public void testMinorCompactionWithMaxTimeMillis() throws Exception { c.setMajorCompactionInterval(240000); // Setup limit on compaction duration. - c.setMinorCompactionMaxTimeMillis(15); - c.setMajorCompactionMaxTimeMillis(15); + // The limit is enough to compact. + c.setMinorCompactionMaxTimeMillis(5000); + c.setMajorCompactionMaxTimeMillis(5000); + return c; }); @@ -477,7 +479,70 @@ public void testMinorCompactionWithMaxTimeMillis() throws Exception { } + @Test + public void testMinorCompactionWithMaxTimeMillisTooShort() throws Exception { + // prepare data + LedgerHandle[] lhs = prepareData(6, false); + + for (LedgerHandle lh : lhs) { + lh.close(); + } + + // disable major compaction + // restart bookies + restartBookies(c-> { + c.setMajorCompactionThreshold(0.0f); + c.setGcWaitTime(60000); + c.setMinorCompactionInterval(120000); + c.setMajorCompactionInterval(240000); + + // Setup limit on compaction duration. + // The limit is not enough to finish the compaction + c.setMinorCompactionMaxTimeMillis(1); + c.setMajorCompactionMaxTimeMillis(1); + + return c; + }); + + getGCThread().enableForceGC(); + getGCThread().triggerGC().get(); + assertTrue( + "ACTIVE_ENTRY_LOG_COUNT should have been updated", + getStatsProvider(0) + .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_COUNT) + .getSample().intValue() > 0); + assertTrue( + "ACTIVE_ENTRY_LOG_SPACE_BYTES should have been updated", + getStatsProvider(0) + .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_SPACE_BYTES) + .getSample().intValue() > 0); + + long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime; + long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime; + assertFalse(getGCThread().enableMajorCompaction); + assertTrue(getGCThread().enableMinorCompaction); + + // remove ledger2 and ledger3 + bkc.deleteLedger(lhs[1].getId()); + bkc.deleteLedger(lhs[2].getId()); + LOG.info("Finished deleting the ledgers contains most entries."); + getGCThread().enableForceGC(); + getGCThread().triggerGC().get(); + + // after garbage collection, major compaction should not be executed + assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime); + assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime); + + // entry logs ([0,1,2].log) should be compacted. + for (File ledgerDirectory : tmpDirs.getDirs()) { + // Compaction of at least one of the files should not finish up + assertTrue("Not found entry log file ([0,1,2].log that should not have been compacted in ledgerDirectory: " + + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2)); + } + + verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed()); + } @Test public void testForceMinorCompaction() throws Exception { From 4debbbf84db09f80ffeb4154e8e6feed134be0c0 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Mon, 14 Feb 2022 10:39:23 +0800 Subject: [PATCH 24/79] change log level from error to warn when dns resolver initialize failed (#2856) Descriptions of the changes in this PR: ### Motivation When start bookie, it will throws the following error message when dns resolver initialize failed. ``` [main] ERROR org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl - Failed to initialize DNS Resolver org.apache.bookkeeper.net.ScriptBasedMapping, used default subnet resolver : java.lang.RuntimeException: No network topology script is found when using script based DNS resolver. ``` It is confusing for users. ### Modification 1. change the log level from error to warn. --- .../bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 50297d121c8..46c5a10786b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -269,7 +269,7 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, } } catch (RuntimeException re) { if (!conf.getEnforceMinNumRacksPerWriteQuorum()) { - LOG.error("Failed to initialize DNS Resolver {}, used default subnet resolver ", + LOG.warn("Failed to initialize DNS Resolver {}, used default subnet resolver ", dnsResolverName, re); dnsResolver = new DefaultResolver(this::getDefaultRack); dnsResolver.setBookieAddressResolver(bookieAddressResolver); From 4710fed8a409736e9e5e0aae5b87fdc6b56ed51a Mon Sep 17 00:00:00 2001 From: Jack Vanlightly Date: Mon, 14 Feb 2022 19:56:19 +0100 Subject: [PATCH 25/79] Ensure BookKeeper process receives sigterm in docker container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation Current official docker images do not handle the SIGTERM sent by the docker runtime and so get killed after the timeout. No graceful shutdown occurs. The reason is that the entrypoint does not use `exec` when executing the `bin/bookkeeper` shell script and so the BookKeeper process cannot receive signals from the docker runtime. ### Changes Use `exec` when calling the `bin/bookkeeper` shell script. Reviewers: Nicolò Boschi , Enrico Olivelli , Lari Hotari , Matteo Merli This closes #2857 from Vanlightly/docker-image-handle-sigterm --- docker/scripts/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index 657eb6b918e..753f59a20db 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -41,11 +41,11 @@ function run_command() { chmod -R +x ${BINDIR} chmod -R +x ${SCRIPTS_DIR} echo "This is root, will use user $BK_USER to run command '$@'" - sudo -s -E -u "$BK_USER" /bin/bash "$@" + exec sudo -s -E -u "$BK_USER" /bin/bash -c 'exec "$@"' -- "$@" exit else echo "Run command '$@'" - $@ + exec "$@" fi } From 50f5287f3637c5fa01a4d146477087b217bdebd5 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Tue, 15 Feb 2022 03:00:06 +0800 Subject: [PATCH 26/79] make rocksdb format version configurable ### Motivation Fix #2823 RocksDB support several format versions which uses different data structure to implement key-values indexes and have huge different performance. https://rocksdb.org/blog/2019/03/08/format-version-4.html https://github.com/facebook/rocksdb/blob/d52b520d5168de6be5f1494b2035b61ff0958c11/include/rocksdb/table.h#L368-L394 ```C++ // We currently have five versions: // 0 -- This version is currently written out by all RocksDB's versions by // default. Can be read by really old RocksDB's. Doesn't support changing // checksum (default is CRC32). // 1 -- Can be read by RocksDB's versions since 3.0. Supports non-default // checksum, like xxHash. It is written by RocksDB when // BlockBasedTableOptions::checksum is something other than kCRC32c. (version // 0 is silently upconverted) // 2 -- Can be read by RocksDB's versions since 3.10. Changes the way we // encode compressed blocks with LZ4, BZip2 and Zlib compression. If you // don't plan to run RocksDB before version 3.10, you should probably use // this. // 3 -- Can be read by RocksDB's versions since 5.15. Changes the way we // encode the keys in index blocks. If you don't plan to run RocksDB before // version 5.15, you should probably use this. // This option only affects newly written tables. When reading existing // tables, the information about version is read from the footer. // 4 -- Can be read by RocksDB's versions since 5.16. Changes the way we // encode the values in index blocks. If you don't plan to run RocksDB before // version 5.16 and you are using index_block_restart_interval > 1, you should // probably use this as it would reduce the index size. // This option only affects newly written tables. When reading existing // tables, the information about version is read from the footer. // 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned // filters use a generally faster and more accurate Bloom filter // implementation, with a different schema. uint32_t format_version = 5; ``` Different format version requires different rocksDB version and it couldn't roll back once upgrade to new format version In our current RocksDB storage code, we hard code the format_version to 2, which is hard to to upgrade format_version to achieve new RocksDB's high performance. ### Changes 1. Make the format_version configurable. Reviewers: Matteo Merli , Enrico Olivelli This closes #2824 from hangc0276/chenhang/make_rocksdb_format_version_configurable --- .../bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java | 4 +++- conf/bk_server.conf | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index bda82725586..e6eb1978c9a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -83,6 +83,7 @@ public class KeyValueStorageRocksDB implements KeyValueStorage { private static final String ROCKSDB_NUM_LEVELS = "dbStorage_rocksDB_numLevels"; private static final String ROCKSDB_NUM_FILES_IN_LEVEL0 = "dbStorage_rocksDB_numFilesInLevel0"; private static final String ROCKSDB_MAX_SIZE_IN_LEVEL1_MB = "dbStorage_rocksDB_maxSizeInLevel1MB"; + private static final String ROCKSDB_FORMAT_VERSION = "dbStorage_rocksDB_format_version"; public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbConfigType, ServerConfiguration conf) throws IOException { @@ -122,6 +123,7 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo int blockSize = conf.getInt(ROCKSDB_BLOCK_SIZE, 64 * 1024); int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10); boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true); + int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 2); if (lz4CompressionEnabled) { options.setCompressionType(CompressionType.LZ4_COMPRESSION); @@ -144,7 +146,7 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); tableOptions.setBlockSize(blockSize); tableOptions.setBlockCache(cache); - tableOptions.setFormatVersion(2); + tableOptions.setFormatVersion(formatVersion); tableOptions.setChecksumType(ChecksumType.kxxHash); if (bloomFilterBitsPerKey > 0) { tableOptions.setFilterPolicy(new BloomFilter(bloomFilterBitsPerKey, false)); diff --git a/conf/bk_server.conf b/conf/bk_server.conf index 801976ebbb3..f83a46e852c 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -740,6 +740,7 @@ gcEntryLogMetadataCacheEnabled=false # dbStorage_rocksDB_numFilesInLevel0=4 # dbStorage_rocksDB_maxSizeInLevel1MB=256 # dbStorage_rocksDB_logPath= +# dbStorage_rocksDB_format_version=2 ############################################## Metadata Services ############################################## From 50b8a2322585ff2269feff064e1a81c65aad6ef5 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Tue, 15 Feb 2022 03:01:22 +0800 Subject: [PATCH 27/79] delete duplicated semicolon As title, delete duplicated semicolon Reviewers: Andrey Yegorov This closes #2810 from gaozhangmin/remove-duplicated-semicolon From 8530d5c973d1106a4508ac024518df254eac7437 Mon Sep 17 00:00:00 2001 From: Jack Vanlightly Date: Mon, 14 Feb 2022 20:11:38 +0100 Subject: [PATCH 28/79] BP-46: Running without journal proposal Includes the BP-46 design proposal markdown document. Master Issue: #2705 Reviewers: Andrey Yegorov , Enrico Olivelli This closes #2706 from Vanlightly/bp-44 --- site/bps/BP-46-run-without-journal.md | 205 ++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 site/bps/BP-46-run-without-journal.md diff --git a/site/bps/BP-46-run-without-journal.md b/site/bps/BP-46-run-without-journal.md new file mode 100644 index 00000000000..83030356db2 --- /dev/null +++ b/site/bps/BP-46-run-without-journal.md @@ -0,0 +1,205 @@ +--- +title: "BP-46: Running without the journal" +issue: https://github.com/apache/bookkeeper/2705 +state: "Under Discussion" +release: "N/A" +--- + +### Motivation + +The journal allows for fast add operations that provide strong data safety guarantees. An add operation is only acked to a client once written to the journal and an fsync performed. This however means that every entry must be written twice: once to the journal and once to an entry log file. + +This double write increases the cost of ownership as more disks must be provisioned to service requests and makes disk provisioning more complex (separating journal from entry log writes onto separate disks). Running without the journal would halve the disk IO required (ignoring indexes) thereby reducing costs and simplifying provisioning. + +However, running without the journal would introduce data consistency problems as the BookKeeper Replication Protocol requires that all writes are persistent for correctness. Running without the journal introduces the possibility of lost writes. In order to continue to offer strong data safety and support running without the journal, changes to the protocol are required. + +### A note on Response Codes + +The following categories are relevant: + +- Positive: OK +- Explicit Negative: NoSuchEntry/NoSuchLedger +- Unknown: Any other non-success response that is not an explicit negative. + +For correctness explicit negatives must be treated differently than other errors. + +### A note on Quorums + +In order to explain the protocol changes, it is useful to first consider how quorums are used for safety. We have the following relevant quorums: + +- Single bookie (S) +- Ack quorum (AQ) +- Write quorum (WQ) +- Quorum Coverage (QC) where QC = (WQ - AQ) + 1 +- Ensemble Coverage (EC) where EC = (E - AQ) + 1 +- Whole Ensemble + +Quorum Coverage (QC) and Ensemble Coverage (EC) are both defined by the following, only the cohorts differ: + +- A given property is satisfied by at least one bookie from every possible ack quorum within the cohort. +- There exists no ack quorum of bookies that do not satisfy the property within the cohort. + +For QC, the cohort is the writeset of a given entry, and therefore QC is only used when we need guarantees regarding a single entry. For EC, the cohort is the ensemble of bookies of the current fragment. EC is required when we need a guarantee across an entire fragment. + +For example: + +- For fencing, we need to ensure that no AQ of bookies is unfenced before starting the read/write phase of recovery. This is true once EC successful fencing responses have been received. +- For a recovery read, a read is only negative once we know that no AQ of bookies could exist that might have the entry. Doing otherwise could truncate committed entries from a ledger. A read is negative once NoSuchEntry responses reach QC. + +Different protocol actions require different quorums: + +- Add entry: AQ success responses +- Read entry: + - Positive when positive response from a single bookie + - Negative when explicit negative from all bookies + - Unknown: when at least one unknown and no positive from all bookies +- Fencing phase, LAC read (sent to ensemble of current fragment): + - Complete when EC positive responses + - Unknown (cannot make progress) when AQ unknown responses (fencing LAC reads cannot cause an explicit negative as fencing creates the ledger on the bookie if it doesn’t exist) +- Recovery read (sent to writeset of entry): + - Entry recoverable: AQ positive read responses + - Entry Unrecoverable: QC negative read responses + - Unknown (cannot make progress): + - QC unknown responses or + - All responses received, but not enough for either a positive or negative + + +### Impact of Undetected Data Loss on Consistency + +The ledger recovery process assumes that ledger entries are never arbitrarily lost. In the event of the loss of an entry, the recovery process can: +- allow the original client to keep writing entries to a ledger that has just been fenced and closed, thus losing those entries +- allow the recovery client to truncate the ledger too soon, closing it with a last entry id lower than that of previously acknowledged entries - thus losing data. + +The following scenarios assume existing behaviour but simply skipping the writing of entries and fencing ops to the journal. + +### Scenario 1 - Lost Fenced Status Allows Writes After Ledger Close + +1. 3 bookies, B1, B2 & B3 +2. 2 clients, C1 & C2 +3. 1 ledger, L1, with e3:w3:a2 configuration. +4. C1 writes entry E1 to L1. The write hits all three bookies. +5. C1 hangs for an indeterminate length of time. +6. C2 sees that C1 is unresponsive, and assumes it has failed. C2 tries to recover the ledger L1. +7. L1 sends a fencing message to all bookies in the ensemble. +8. The fencing message succeeds in arriving at B1 & B2 and is acknowledged by both. The message to B3 is lost. +9. C2 sees that at least one bookie in each possible ack quorum has acknowledged the fencing message (EC threshold reached), so continues with the read/write phase of recovery, finding that E1 is the last entry of the ledger, and committing the endpoint of the ledger in the ZK. +10. B2 crashes and boots again with all unflushed operations lost. +11. C1 wakes up and writes entry E2 to all bookies. B2 & B3 acknowledge positively, so C1 considers E2 as persisted. B1 rejects the message as the ledger is fenced, but since ack quorum is 2, B2 & B3 are enough to consider the entry written. + +### Scenario 2 - Recovery Truncates Previously Acknowledged Entries + +1. C1 adds E0 to B1, B2, B3 +2. B1 and B3 confirms. C1 confirms the write to its client. +3. C2 starts recovery +4. B2 fails to respond. C1 tries to change ensemble but gets a metadata version conflict. +5. B1 crashes and restarts, has lost E0 (undetected) +6. C2 fences the ledger on B1, B2, B3 +7. C2 sends Read E0 to B1, B2, B3 +8. B1 responds with NoSuchEntry +9. B2 responds with NoSuchEntry +10. QC negative response threshold reached. C2 closes the ledger as empty. Losing E0. + +The problem is that without the journal (and syncing to entry log files before acknowledgement) a bookie can: +- lose the fenced status of a previously existing ledger +- respond with an explicit negative even though it had previously seen an entry. + +Undetected data loss could occur when running without the journal. Bookie crashes and loses most recent entries and fence statuses that had not yet been written and synced to disk. + +### A note on cookies + +Cookies play an essential part in the bookkeeper replication protocol, but their purpose is often unclear. + +When a bookie boots for the first time, it generates a cookie. The cookie encapsulates the identity of the bookie and should be considered immutable. This identity contains the advertised address of the bookie, the disks used for the journal, index, and ledger storage, and a unique ID. The bookie writes the cookie to ZK and each of the disks in use. On all subsequent boots, if the cookie is missing from any of these places, the bookie fails to boot. + +The absence of a disk's cookie implies that the rest of the disk's data is also missing. Cookie validation is performed on boot-up and prevents the boot from succeeding if the validation fails, thus preventing the bookie starting with undetected data loss. + +This proposal improves the cookie mechanism by automating the resolution of a cookie validation error which currently requires human intervention to resolve. This automated feature will be configurable (enabled or disabled) and additionally a CLI command will be made available so an admin can manually run the operation (for when this feature is disabled - likely to be the default). + +### Proposed Changes + +The proposed changes involve: +- A new config that controls whether add operations go into the journal +- Detecting possible data loss on boot +- Prevent explicit negative responses when data loss may have occurred, instead reply with unknown code, until data is repaired. +- Repair data loss +- Auto fix cookies (with new config to enable or disable the feature) +- CLI command for admin to run fix cookie logic in the case that auto fix is disabled + +In these proposed changes, when running "without" the journal, the journal still exists, but add entry operations skip the addition to the journal. The boot-up sequence still replays the journal. + +Add operations can be configured to be written to the journal or not based on the config `journalWriteData`. When set to `false`, add operations are not added to the journal. + +### Detecting Data Loss On Boot + +The new mechanism for data loss detection is checking for an unclean shutdown (aka a crash or abrupt termination of the bookie). When an unclean shutdown is detected further measures are taken to prevent data inconsistency. + +The unclean shutdown detection will consist of setting a bit in the index on start-up and clearing it on shutdown. On subsequent start-up, the value will be checked and if it remains set, it knows that the prior shutdown was not clean. + +Cookie validation will continue to be used to detect booting with one or more missing or empty disks (that once existed and contained a cookie). + +### Protection Mechanism + +Once possible data loss has been detected the following protection mechanism is carried out during the boot: + +- Fencing: Ledger metadata for all ledgers of the cluster are obtained and all those ledgers are fenced on this bookie. This prevents data loss scenario 1. +- Limbo: All open ledgers are placed in the limbo status. Limbo ledgers can serve read requests, but never respond with an explicit negative, all explicit negatives are converted to unknowns (with the use of a new code EUNKNOWN). +- Recovery: All open ledgers are opened and recovered. +- Repair: Each ledger is scanned and any missing entries are sourced from peers. +- Limbo ledgers that have been repaired have their limbo status cleared. + +### The Full Boot-Up Sequence + +This mechanism of limbo ledgers and self-repair needs to work hand-in hand with the cookie validation check. Combining everything together: + +On boot: +1. Check for unclean shutdown and validate cookies +2. Fetch the metadata for all ledgers in the cluster from ZK where the bookie is a member of its ensemble. +3. Phase one: + - If the cookie check fails or unclean shutdown is detected: + - For each non-closed ledger, mark the ledger as fenced and in-limbo in the index. + - Update the cookie if it was a cookie failure +4. Phase two + - For each ledger + 1. If the ledger is in-limbo, open and recover the ledger. + 2. Check that all entries assigned to this bookie exist in the index. + 3. For any entries that are missing, copy from another bookie. + 4. Clear limbo status if set + +When booting a bookie with empty disks, only phase one needs to be complete before the bookie makes itself available for client requests. + +In phase one, if the cookie check fails, we mark all non-closed ledgers as “fenced”. This prevents any future writes to these ledgers on this bookie. This solves the problem of an empty bookie disk allowing writes to closed ledgers (Scenario 1). + +Given that the algorithm solves both the issues that cookies are designed to solve, we can now allow the bookie to update its cookie without operator intervention. + +### Formal Verification of Proposed Changes + +The use of the limbo status and fencing of all ledgers on boot-up when detecting an unclean shutdown has been modelled in TLA+. It does not model the whole boot-up sequence but a simplified version with only fencing and limbo status. + +The specification models the lifetime of a single ledger and includes a single bookie crashing, losing all data. The specification allows the testing of: + +- enabling/disabling the fencing +- enabling/disabling the limbo status. + +When running without limbo status, the model checker finds the counterexample of scenario 2. When running without fencing of all ledgers, the model checker finds the counterexample of scenario 1. When running with both enabled, the model checker finds no invariant violation. + +The specification can be found here: https://github.com/Vanlightly/bookkeeper-tlaplus + +### Public Interfaces + +- Return codes. Addition of a new return code: `EUNKNOWN` which is returned when a read hits an in-limbo ledger and that ledger not contain the requested entry id. +- Bookie ledger metadata format (LedgerData). Addition of the limbo status. + +### Compatibility, Deprecation, and Migration Plan + +- Because we only skip the journal for add operations, there is no impact on existing deployments. When a bookie is booted with the new version, and `journalWriteData` is set to false, the journal is still replayed on boot-up causing no risk of data loss in the transition. + +### Test Plan + +- There is confidence in the design due to the modelling in TLA+ but this model does not include the full boot sequence. +- The implementation will require aggressive chaos testing to ensure correctness. + +### Rejected Alternatives + +Entry Log Per Ledger (ELPL) without the journal. From our performance testing of ELPL, performance degrades significantly with a large number of active ledgers and syncing to disk multiple times a second (which is required to offer low latency writes). + +In the future this design could be extended to offer ledger level configuration of journal use. The scope of this BP is limited to cluster level. \ No newline at end of file From 9fd35f9cefa1b5925dc29e2e734cc5be9df346f1 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Tue, 15 Feb 2022 03:17:28 +0800 Subject: [PATCH 29/79] Replication stat num-under-replicated-ledgers changed as with the process of replication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation Now ReplicationStats numUnderReplicatedLedger registers when `publishSuspectedLedgersAsync`, but its value doesn't decrease as with the ledger replicated successfully, We cannot know the progress of replication from the stat. Changes registers a notifyUnderReplicationLedgerChanged when auditor starts. numUnderReplicatedLedger value will decrease when the ledger path under replicate deleted. Reviewers: Nicolò Boschi , Enrico Olivelli , Andrey Yegorov This closes #2805 from gaozhangmin/replication-stats-num-under-replicated-ledgers --- .../meta/LedgerUnderreplicationManager.java | 9 ++++ .../meta/NullMetadataBookieDriver.java | 2 + .../meta/ZkLedgerUnderreplicationManager.java | 23 +++++++++ .../bookkeeper/replication/Auditor.java | 47 +++++++++++++++++++ .../replication/ReplicationStats.java | 2 + 5 files changed, 83 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java index ac468d31563..c24f9208023 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java @@ -236,6 +236,15 @@ boolean initializeLostBookieRecoveryDelay(int lostBookieRecoveryDelay) */ long getReplicasCheckCTime() throws ReplicationException.UnavailableException; + /** + * Receive notification asynchronously when the num of under-replicated ledgers Changed. + * + * @param cb + * @throws ReplicationException.UnavailableException + */ + void notifyUnderReplicationLedgerChanged(GenericCallback cb) + throws ReplicationException.UnavailableException; + /** * Receive notification asynchronously when the lostBookieRecoveryDelay value is Changed. * diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java index 3a4247f6c61..4a1e6e419fa 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java @@ -394,5 +394,7 @@ public String getReplicationWorkerIdRereplicatingLedger(long ledgerId) throws ReplicationException.UnavailableException { throw new ReplicationException.UnavailableException("null"); } + @Override + public void notifyUnderReplicationLedgerChanged(GenericCallback cb) {} } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java index b340e0721cb..9dcc81c1622 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java @@ -59,6 +59,7 @@ import org.apache.bookkeeper.util.BookKeeperConstants; import org.apache.bookkeeper.util.SubTreeCache; import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.AddWatchMode; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.Code; @@ -866,6 +867,28 @@ public int getLostBookieRecoveryDelay() throws UnavailableException { } } + @Override + public void notifyUnderReplicationLedgerChanged(GenericCallback cb) throws UnavailableException { + LOG.debug("notifyUnderReplicationLedgerChanged()"); + Watcher w = new Watcher() { + @Override + public void process(WatchedEvent e) { + if (e.getType() == Event.EventType.NodeDeleted && idExtractionPattern.matcher(e.getPath()).find()) { + cb.operationComplete(0, null); + } + } + }; + try { + zkc.addWatch(urLedgerPath, w, AddWatchMode.PERSISTENT_RECURSIVE); + } catch (KeeperException ke) { + LOG.error("Error while checking the state of underReplicated ledgers", ke); + throw new ReplicationException.UnavailableException("Error contacting zookeeper", ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new ReplicationException.UnavailableException("Interrupted while contacting zookeeper", ie); + } + } + @Override public void notifyLostBookieRecoveryDelayChanged(GenericCallback cb) throws UnavailableException { LOG.debug("notifyLostBookieRecoveryDelayChanged()"); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java index cd85145c1b3..c3b69dd4beb 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java @@ -34,8 +34,10 @@ import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY; +import static org.apache.bookkeeper.replication.ReplicationStats.NUM_REPLICATED_LEDGERS; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS; +import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS_GUAGE; import static org.apache.bookkeeper.replication.ReplicationStats.PLACEMENT_POLICY_CHECK_TIME; import static org.apache.bookkeeper.replication.ReplicationStats.REPLICAS_CHECK_TIME; import static org.apache.bookkeeper.replication.ReplicationStats.UNDER_REPLICATED_LEDGERS_TOTAL_SIZE; @@ -45,6 +47,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.collect.HashMultiset; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Multiset; import com.google.common.collect.Sets; @@ -162,6 +165,7 @@ public class Auditor implements AutoCloseable { private final AtomicInteger numLedgersFoundHavingLessThanAQReplicasOfAnEntry; private final AtomicInteger numLedgersHavingLessThanWQReplicasOfAnEntryGuageValue; private final AtomicInteger numLedgersFoundHavingLessThanWQReplicasOfAnEntry; + private final AtomicInteger underReplicatedLedgersGuageValue; private final long underreplicatedLedgerRecoveryGracePeriod; private final int zkOpTimeoutMs; private final Semaphore openLedgerNoRecoverySemaphore; @@ -234,6 +238,11 @@ public class Auditor implements AutoCloseable { help = "the number of delayed-bookie-audits cancelled" ) private final Counter numDelayedBookieAuditsCancelled; + @StatsDoc( + name = NUM_REPLICATED_LEDGERS, + help = "the number of replicated ledgers" + ) + private final Counter numReplicatedLedgers; @StatsDoc( name = NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY, help = "Gauge for number of ledgers not adhering to placement policy found in placement policy check" @@ -266,6 +275,11 @@ public class Auditor implements AutoCloseable { + ", this doesn't include ledgers counted towards numLedgersHavingLessThanAQReplicasOfAnEntry" ) private final Gauge numLedgersHavingLessThanWQReplicasOfAnEntry; + @StatsDoc( + name = NUM_UNDER_REPLICATED_LEDGERS_GUAGE, + help = "Gauge for num of underreplicated ledgers" + ) + private final Gauge numUnderReplicatedLedgers; static BookKeeper createBookKeeperClient(ServerConfiguration conf) throws InterruptedException, IOException { return createBookKeeperClient(conf, NullStatsLogger.INSTANCE); @@ -363,6 +377,7 @@ public Auditor(final String bookieIdentifier, this.openLedgerNoRecoverySemaphoreWaitTimeoutMSec = conf.getAuditorAcquireConcurrentOpenLedgerOperationsTimeoutMSec(); + this.underReplicatedLedgersGuageValue = new AtomicInteger(0); numUnderReplicatedLedger = this.statsLogger.getOpStatsLogger(ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS); underReplicatedLedgerTotalSize = this.statsLogger.getOpStatsLogger(UNDER_REPLICATED_LEDGERS_TOTAL_SIZE); uRLPublishTimeForLostBookies = this.statsLogger @@ -379,6 +394,7 @@ public Auditor(final String bookieIdentifier, numBookieAuditsDelayed = this.statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); numDelayedBookieAuditsCancelled = this.statsLogger .getCounter(ReplicationStats.NUM_DELAYED_BOOKIE_AUDITS_DELAYES_CANCELLED); + numReplicatedLedgers = this.statsLogger.getCounter(NUM_REPLICATED_LEDGERS); numLedgersNotAdheringToPlacementPolicy = new Gauge() { @Override public Integer getDefaultValue() { @@ -459,6 +475,18 @@ public Integer getSample() { }; this.statsLogger.registerGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY, numLedgersHavingLessThanWQReplicasOfAnEntry); + numUnderReplicatedLedgers = new Gauge() { + @Override + public Integer getDefaultValue() { + return 0; + } + + @Override + public Integer getSample() { + return underReplicatedLedgersGuageValue.get(); + } + }; + this.statsLogger.registerGauge(NUM_UNDER_REPLICATED_LEDGERS_GUAGE, numUnderReplicatedLedgers); this.bkc = bkc; this.ownBkc = ownBkc; @@ -703,6 +731,15 @@ public void start() { submitShutdownTask(); } + try { + this.ledgerUnderreplicationManager.notifyUnderReplicationLedgerChanged( + new UnderReplicatedLedgersChangedCb()); + } catch (UnavailableException ue) { + LOG.error("Exception while registering for under-replicated ledgers change notification, so exiting", + ue); + submitShutdownTask(); + } + scheduleBookieCheckTask(); scheduleCheckAllLedgersTask(); schedulePlacementPolicyCheckTask(); @@ -1010,6 +1047,16 @@ public void run() { }), initialDelay, interval, TimeUnit.SECONDS); } + private class UnderReplicatedLedgersChangedCb implements GenericCallback { + @Override + public void operationComplete(int rc, Void result) { + Iterator underreplicatedLedgersInfo = ledgerUnderreplicationManager + .listLedgersToRereplicate(null); + underReplicatedLedgersGuageValue.set(Iterators.size(underreplicatedLedgersInfo)); + numReplicatedLedgers.inc(); + } + } + private class LostBookieRecoveryDelayChangedCb implements GenericCallback { @Override public void operationComplete(int rc, Void result) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java index d04fcf04279..74b76b23b22 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java @@ -64,4 +64,6 @@ public interface ReplicationStats { String REPLICATE_EXCEPTION = "exceptions"; String NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER = "NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER"; String NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION = "NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION"; + String NUM_UNDER_REPLICATED_LEDGERS_GUAGE = "NUM_UNDER_REPLICATED_LEDGERS_GUAGE"; + String NUM_REPLICATED_LEDGERS = "NUM_REPLICATED_LEDGERS"; } From 743def3634cc95e651faa0ca1a6d3c50483f2d37 Mon Sep 17 00:00:00 2001 From: shustsud <51769018+shustsud@users.noreply.github.com> Date: Tue, 15 Feb 2022 04:20:49 +0900 Subject: [PATCH 30/79] Explicit error message if an exception other than BKNoSuchLedgerExistsOnMetadataServerException occurs in over-replicated ledger GC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation - Even if an exception other than BKNoSuchLedgerExistsOnMetadataServerException occurs of readLedgerMetadata in over-replicated ledger GC, nothing will be output to the log. (https://github.com/apache/bookkeeper/pull/2844#discussion_r735219876) ### Changes - If an exception other than BKNoSuchLedgerExistsOnMetadataServerException occurs in readLedgerMetadata, output information to the log. Reviewers: Andrey Yegorov , Nicolò Boschi This closes #2873 from shustsud/improved_error_handling --- .../bookkeeper/bookie/ScanAndCompareGarbageCollector.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java index 93bf47c4e03..b3c724b231e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java @@ -243,6 +243,10 @@ private Set removeOverReplicatedledgers(Set bkActiveledgers, final G continue; } } catch (Throwable t) { + if (!(t.getCause() instanceof BKException.BKNoSuchLedgerExistsOnMetadataServerException)) { + LOG.warn("Failed to get metadata for ledger {}. {}: {}", + ledgerId, t.getClass().getName(), t.getMessage()); + } latch.countDown(); continue; } @@ -268,6 +272,10 @@ private Set removeOverReplicatedledgers(Set bkActiveledgers, final G overReplicatedLedgers.add(ledgerId); garbageCleaner.clean(ledgerId); } + } else if (!(exception + instanceof BKException.BKNoSuchLedgerExistsOnMetadataServerException)) { + LOG.warn("Failed to get metadata for ledger {}. {}: {}", + ledgerId, exception.getClass().getName(), exception.getMessage()); } } finally { semaphore.release(); From c5a516702e69bd38ca0fe77fb7300c8e7e36f065 Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Tue, 15 Feb 2022 03:32:55 +0800 Subject: [PATCH 31/79] fix(cli): incorrect description for autodiscovery Signed-off-by: Eric Shen Descriptions of the changes in this PR: ### Motivation The description of `bin/bookkeeper autorecovery` is wrong, it won't start in daemon. ### Changes * Changed the description in bookkeeper shell * Update the doc Reviewers: Yong Zhang This closes #2910 from ericsyh/fix-bk-cli --- bin/bookkeeper | 2 +- site/_data/cli/bookkeeper.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bookkeeper b/bin/bookkeeper index 47cb42de270..ac409684ff5 100755 --- a/bin/bookkeeper +++ b/bin/bookkeeper @@ -62,7 +62,7 @@ where command is one of: [service commands] bookie Run a bookie server - autorecovery Run AutoRecovery service daemon + autorecovery Run AutoRecovery service zookeeper Run zookeeper server [development commands] diff --git a/site/_data/cli/bookkeeper.yaml b/site/_data/cli/bookkeeper.yaml index cc4b2eca5d5..fd780f151f3 100644 --- a/site/_data/cli/bookkeeper.yaml +++ b/site/_data/cli/bookkeeper.yaml @@ -7,7 +7,7 @@ commands: description: Starts up an ensemble of N bookies in a single JVM process. Typically used for local experimentation and development. argument: N - name: autorecovery - description: Runs the autorecovery service daemon. + description: Runs the autorecovery service. - name: upgrade description: Upgrades the bookie's filesystem. options: From 0d98266c1de479e9741949d163a3764030569efb Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Tue, 15 Feb 2022 03:36:06 +0800 Subject: [PATCH 32/79] Add flaky-test template to track many flaky-test. ### Motivation I found many flaky-test like #3031 #3034 #3033. Because many flaky tests are actually production code issues so I think it's a good way to add flaky-test template to track them ### Changes - Add flaky-test template. Reviewers: Andrey Yegorov This closes #3035 from mattisonchao/template_flaky_test --- .github/ISSUE_TEMPLATE/flaky_test.md | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/flaky_test.md diff --git a/.github/ISSUE_TEMPLATE/flaky_test.md b/.github/ISSUE_TEMPLATE/flaky_test.md new file mode 100644 index 00000000000..fc8159776c7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/flaky_test.md @@ -0,0 +1,31 @@ +--- +name: Flaky test +about: Report a flaky test failure +title: 'Flaky-test: test_class.test_method' +labels: ["area/tests", "flaky-tests"] +assignees: '' +--- + +test_class.test_method is flaky. It fails sporadically. + +``` +[relevant parts of the exception stacktrace here] +``` + + - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper - bookkeeper-benchmark - Apache BookKeeper :: Benchmark - - UTF-8 - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - target/latencyDump.dat - - - - - - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - compile - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - diff --git a/bookkeeper-common-allocator/pom.xml b/bookkeeper-common-allocator/pom.xml deleted file mode 100644 index 31315773665..00000000000 --- a/bookkeeper-common-allocator/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-common-allocator - Apache BookKeeper :: Common :: Allocator - - - io.netty - netty-buffer - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml deleted file mode 100644 index e7709f4597a..00000000000 --- a/bookkeeper-common/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-common - Apache BookKeeper :: Common - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - org.apache.bookkeeper - cpu-affinity - ${project.parent.version} - - - com.google.guava - guava - - - io.netty - netty-common - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - org.jctools - jctools-core - - - com.google.code.findbugs - jsr305 - provided - - - com.google.errorprone - error_prone_annotations - provided - - - org.apache.commons - commons-lang3 - test - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - diff --git a/bookkeeper-dist/all/pom.xml b/bookkeeper-dist/all/pom.xml deleted file mode 100644 index 146dafa3271..00000000000 --- a/bookkeeper-dist/all/pom.xml +++ /dev/null @@ -1,159 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bookkeeper-dist-all - jar - Apache BookKeeper :: Dist (All) - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - - org.apache.bookkeeper.stats - codahale-metrics-provider - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper.http - vertx-http-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-benchmark - ${project.version} - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - org.xerial.snappy - snappy-java - - - - io.dropwizard.metrics - metrics-core - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-all-${project.version} - false - - ../src/assemble/bin-all.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/bookkeeper-dist/bkctl/pom.xml b/bookkeeper-dist/bkctl/pom.xml deleted file mode 100644 index 8ac6545851e..00000000000 --- a/bookkeeper-dist/bkctl/pom.xml +++ /dev/null @@ -1,96 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bkctl - jar - Apache BookKeeper :: Dist (Bkctl) - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - org.rocksdb - rocksdbjni - - - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bkctl-${project.version} - true - - ../src/assemble/bkctl.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/bookkeeper-dist/build.gradle b/bookkeeper-dist/build.gradle index e8c086821da..aeac682345a 100644 --- a/bookkeeper-dist/build.gradle +++ b/bookkeeper-dist/build.gradle @@ -38,7 +38,6 @@ distributions { "**/README.md", "**/bin/**", "**/conf/**", - "**/pom.xml", "**/src/**", "deploy/**", "doc/**", diff --git a/bookkeeper-dist/pom.xml b/bookkeeper-dist/pom.xml deleted file mode 100644 index 5285131dbcb..00000000000 --- a/bookkeeper-dist/pom.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - bookkeeper-dist - pom - Apache BookKeeper :: Dist (Parent) - - all - server - bkctl - - - UTF-8 - UTF-8 - - - - - maven-jar-plugin - ${maven-jar-plugin.version} - - - default-jar - none - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-${project.version} - - src/assemble/src.xml - - posix - - - - package - - single - - - - - - - diff --git a/bookkeeper-dist/server/pom.xml b/bookkeeper-dist/server/pom.xml deleted file mode 100644 index 83777f5f342..00000000000 --- a/bookkeeper-dist/server/pom.xml +++ /dev/null @@ -1,146 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bookkeeper-dist-server - jar - Apache BookKeeper :: Dist (Server) - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper.http - vertx-http-server - ${project.version} - - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - org.xerial.snappy - snappy-java - - - - io.dropwizard.metrics - metrics-core - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-server-${project.version} - true - - ../src/assemble/bin-server.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - - diff --git a/bookkeeper-dist/src/assemble/src.xml b/bookkeeper-dist/src/assemble/src.xml index 146061e28a4..0301c8a9bde 100644 --- a/bookkeeper-dist/src/assemble/src.xml +++ b/bookkeeper-dist/src/assemble/src.xml @@ -30,7 +30,6 @@ **/README.md **/LICENSE **/NOTICE - **/pom.xml **/*gradle* **/src/** **/conf/** diff --git a/bookkeeper-http/http-server/pom.xml b/bookkeeper-http/http-server/pom.xml deleted file mode 100644 index 044a5c02850..00000000000 --- a/bookkeeper-http/http-server/pom.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - http-server - Apache BookKeeper :: Http :: Http Server - http://maven.apache.org - - - commons-configuration - commons-configuration - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - diff --git a/bookkeeper-http/pom.xml b/bookkeeper-http/pom.xml deleted file mode 100644 index bb9307322bf..00000000000 --- a/bookkeeper-http/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.http - bookkeeper-http - pom - Apache BookKeeper :: Http - - http-server - vertx-http-server - servlet-http-server - - diff --git a/bookkeeper-http/servlet-http-server/pom.xml b/bookkeeper-http/servlet-http-server/pom.xml deleted file mode 100644 index 2efa234a518..00000000000 --- a/bookkeeper-http/servlet-http-server/pom.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - servlet-http-server - Apache BookKeeper :: Bookkeeper Http :: Servlet Http Server - http://maven.apache.org - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - javax.servlet - javax.servlet-api - - - commons-io - commons-io - - - org.eclipse.jetty - jetty-server - test - - - org.eclipse.jetty - jetty-webapp - test - - - - diff --git a/bookkeeper-http/vertx-http-server/pom.xml b/bookkeeper-http/vertx-http-server/pom.xml deleted file mode 100644 index c8914241361..00000000000 --- a/bookkeeper-http/vertx-http-server/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - vertx-http-server - Apache BookKeeper :: Bookkeeper Http :: Vertx Http Server - http://maven.apache.org - - - io.vertx - vertx-core - - - io.vertx - vertx-web - - - com.google.guava - guava - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - diff --git a/bookkeeper-proto/pom.xml b/bookkeeper-proto/pom.xml deleted file mode 100644 index 0894fd064ec..00000000000 --- a/bookkeeper-proto/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-proto - Apache BookKeeper :: Protocols - - - com.google.protobuf - protobuf-java - - - - - - org.apache.rat - apache-rat-plugin - - - - **/DataFormats.java - **/BookkeeperProtocol.java - **/DbLedgerStorageDataFormats.java - **/.checkstyle - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - true - - - - - compile - - - - - - - diff --git a/bookkeeper-server/build.gradle b/bookkeeper-server/build.gradle index 5b0cd26ba5d..d55aa8a9c2a 100644 --- a/bookkeeper-server/build.gradle +++ b/bookkeeper-server/build.gradle @@ -80,11 +80,6 @@ dependencies { testImplementation depLibs.log4jCore } -task writeClasspath { - buildDir.mkdirs() - new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" -} - test { retry { maxFailures = 200 @@ -101,3 +96,6 @@ test.doFirst { jvmArgs("-Djunit.timeout.test=600000", "-Djunit.max.retry=3", "-Djava.net.preferIPv4Stack=true", "-Dio.netty.leakDetection.level=paranoid") } +jar { + dependsOn tasks.named("writeClasspath") +} diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml deleted file mode 100644 index 469aab161d9..00000000000 --- a/bookkeeper-server/pom.xml +++ /dev/null @@ -1,336 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-server - Apache BookKeeper :: Server - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-common-allocator - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-proto - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.parent.version} - - - org.rocksdb - rocksdbjni - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - org.apache.zookeeper - zookeeper - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - linux-x86_64 - - - io.netty - netty-tcnative-boringssl-static - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper - circe-checksum - ${project.version} - - - commons-cli - commons-cli - - - commons-codec - commons-codec - - - commons-io - commons-io - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - org.bouncycastle - bc-fips - - - com.beust - jcommander - - - net.java.dev.jna - jna - - - org.apache.httpcomponents - httpclient - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - test-jar - test - - - org.apache.kerby - kerby-config - ${kerby.version} - test - - - org.slf4j - * - - - - - org.apache.kerby - kerb-simplekdc - ${kerby.version} - test - - - org.slf4j - * - - - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.parent.version} - test - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - test - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.rat - apache-rat-plugin - - - - **/target/**/* - **/.classpath - **/.gitignore - **/.project - **/.checkstyle - **/.settings/* - src/test/resources/server-key.pem - src/test/resources/server-key.p12 - src/test/resources/server-key.jks - src/test/resources/server-cert.pem - src/test/resources/client-key.pem - src/test/resources/client-key.p12 - src/test/resources/client-key.jks - src/test/resources/client-cert.pem - src/test/resources/keyStoreClientPassword.txt - src/test/resources/keyStoreServerPassword.txt - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - false - - - listener - org.apache.bookkeeper.common.testing.util.TimedOutTestsListener - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - none - org.apache.bookkeeper.client:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature - - - Bookkeeper - org.apache.bookkeeper* - - - - - - attach-javadocs - - jar - - - - - - - - - vertx-http-server - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - - - - - tls-certs - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - Generate Self-Signed Certificates - generate-test-resources - - exec - - - ${basedir}/src/test/resources - ${basedir}/src/test/resources/generateKeysAndCerts.sh - - - - - - - - - - skipBookKeeperServerTests - - - skipBookKeeperServerTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - - - diff --git a/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml b/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml deleted file mode 100644 index 7ba68aa96e2..00000000000 --- a/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.bookkeeper.stats - codahale-metrics-provider - Apache BookKeeper :: Stats Providers :: Codahale Metrics - http://maven.apache.org - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - io.dropwizard.metrics - metrics-core - - - io.dropwizard.metrics - metrics-jvm - - - io.dropwizard.metrics - metrics-graphite - - - com.google.guava - guava - - - diff --git a/bookkeeper-stats-providers/pom.xml b/bookkeeper-stats-providers/pom.xml deleted file mode 100644 index faed3ad46de..00000000000 --- a/bookkeeper-stats-providers/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - bookkeeper-stats-providers - pom - Apache BookKeeper :: Stats Providers - - codahale-metrics-provider - prometheus-metrics-provider - - - diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml deleted file mode 100644 index 3dfe19c0b2a..00000000000 --- a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.bookkeeper.stats - prometheus-metrics-provider - Apache BookKeeper :: Stats Providers :: Prometheus - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - - io.prometheus - simpleclient - - - - io.prometheus - simpleclient_hotspot - - - - io.prometheus - simpleclient_servlet - - - - io.netty - netty-common - - - - org.eclipse.jetty - jetty-servlet - - - - com.google.guava - guava - - - - com.yahoo.datasketches - sketches-core - - - - diff --git a/bookkeeper-stats/pom.xml b/bookkeeper-stats/pom.xml deleted file mode 100644 index b89cb4bcc97..00000000000 --- a/bookkeeper-stats/pom.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.stats - bookkeeper-stats-api - Apache BookKeeper :: Stats API - http://maven.apache.org - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - none - org.apache.bookkeeper.stats - - - Bookkeeper Stats API - org.apache.bookkeeper.stats - - - - - - attach-javadocs - - jar - - - - - - - - - commons-configuration - commons-configuration - - - diff --git a/build.gradle b/build.gradle index 881981592f4..b92a70cbf41 100644 --- a/build.gradle +++ b/build.gradle @@ -89,6 +89,8 @@ allprojects { reportLevel = 'high' } + + task testJar(type: Jar, dependsOn: testClasses) { classifier = 'tests' from sourceSets.test.output @@ -234,6 +236,12 @@ allprojects { showStandardStreams = true } } + tasks.register('writeClasspath') { + doLast { + buildDir.mkdirs() + new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" + } + } } repositories { diff --git a/buildtools/pom.xml b/buildtools/pom.xml deleted file mode 100644 index 1dced8306c6..00000000000 --- a/buildtools/pom.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - buildtools - Apache BookKeeper :: Build Tools - 4.15.0-SNAPSHOT - diff --git a/circe-checksum/pom.xml b/circe-checksum/pom.xml deleted file mode 100644 index 75fac546587..00000000000 --- a/circe-checksum/pom.xml +++ /dev/null @@ -1,236 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - - circe-checksum - nar - Apache BookKeeper :: Circe Checksum Library - Circe Checksum Library - - - dynamic - -msse4.2 -mpclmul - - - - - - com.google.guava - guava - - - - io.netty - netty-buffer - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - src/main/assembly/assembly.xml - - false - posix - - - - make-assembly - package - - single - - - - - - org.jacoco - jacoco-maven-plugin - ${jacoco-maven-plugin.version} - - - - com/scurrilous/circe/checksum/NarSystem* - - - - - - - - - - jdk-without-javah - - [10,) - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - - default-nar-javah - none - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - -h - ${project.build.directory}/nar/javah-include - - - - - - - - mac - - - Mac OS X - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - circe-checksum - - - jni - com.scurrilous.circe.checksum - - - - ${nar.cpp.optionSet} - false - false - full - - - - - - - - Linux - - - Linux - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - circe-checksum - - - jni - com.scurrilous.circe.checksum - - - - ${nar.cpp.optionSet} - false - false - full - - - rt - - - - - - - - - diff --git a/cpu-affinity/pom.xml b/cpu-affinity/pom.xml deleted file mode 100644 index c1f8eba9d92..00000000000 --- a/cpu-affinity/pom.xml +++ /dev/null @@ -1,226 +0,0 @@ - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - - cpu-affinity - nar - Apache BookKeeper :: CPU Affinity Library - CPU Affinity Library - - - dynamic - - - - - com.google.guava - guava - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - src/main/assembly/assembly.xml - - false - posix - - - - make-assembly - package - - single - - - - - - org.apache.rat - apache-rat-plugin - - - - **/src/test/resources/proc_cpuinfo.txt - - - - - - - - - - jdk-without-javah - - [10,) - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - - default-nar-javah - none - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - -h - ${project.build.directory}/nar/javah-include - - - - - - - - mac - - - Mac OS X - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - cpu-affinity - - - jni - org.apache.bookkeeper.utils.affinity - - - - ${nar.cpp.optionSet} - false - false - full - - - - - - - - - Linux - - - Linux - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - cpu-affinity - - - jni - org.apache.bookkeeper.utils.affinity - - - - ${nar.cpp.optionSet} - false - false - full - - - rt - - - - - - - - - diff --git a/dev/check-all-licenses b/dev/check-all-licenses index 1f548c3cef1..03eeb51c65b 100755 --- a/dev/check-all-licenses +++ b/dev/check-all-licenses @@ -27,7 +27,7 @@ set -e -x HERE=$(dirname $0) BOOKKEEPER_DIST=$HERE/../bookkeeper-dist -$HERE/check-binary-license $BOOKKEEPER_DIST/server/target/bookkeeper-server-*-bin.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/all/target/bookkeeper-all-*-bin.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/target/bkctl-*-bin.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/server/build/distributions/bookkeeper-server-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/all/build/distributions/bookkeeper-all-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/build/distributions/bkctl-*.tar.gz diff --git a/dev/check-all-licenses-gradle b/dev/check-all-licenses-gradle deleted file mode 100755 index 03eeb51c65b..00000000000 --- a/dev/check-all-licenses-gradle +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Script to check licenses on a binary tarball. -# It extracts the list of bundled jars, the NOTICE, and the LICENSE -# files. It checked that every non-bk jar bundled is mentioned in the -# LICENSE file. It checked that all jar files mentioned in NOTICE and -# LICENSE are actually bundled. - -# all error fatal -set -e -x - -HERE=$(dirname $0) -BOOKKEEPER_DIST=$HERE/../bookkeeper-dist -$HERE/check-binary-license $BOOKKEEPER_DIST/server/build/distributions/bookkeeper-server-*.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/all/build/distributions/bookkeeper-all-*.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/build/distributions/bkctl-*.tar.gz - diff --git a/dev/common.sh b/dev/common.sh index d4f3d3f2efa..49825406a55 100644 --- a/dev/common.sh +++ b/dev/common.sh @@ -21,11 +21,7 @@ # function get_bk_version() { - bk_version=$(mvn -q \ - -Dexec.executable="echo" \ - -Dexec.args='${project.version}' \ - --non-recursive \ - org.codehaus.mojo:exec-maven-plugin:1.3.1:exec) + bk_version=$(${BK_HOME}/gradlew --no-daemon properties -q | grep "version:" | awk '{print $2}') echo ${bk_version} } diff --git a/dev/docker/ci.sh b/dev/docker/ci.sh deleted file mode 100755 index 75588a3d3e2..00000000000 --- a/dev/docker/ci.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -x -u - -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - -export IMAGE_NAME="bookkeeper/dev" - -pushd ${SCRIPT_DIR} - -docker build --rm=true -t ${IMAGE_NAME} . - -popd - -if [ "$(uname -s)" == "Linux" ]; then - USER_NAME=${SUDO_USER:=$USER} - USER_ID=$(id -u "${USER_NAME}") - GROUP_ID=$(id -g "${USER_NAME}") - LOCAL_HOME="/home/${USER_NAME}" -else # boot2docker uid and gid - USER_NAME=$USER - USER_ID=1000 - GROUP_ID=50 - LOCAL_HOME="/Users/${USER_NAME}" -fi - -docker build -t "${IMAGE_NAME}-${USER_NAME}" - < - - - - org.apache.bookkeeper.metadata.drivers - metadata-drivers-parent - 4.15.0-SNAPSHOT - .. - - 4.0.0 - org.apache.bookkeeper.metadata.drivers - metadata-stores-etcd - Apache BookKeeper :: Metadata Drivers:: Etcd - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - io.etcd - jetcd-core - ${etcd.version} - - - io.grpc - * - - - - - - io.grpc - grpc-all - ${grpc.version} - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - - org.arquillian.cube - arquillian-cube-docker - ${arquillian-cube.version} - - - com.github.docker-java - * - - - test - - - org.jboss.arquillian.junit - arquillian-junit-standalone - ${arquillian-junit.version} - - - com.github.docker-java - * - - - test - - - org.testcontainers - testcontainers - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/metadata-drivers/pom.xml b/metadata-drivers/pom.xml deleted file mode 100644 index e44e116edeb..00000000000 --- a/metadata-drivers/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.metadata.drivers - metadata-drivers-parent - pom - Apache BookKeeper :: Metadata Drivers :: Parent - - etcd - - diff --git a/microbenchmarks/pom.xml b/microbenchmarks/pom.xml deleted file mode 100644 index dc0093954db..00000000000 --- a/microbenchmarks/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - microbenchmarks - Apache BookKeeper :: microbenchmarks - http://maven.apache.org - - benchmarks - - - - org.openjdk.jmh - jmh-core - - - org.openjdk.jmh - jmh-generator-annprocess - provided - - - org.slf4j - slf4j-api - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - compile - jar - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - org.apache.bookkeeper.stats - codahale-metrics-provider - ${project.version} - - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - ${javac.target} - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - ${uberjar.name} - - - org.openjdk.jmh.Main - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - - diff --git a/pom.xml b/pom.xml deleted file mode 100644 index ac7658821b0..00000000000 --- a/pom.xml +++ /dev/null @@ -1,1164 +0,0 @@ - - - - - org.apache - apache - 19 - - 4.0.0 - org.apache.bookkeeper - 4.15.0-SNAPSHOT - bookkeeper - pom - Apache BookKeeper :: Parent - http://bookkeeeper.apache.org - 2011 - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - scm:git:https://github.com/apache/bookkeeper.git - scm:git:https://github.com/apache/bookkeeper.git - https://github.com/apache/bookkeeper - branch-4.13 - - - JIRA - https://issues.apache.org/jira/browse/BOOKKEEPER - - - Jenkins - https://builds.apache.org/job/bookkeeper-master - - - buildtools - circe-checksum - bookkeeper-common - bookkeeper-common-allocator - stats - - bookkeeper-stats - bookkeeper-proto - bookkeeper-server - bookkeeper-benchmark - bookkeeper-stats-providers - bookkeeper-http - stream - tools - cpu-affinity - metadata-drivers - bookkeeper-dist - shaded - microbenchmarks - tests - - - - BookKeeper User - user-subscribe@bookkeeper.apache.org - user-unsubscribe@bookkeeper.apache.org - user@bookkeeper.apache.org - http://www.mail-archive.com/user@bookkeeper.apache.org - - - BookKeeper Dev - dev-subscribe@bookkeeper.apache.org - dev-unsubscribe@bookkeeper.apache.org - dev@bookkeeper.apache.org - http://www.mail-archive.com/dev@bookkeeper.apache.org - - - BookKeeper Commits - commits-subscribe@bookkeeper.apache.org - commits-unsubscribe@bookkeeper.apache.org - commits@bookkeeper.apache.org - http://www.mail-archive.com/commits@bookkeeper.apache.org - - - - - The Apache BookKeeper Team - dev@bookkeeper.apache.org - http://bookkeeper.apache.org - Apache Software Foundation - http://www.apache.org - - - - UTF-8 - UTF-8 - 1.8 - true - 2 - - - 1.18.2 - 1.6.0.Final - 3.0.1 - 1.2 - 4.1 - 1.6 - 1.10 - 1.19 - 2.6 - 3.6 - 2.7 - 1.0.2.1 - 5.1.0 - 3.2.5 - 0.5.11 - 2.7.0 - 3.0.2 - 2.4.0 - 1.42.1 - 30.0-jre - 1.1.1 - 2.10.0 - 1.3 - 2.1.10 - 2.11.0 - 1.78 - 9.4.43.v20210629 - 1.19 - 2.8.2 - 3.2.7 - 4.12 - 0.14.2 - 1.18.20 - 2.17.1 - 1.3.0 - 3.0.0 - 4.1.72.Final - 2.0.46.Final - 9.1.3 - 2.0.2 - 0.8.1 - 0.8.3 - 4.5.13 - 3.14.0 - 3.14.0 - ${grpc.version} - 0.9.11 - 6.22.1.1 - 3.0.1 - 1.7.32 - 1.19 - 3.1.8 - 1.3.2 - 1.15.1 - 3.9.8 - 3.6.2 - 1.1.7 - 2.1.2 - - 0.12 - 2.7 - 4.3.0 - 1.4.13 - 1.6.0 - 1.6 - 0.8.0 - 1.8 - 3.1.0 - 3.0.0 - 2.5 - 3.8.1 - 3.0.2 - 2.7 - 2.5.1 - 2.4 - 3.1.1 - 3.2.0 - 2.2.1 - 3.0.0-M5 - 3.5.2 - 1.4.1.Final - 0.6.1 - 6.19 - 3.1.8 - 1 - 4.0.0 - - - - - - - - com.github.spotbugs - spotbugs-annotations - ${spotbugs-annotations.version} - - - javax.annotation - javax.annotation-api - ${javax-annotations-api.version} - - - com.google.code.findbugs - jsr305 - ${google.code.version} - - - com.google.errorprone - error_prone_annotations - ${google.errorprone.version} - - - org.projectlombok - lombok - ${lombok.version} - - - org.inferred - freebuilder - ${freebuilder.version} - true - - - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - - - - commons-cli - commons-cli - ${commons-cli.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-configuration - commons-configuration - ${commons-configuration.version} - - - commons-io - commons-io - ${commons-io.version} - - - commons-lang - commons-lang - ${commons-lang.version} - - - com.google.guava - guava - ${guava.version} - - - org.apache.commons - commons-collections4 - ${commons-collections4.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - - org.bouncycastle - bc-fips - ${bouncycastle.version} - - - - org.reflections - reflections - ${reflections.version} - - - - - net.jpountz.lz4 - lz4 - ${lz4.version} - - - - - net.java.dev.jna - jna - ${jna.version} - - - - - org.yaml - snakeyaml - ${snakeyaml.version} - - - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - javax.servlet - javax.servlet-api - ${servlet-api.version} - - - com.fasterxml.jackson.module - jackson-module-paranamer - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-scala_2.11 - ${jackson.version} - - - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - - - org.apache.thrift - libthrift - ${libthrift.version} - - - org.apache.tomcat.embed - tomcat-embed-core - - - javax.annotation - javax.annotation-api - - - - - - - io.netty - netty-common - ${netty.version} - - - io.netty - netty-buffer - ${netty.version} - - - io.netty - netty-transport - ${netty.version} - - - io.netty - netty-handler - ${netty.version} - - - io.netty - netty-transport-native-epoll - ${netty.version} - - - io.netty - netty-transport-native-epoll - ${netty.version} - linux-x86_64 - - - io.netty - netty-codec-dns - ${netty.version} - - - io.netty - netty-codec-http - ${netty.version} - - - io.netty - netty-codec-http2 - ${netty.version} - - - io.netty - netty-codec-socks - ${netty.version} - - - io.netty - netty-handler-proxy - ${netty.version} - - - io.netty - netty-resolver - ${netty.version} - - - io.netty - netty-resolver-dns - ${netty.version} - - - io.netty - netty-tcnative-boringssl-static - ${netty-boringssl.version} - - - - - io.grpc - grpc-bom - ${grpc.version} - pom - import - - - - - org.rocksdb - rocksdbjni - ${rocksdb.version} - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.zookeeper - zookeeper - - - - - - - io.vertx - vertx-core - ${vertx.version} - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - - - io.vertx - vertx-web - ${vertx.version} - - - org.eclipse.jetty - jetty-server - ${jetty.version} - - - org.eclipse.jetty - jetty-webapp - ${jetty.version} - - - org.eclipse.jetty - jetty-servlet - ${jetty.version} - - - - - org.jctools - jctools-core - ${jctools.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - io.dropwizard.metrics - metrics-jvm - ${dropwizard.version} - - - io.dropwizard.metrics - metrics-graphite - ${dropwizard.version} - - - - io.prometheus - simpleclient - ${prometheus.version} - - - io.prometheus - simpleclient_hotspot - ${prometheus.version} - - - io.prometheus - simpleclient_servlet - ${prometheus.version} - - - - com.yahoo.datasketches - sketches-core - ${datasketches.version} - - - - - org.apache.httpcomponents - httpclient - ${httpclient.version} - - - - - com.beust - jcommander - ${jcommander.version} - - - - - org.hdrhistogram - HdrHistogram - ${hdrhistogram.version} - - - - - junit - junit - ${junit.version} - - - org.hamcrest - hamcrest-all - ${hamcrest.version} - - - org.jmock - jmock - ${jmock.version} - - - org.mockito - mockito-core - ${mockito.version} - - - org.powermock - powermock-api-mockito2 - ${powermock.version} - - - org.powermock - powermock-module-junit4 - ${powermock.version} - - - org.apache.hadoop - hadoop-minikdc - ${hadoop.minikdc.version} - - - org.arquillian.cube - arquillian-cube-docker - ${arquillian-cube.version} - - - com.github.docker-java - * - - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - ${arquillian-junit.version} - - - com.github.docker-java - * - - - - - org.codehaus.groovy - groovy-all - ${groovy.version} - pom - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-impl-maven - ${shrinkwrap.version} - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-api - ${shrinkwrap.version} - - - org.testcontainers - testcontainers - ${testcontainers.version} - - - - - org.openjdk.jmh - jmh-core - ${jmh.version} - - - org.openjdk.jmh - jmh-generator-annprocess - ${jmh.version} - - - - - - - - - org.projectlombok - lombok - provided - - - com.github.spotbugs - spotbugs-annotations - provided - - - - - org.slf4j - slf4j-api - - - commons-configuration - commons-configuration - - - - - junit - junit - test - - - org.hamcrest - hamcrest-all - test - - - org.apache.logging.log4j - log4j-core - test - - - org.apache.logging.log4j - log4j-slf4j-impl - test - - - org.mockito - mockito-core - test - - - org.powermock - powermock-api-mockito2 - test - - - org.powermock - powermock-module-junit4 - test - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - ${maven-checkstyle-plugin.version} - - - com.puppycrawl.tools - checkstyle - ${puppycrawl.checkstyle.version} - - - - buildtools/src/main/resources/bookkeeper/checkstyle.xml - buildtools/src/main/resources/bookkeeper/suppressions.xml - UTF-8 - true - true - false - true - - - - checkstyle - validate - - check - - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${session.executionRootDirectory}/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Werror - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid - ${redirectTestOutputToFile} - ${forkCount.variable} - false - false - 1800 - ${testRetryCount} - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - -notimestamp - - none - org.apache.bookkeeper.client:org.apache.bookkeeper.client.api:org.apache.bookkeeper.common.annotation:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature:org.apache.bookkeeper.stats - - - Bookkeeper Client - org.apache.bookkeeper.client:org.apache.bookkeeper.common.annotation:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature - - - Bookkeeper Client (New Fluent API - Experimental) - org.apache.bookkeeper.client.api - - - Bookkeeper Stats API - - org.apache.bookkeeper.stats - - - Bookkeeper Stats Providers - org.apache.bookkeeper.stats.codahale:org.apache.bookkeeper.stats.prometheus - - - BookKeeper Java API (version ${project.version}) - site/_site/overview/index.html - package - false - - - - aggregate - - aggregate - - site - - - - - org.apache.maven.plugins - maven-source-plugin - ${maven-source-plugin.version} - - - attach-sources - - jar - - - - - - org.apache.rat - apache-rat-plugin - ${apache-rat-plugin.version} - - - - **/.idea/** - - - .git/**/* - .github/**/* - **/.gitignore - - - **/.svn/**/* - - - **/target/**/* - - - **/README.md - **/README.rst - **/apidocs/* - **/src/main/resources/deps/** - **/META-INF/** - - - **/.classpath - **/.project - **/.checkstyle - **/.settings/* - **/*.iml - **/*.iws - **/*.ipr - - - .repository/** - - - site/** - site2/** - - - **/org/apache/distributedlog/thrift/* - - - **/*.log - - - data/** - - - dev/.vagrant/** - - - **/proto/**.py - - **/python/.coverage - **/python/.Python - **/python/bin/** - **/python/include/** - **/python/lib/** - **/**.pyc - **/.nox/** - **/.pytest_cache/** - **/__pycache__/** - **/bookkeeper.egg-info/** - **/pip-selfcheck.json - - - **/test_conf_2.conf - - true - - - - - - - - code-coverage - - - - org.eluder.coveralls - coveralls-maven-plugin - ${coveralls-maven-plugin.version} - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - @{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true - ${redirectTestOutputToFile} - ${forkCount.variable} - false - 1800 - - true - - - - org.jacoco - jacoco-maven-plugin - 0.8.0 - - - - prepare-agent - - - - - - - - - - - dev - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true - false - ${forkCount.variable} - false - 1800 - false - 0 - - - - - - - dev-debug - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true -Dbookkeeper.root.logger=DEBUG,CONSOLE - false - ${forkCount.variable} - false - 1800 - false - - - - - - - - jdk11-no-spotbugs - - [11,) - - - - - com.github.spotbugs - spotbugs-maven-plugin - - true - - - - - - - - apache-release - - - - maven-assembly-plugin - - - source-release-assembly - - - true - - - - - - - - - diff --git a/settings.gradle b/settings.gradle index 3c72c6e5f80..3d192cfd755 100644 --- a/settings.gradle +++ b/settings.gradle @@ -30,6 +30,8 @@ pluginManagement { rootProject.name = 'bookkeeper' + + include(':bookkeeper-benchmark', ':bookkeeper-tools', ':bookkeeper-tools-framework', diff --git a/shaded/bookkeeper-server-shaded/pom.xml b/shaded/bookkeeper-server-shaded/pom.xml deleted file mode 100644 index 0644f91bcf9..00000000000 --- a/shaded/bookkeeper-server-shaded/pom.xml +++ /dev/null @@ -1,128 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-shaded - Apache BookKeeper :: Shaded :: bookkeeper-server-shaded - - UTF-8 - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - com.google.guava:guava - com.google.protobuf:protobuf-java - org.apache.bookkeeper:bookkeeper-common - org.apache.bookkeeper:bookkeeper-common-allocator - org.apache.bookkeeper:cpu-affinity - org.apache.bookkeeper:bookkeeper-tools-framework - org.apache.bookkeeper:bookkeeper-proto - org.apache.bookkeeper:bookkeeper-server - org.apache.bookkeeper:circe-checksum - org.apache.bookkeeper.stats:bookkeeper-stats-api - - - - - com.google - org.apache.bookkeeper.shaded.com.google - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/bookkeeper-server-tests-shaded/pom.xml b/shaded/bookkeeper-server-tests-shaded/pom.xml deleted file mode 100644 index a5cd9f6a177..00000000000 --- a/shaded/bookkeeper-server-tests-shaded/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-tests-shaded - Apache BookKeeper :: Shaded :: bookkeeper-server-tests-shaded - - UTF-8 - - - - org.apache.bookkeeper - bookkeeper-server - test-jar - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - org.apache.bookkeeper - bookkeeper-common - - - org.apache.bookkeeper - bookkeeper-tools-framework - - - org.apache.bookkeeper - bookkeeper-proto - - - org.apache.bookkeeper - bookkeeper-server - - - org.apache.bookkeeper - circe-checksum - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - com.google.guava:guava - com.google.protobuf:protobuf-java - org.apache.bookkeeper:bookkeeper-server:test-jar:tests - - - - - - com.google - org.apache.bookkeeper.shaded.com.google - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml deleted file mode 100644 index c1cf1fef955..00000000000 --- a/shaded/distributedlog-core-shaded/pom.xml +++ /dev/null @@ -1,251 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - org.apache.distributedlog - distributedlog-core-shaded - Apache BookKeeper :: Shaded :: distributedlog-core-shaded - - UTF-8 - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - io.netty - netty-common - - - io.netty - netty-buffer - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - commons-codec:commons-codec - commons-cli:commons-cli - commons-io:commons-io - commons-lang:commons-lang - commons-logging:commons-logging - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.core:jackson-annotations - com.google.guava:guava - com.google.protobuf:protobuf-java - net.java.dev.jna:jna - net.jpountz.lz4:lz4 - org.apache.bookkeeper:bookkeeper-common - org.apache.bookkeeper:bookkeeper-common-allocator - org.apache.bookkeeper:cpu-affinity - org.apache.bookkeeper:bookkeeper-tools-framework - org.apache.bookkeeper:bookkeeper-proto - org.apache.bookkeeper:bookkeeper-server - org.apache.bookkeeper:circe-checksum - org.apache.bookkeeper.http:http-server - org.apache.bookkeeper.stats:bookkeeper-stats-api - org.apache.commons:commons-collections4 - org.apache.commons:commons-lang3 - org.apache.distributedlog:distributedlog-common - org.apache.distributedlog:distributedlog-core - org.apache.distributedlog:distributedlog-protocol - org.apache.httpcomponents:httpclient - org.apache.httpcomponents:httpcore - org.apache.thrift:libthrift - org.apache.zookeeper:zookeeper - org.apache.zookeeper:zookeeper-jute - org.rocksdb:rocksdbjni - - - - - - org.apache.commons.cli - dlshade.org.apache.commons.cli - - - org.apache.commons.codec - dlshade.org.apache.commons.codec - - - org.apache.commons.collections4 - dlshade.org.apache.commons.collections4 - - - org.apache.commons.lang - dlshade.org.apache.commons.lang - - - org.apache.commons.lang3 - dlshade.org.apache.commons.lang3 - - - org.apache.commons.logging - dlshade.org.apache.commons.logging - - - org.apache.commons.io - dlshade.org.apache.commons.io - - - - org.apache.httpcomponents - dlshade.org.apache.httpcomponents - - - org.apache.http - dlshade.org.apache.http - - - - org.apache.thrift - dlshade.org.apache.thrift - - - - org.apache.zookeeper - dlshade.org.apache.zookeeper - - - org.apache.jute - dlshade.org.apache.jute - - - - com.fasterxml.jackson - dlshade.com.fasterxml.jackson - - - - com.sun.jna - dlshade.com.sun.jna - - - - com.google - dlshade.com.google - - - - org.jboss.netty - dlshade.org.jboss.netty - - - - net.jpountz - dlshade.net.jpountz - - - - org.rocksdb - dlshade.org.rocksdb - - - - com.scurrilous.circe - dlshade.com.scurrilous.circe - - - org.apache.bookkeeper - dlshade.org.apache.bookkeeper - - - - org.apache.distributedlog - org.apache.distributedlog - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/pom.xml b/shaded/pom.xml deleted file mode 100644 index 254de71d961..00000000000 --- a/shaded/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper - shaded-parent - Apache BookKeeper :: Shaded :: Parent - - bookkeeper-server-shaded - bookkeeper-server-tests-shaded - distributedlog-core-shaded - - diff --git a/stats/pom.xml b/stats/pom.xml deleted file mode 100644 index d2c89aa671f..00000000000 --- a/stats/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - pom - org.apache.bookkeeper.stats - bookkeeper-stats-parent - Apache BookKeeper :: Stats :: Parent - - - utils - - - - diff --git a/stats/utils/pom.xml b/stats/utils/pom.xml deleted file mode 100644 index cdfebcd08cc..00000000000 --- a/stats/utils/pom.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - 4.0.0 - - bookkeeper-stats-parent - org.apache.bookkeeper.stats - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.stats - bookkeeper-stats-utils - Apache BookKeeper :: Stats :: Utils - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - org.reflections - reflections - - - org.yaml - snakeyaml - - - com.beust - jcommander - - - com.fasterxml.jackson.core - jackson-annotations - - - diff --git a/stream/api/pom.xml b/stream/api/pom.xml deleted file mode 100644 index 82906fdb484..00000000000 --- a/stream/api/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-api - Apache BookKeeper :: Stream Storage :: API - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - io.netty - netty-buffer - - - org.inferred - freebuilder - - - diff --git a/stream/bin/streamstorage b/stream/bin/streamstorage index a85c6e70d32..1f41b2258ac 100755 --- a/stream/bin/streamstorage +++ b/stream/bin/streamstorage @@ -20,6 +20,7 @@ BINDIR=$(dirname "$0") SS_HOME=`cd $BINDIR/..;pwd` +BK_HOME=${BK_HOME:-"`cd ${BINDIR}/../..;pwd`"} DEFAULT_STANDALONE_CONF=$SS_HOME/conf/standalone.conf DEFAULT_LOG_CONF=$SS_HOME/conf/log4j.properties @@ -79,18 +80,14 @@ EOF } add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${SS_HOME}/server/target/classpath.txt" + f="${SS_HOME}/server/build/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${SS_HOME}/server/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + echo "no classpath.txt found at ${SS_HOME}/server/build" + exit 1 fi SS_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/stream/bin/streamstorage-cli b/stream/bin/streamstorage-cli index 6a1df7810d8..365c50dcf16 100755 --- a/stream/bin/streamstorage-cli +++ b/stream/bin/streamstorage-cli @@ -57,18 +57,14 @@ elif [ -e "$BUILT_JAR" ]; then fi add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${SS_HOME}/cli/target/classpath.txt" + f="${SS_HOME}/cli/build/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${SS_HOME}/cli/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + echo "no classpath.txt found at ${SS_HOME}/cli/build" + exit 1 fi SS_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/stream/bk-grpc-name-resolver/pom.xml b/stream/bk-grpc-name-resolver/pom.xml deleted file mode 100644 index 37feb436205..00000000000 --- a/stream/bk-grpc-name-resolver/pom.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - bk-grpc-name-resolver - Apache BookKeeper :: Stream Storage :: Common :: BK Grpc Name Resolver - - - org.apache.bookkeeper - stream-storage-common - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.version} - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - diff --git a/stream/clients/java/all/pom.xml b/stream/clients/java/all/pom.xml deleted file mode 100644 index 1aa66fd526a..00000000000 --- a/stream/clients/java/all/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-client - Apache BookKeeper :: Stream Storage :: Clients :: Java Client - - - - org.apache.bookkeeper - stream-storage-java-kv-client - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - org.apache.bookkeeper:stream-storage-* - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - - ${project.basedir} - - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/stream/clients/java/base/pom.xml b/stream/clients/java/base/pom.xml deleted file mode 100644 index c8d8388420b..00000000000 --- a/stream/clients/java/base/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-client-base - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: Base - - - - org.apache.bookkeeper - stream-storage-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/clients/java/kv/pom.xml b/stream/clients/java/kv/pom.xml deleted file mode 100644 index d9c9e359324..00000000000 --- a/stream/clients/java/kv/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-kv-client - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: KV - - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - diff --git a/stream/clients/java/pom.xml b/stream/clients/java/pom.xml deleted file mode 100644 index 44a462220e9..00000000000 --- a/stream/clients/java/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-clients-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-java-client-parent - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: Parent - - base - kv - all - - diff --git a/stream/clients/pom.xml b/stream/clients/pom.xml deleted file mode 100644 index 3001eeb8d90..00000000000 --- a/stream/clients/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-clients-parent - Apache BookKeeper :: Stream Storage :: Clients :: Parent - - java - - diff --git a/stream/common/pom.xml b/stream/common/pom.xml deleted file mode 100644 index 6f5091ad933..00000000000 --- a/stream/common/pom.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-common - Apache BookKeeper :: Stream Storage :: Common - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - io.netty - netty-buffer - - - io.grpc - grpc-all - - - io.grpc - grpc-netty-shaded - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - javax.annotation - javax.annotation-api - - true - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - test - - - diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml deleted file mode 100644 index f91055fa99d..00000000000 --- a/stream/distributedlog/common/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-common - Apache BookKeeper :: DistributedLog :: Common - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - org.slf4j - slf4j-log4j12 - - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.commons - commons-lang3 - - - com.google.guava - guava - - - commons-lang - commons-lang - - - commons-codec - commons-codec - - - io.netty - netty-buffer - - - net.jpountz.lz4 - lz4 - - - org.jmock - jmock - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G - always - 1800 - - - - - diff --git a/stream/distributedlog/core/build.gradle b/stream/distributedlog/core/build.gradle index ee9a79fc10a..d8f88a366fc 100644 --- a/stream/distributedlog/core/build.gradle +++ b/stream/distributedlog/core/build.gradle @@ -75,6 +75,7 @@ test.doFirst { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'distributedlog-core' } diff --git a/stream/distributedlog/core/pom.xml b/stream/distributedlog/core/pom.xml deleted file mode 100644 index 78c6ff62b28..00000000000 --- a/stream/distributedlog/core/pom.xml +++ /dev/null @@ -1,135 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-core - Apache BookKeeper :: DistributedLog :: Core Library - - - org.apache.distributedlog - distributedlog-protocol - ${project.parent.version} - - - org.apache.zookeeper - zookeeper - - - org.apache.thrift - libthrift - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - - - javax.annotation - javax.annotation-api - - true - - - org.jmock - jmock - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.distributedlog - distributedlog-common - ${project.parent.version} - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - false - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G - always - 1800 - - - listener - org.apache.bookkeeper.common.testing.util.TimedOutTestsListener - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - ../../buildtools/src/main/resources/distributedlog/suppressions.xml - ../../buildtools/src/main/resources/bookkeeper/checkstyle.xml - **/thrift/**/* - - - - - diff --git a/stream/distributedlog/io/dlfs/pom.xml b/stream/distributedlog/io/dlfs/pom.xml deleted file mode 100644 index 31366ed9fc0..00000000000 --- a/stream/distributedlog/io/dlfs/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - 4.0.0 - - distributedlog - org.apache.distributedlog - 4.15.0-SNAPSHOT - ../.. - - org.apache.distributedlog - dlfs - Apache BookKeeper :: DistributedLog :: IO :: FileSystem - http://maven.apache.org - - UTF-8 - ${basedir}/lib - - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/stream/distributedlog/io/pom.xml b/stream/distributedlog/io/pom.xml deleted file mode 100644 index 38572cb2e0b..00000000000 --- a/stream/distributedlog/io/pom.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - 4.0.0 - distributedlog-io - pom - Apache BookKeeper :: DistributedLog :: IO - - dlfs - - diff --git a/stream/distributedlog/pom.xml b/stream/distributedlog/pom.xml deleted file mode 100644 index 2f77d0fc025..00000000000 --- a/stream/distributedlog/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.distributedlog - distributedlog - pom - Apache BookKeeper :: DistributedLog :: Parent - - Apache DistributedLog provides a high performance replicated log service. - - 2016 - - common - protocol - core - io - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - -notimestamp - none - - - Core Library - org.apache.distributedlog:org.apache.distributedlog.annotations:org.apache.distributedlog.callback:org.apache.distributedlog.exceptions:org.apache.distributedlog.feature:org.apache.distributedlog.io:org.apache.distributedlog.lock:org.apache.distributedlog.logsegment:org.apache.distributedlog.metadata:org.apache.distributedlog.namespace:org.apache.distributedlog.net:org.apache.distributedlog.stats:org.apache.distributedlog.api.subscription - - - - org.apache.distributedlog.acl:org.apache.distributedlog.admin:org.apache.distributedlog.auditor:org.apache.distributedlog.basic:org.apache.distributedlog.benchmark*:org.apache.distributedlog.bk:org.apache.distributedlog.ownership:org.apache.distributedlog.proxy:org.apache.distributedlog.resolver:org.apache.distributedlog.service.*:org.apache.distributedlog.config:org.apache.distributedlog.function:org.apache.distributedlog.impl*:org.apache.distributedlog.injector:org.apache.distributedlog.kafka:org.apache.distributedlog.limiter:org.apache.distributedlog.mapreduce:org.apache.distributedlog.messaging:org.apache.distributedlog.common.rate:org.apache.distributedlog.readahead:org.apache.distributedlog.selector:org.apache.distributedlog.stats:org.apache.distributedlog.thrift*:org.apache.distributedlog.tools:org.apache.distributedlog.util:org.apache.distributedlog.zk:org.apache.bookkeeper.client:org.apache.bookkeeper.stats - - - - - aggregate - - aggregate - - site - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID - always - 1800 - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${session.executionRootDirectory}/buildtools/src/main/resources/distributedlog/findbugsExclude.xml - - - - - - diff --git a/stream/distributedlog/protocol/pom.xml b/stream/distributedlog/protocol/pom.xml deleted file mode 100644 index 727a8009c21..00000000000 --- a/stream/distributedlog/protocol/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-protocol - Apache BookKeeper :: DistributedLog :: Protocol - - - org.apache.distributedlog - distributedlog-common - ${project.version} - - - io.netty - netty-buffer - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/stream/pom.xml b/stream/pom.xml deleted file mode 100644 index d1b01cec7e0..00000000000 --- a/stream/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - pom - org.apache.bookkeeper - stream-storage-parent - Apache BookKeeper :: Stream Storage :: Parent - - - distributedlog - common - tests-common - statelib - api - proto - clients - storage - server - bk-grpc-name-resolver - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - true - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - true - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID - always - 1800 - - - - - - - - streamTests - - - streamTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - - - diff --git a/stream/proto/pom.xml b/stream/proto/pom.xml deleted file mode 100644 index 7827096cb49..00000000000 --- a/stream/proto/pom.xml +++ /dev/null @@ -1,115 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-proto - Apache BookKeeper :: Stream Storage :: Proto - - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - org.apache.commons - commons-lang3 - - - com.google.protobuf - protobuf-java - - - javax.annotation - javax.annotation-api - - true - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - test - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Xlint:unchecked - - -Xpkginfo:always - - false - false - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - true - - - - - compile - compile-custom - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/server/build.gradle b/stream/server/build.gradle index 19faa189683..d8cba32767f 100644 --- a/stream/server/build.gradle +++ b/stream/server/build.gradle @@ -74,12 +74,8 @@ application { mainClassName = "org.apache.bookkeeper.stream.cluster.StandaloneStarter" } -task writeClasspath { - buildDir.mkdirs() - new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" -} - jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'stream-storage-server' } diff --git a/stream/server/pom.xml b/stream/server/pom.xml deleted file mode 100644 index 386b611be91..00000000000 --- a/stream/server/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - - stream-storage-server - Apache BookKeeper :: Stream Storage :: Server - - - - org.apache.bookkeeper - stream-storage-service-impl - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client - ${project.parent.version} - - - com.beust - jcommander - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - runtime - - - - org.xerial.snappy - snappy-java - runtime - - - - io.dropwizard.metrics - metrics-core - runtime - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/statelib/pom.xml b/stream/statelib/pom.xml deleted file mode 100644 index d2466182dad..00000000000 --- a/stream/statelib/pom.xml +++ /dev/null @@ -1,169 +0,0 @@ - - - - 4.0.0 - - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - statelib - Apache BookKeeper :: Stream Storage :: State Library - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - com.google.guava - guava - - - org.rocksdb - rocksdbjni - - - com.google.protobuf - protobuf-java - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - - - - - compile - - - - - - - diff --git a/stream/storage/api/pom.xml b/stream/storage/api/pom.xml deleted file mode 100644 index bc415e3e28b..00000000000 --- a/stream/storage/api/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-service-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-api - Apache BookKeeper :: Stream Storage :: Storage :: Api - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/storage/impl/pom.xml b/stream/storage/impl/pom.xml deleted file mode 100644 index 7b2ff7d5672..00000000000 --- a/stream/storage/impl/pom.xml +++ /dev/null @@ -1,160 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-service-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-impl - Apache BookKeeper :: Stream Storage :: Storage :: Impl - - - - org.apache.bookkeeper - stream-storage-service-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - - - org.apache.bookkeeper - statelib - ${project.parent.version} - - - org.apache.curator - curator-recipes - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - tests - test - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/storage/pom.xml b/stream/storage/pom.xml deleted file mode 100644 index 22e3eaf395d..00000000000 --- a/stream/storage/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-parent - Apache BookKeeper :: Stream Storage :: Storage :: Parent - - api - impl - - diff --git a/stream/tests-common/pom.xml b/stream/tests-common/pom.xml deleted file mode 100644 index c14fde018ae..00000000000 --- a/stream/tests-common/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.tests - stream-storage-tests-common - Apache BookKeeper :: Stream Storage :: Common Classes for Tests - - - - io.grpc - grpc-all - - - io.grpc - grpc-netty-shaded - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - com.google.protobuf - protobuf-java - - - javax.annotation - javax.annotation-api - - true - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Xlint:unchecked - - -Xpkginfo:always - - false - false - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - true - - - - - compile - compile-custom - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/tests/backward-compat/bc-non-fips/pom.xml b/tests/backward-compat/bc-non-fips/pom.xml deleted file mode 100644 index eba5b1d7219..00000000000 --- a/tests/backward-compat/bc-non-fips/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - bc-non-fips - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test Bouncy Castle Provider load non FIPS version - - 1.68 - - - - - junit - junit - ${junit.version} - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.bouncycastle - * - - - test - - - - org.bouncycastle - bcpkix-jdk15on - ${bc-non-fips.version} - - - - org.bouncycastle - bcprov-ext-jdk15on - ${bc-non-fips.version} - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - - diff --git a/tests/backward-compat/current-server-old-clients/pom.xml b/tests/backward-compat/current-server-old-clients/pom.xml deleted file mode 100644 index 2379977810d..00000000000 --- a/tests/backward-compat/current-server-old-clients/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - backward-compat-current-server-old-clients - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test old clients working on current server - - diff --git a/tests/backward-compat/hierarchical-ledger-manager/pom.xml b/tests/backward-compat/hierarchical-ledger-manager/pom.xml deleted file mode 100644 index a75736839c2..00000000000 --- a/tests/backward-compat/hierarchical-ledger-manager/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - hierarchical-ledger-manager - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test compat between old version and new version of hierarchical ledger manager - - diff --git a/tests/backward-compat/hostname-bookieid/pom.xml b/tests/backward-compat/hostname-bookieid/pom.xml deleted file mode 100644 index 28d64fce9c7..00000000000 --- a/tests/backward-compat/hostname-bookieid/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - hostname-bookieid - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between 4.1.0 and current version (with hostname bookie ID) - - diff --git a/tests/backward-compat/old-cookie-new-cluster/pom.xml b/tests/backward-compat/old-cookie-new-cluster/pom.xml deleted file mode 100644 index c7705c999cd..00000000000 --- a/tests/backward-compat/old-cookie-new-cluster/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - old-cookie-new-cluster - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade 4.1.0 to current in cluster with cookies - - diff --git a/tests/backward-compat/pom.xml b/tests/backward-compat/pom.xml deleted file mode 100644 index 396840bb8f4..00000000000 --- a/tests/backward-compat/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - integration-tests-base-groovy - 4.15.0-SNAPSHOT - ../integration-tests-base-groovy - - org.apache.bookkeeper.tests - backward-compat - Apache BookKeeper :: Tests :: Backward Compatibility - - upgrade - upgrade-direct - hierarchical-ledger-manager - hostname-bookieid - recovery-no-password - old-cookie-new-cluster - current-server-old-clients - yahoo-custom-version - bc-non-fips - - diff --git a/tests/backward-compat/recovery-no-password/pom.xml b/tests/backward-compat/recovery-no-password/pom.xml deleted file mode 100644 index 8eb53fc7f07..00000000000 --- a/tests/backward-compat/recovery-no-password/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - recovery-no-password - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test recovery does not work when password no in metadata - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - diff --git a/tests/backward-compat/upgrade-direct/pom.xml b/tests/backward-compat/upgrade-direct/pom.xml deleted file mode 100644 index c03d5686c38..00000000000 --- a/tests/backward-compat/upgrade-direct/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - upgrade-direct - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between 4.1.0 and current version - - diff --git a/tests/backward-compat/upgrade/pom.xml b/tests/backward-compat/upgrade/pom.xml deleted file mode 100644 index e8377df4e68..00000000000 --- a/tests/backward-compat/upgrade/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - upgrade - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between all released versions and current version - - diff --git a/tests/backward-compat/yahoo-custom-version/pom.xml b/tests/backward-compat/yahoo-custom-version/pom.xml deleted file mode 100644 index 6061316a0e7..00000000000 --- a/tests/backward-compat/yahoo-custom-version/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - yahoo-custom-version - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between yahoo custom version and current - - diff --git a/tests/docker-images/all-released-versions-image/pom.xml b/tests/docker-images/all-released-versions-image/pom.xml deleted file mode 100644 index 7abcbb03f75..00000000000 --- a/tests/docker-images/all-released-versions-image/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - all-released-versions-image - Apache BookKeeper :: Tests :: Docker Images :: All Released Versions - pom - - - docker - - - integrationTests - - - - - - com.spotify - dockerfile-maven-plugin - 1.4.13 - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-all-released-versions - latest - - - - - apachebookkeeper/bookkeeper-all-released-versions - ${project.version} - false - - - - - - - diff --git a/tests/docker-images/all-versions-image/pom.xml b/tests/docker-images/all-versions-image/pom.xml deleted file mode 100644 index 1eb92bb6d5c..00000000000 --- a/tests/docker-images/all-versions-image/pom.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - all-versions-image - Apache BookKeeper :: Tests :: Docker Images :: All Versions - pom - - - org.apache.bookkeeper.tests - all-released-versions-image - ${project.parent.version} - pom - - - - org.apache.bookkeeper - bookkeeper-dist-server - ${project.parent.version} - bin - tar.gz - provided - - - - - docker - - - integrationTests - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile-maven-plugin.version} - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-all-versions - latest - - - - - apachebookkeeper/bookkeeper-all-versions - ${project.version} - false - true - - target/bookkeeper-dist-server-${project.version}-bin.tar.gz - - - - - org.apache.maven.plugins - maven-dependency-plugin - ${maven-dependency-plugin.version} - - - copy-tarball - - copy-dependencies - - generate-resources - - ${project.build.directory}/ - bookkeeper-dist-server - true - - - - - - - - - diff --git a/tests/docker-images/current-version-image/pom.xml b/tests/docker-images/current-version-image/pom.xml deleted file mode 100644 index 5e221904e3a..00000000000 --- a/tests/docker-images/current-version-image/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - current-version-image - Apache BookKeeper :: Tests :: Docker Images :: Current Version - pom - - - org.apache.bookkeeper - bookkeeper-dist-server - ${project.parent.version} - bin - tar.gz - provided - - - - - docker - - - integrationTests - - - - - - - org.codehaus.mojo - exec-maven-plugin - ${exec-maven-plugin.version} - - - build-python-client - generate-resources - - exec - - - ${project.basedir}/target - ${project.basedir}/../../../stream/clients/python/scripts/docker_build.sh - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - generate-resources - - run - - - - copy python wheel file - - - copying docker scripts - - - - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile-maven-plugin.version} - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-current - latest - - - - - apachebookkeeper/bookkeeper-current - ${project.version} - false - true - - ${project.version} - - - - - org.apache.maven.plugins - maven-dependency-plugin - ${maven-dependency-plugin.version} - - - copy-docker-dependencies - - copy-dependencies - - generate-resources - - ${project.build.directory}/ - bookkeeper-dist-server - true - - - - - - - - - diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml deleted file mode 100644 index a337e91176a..00000000000 --- a/tests/docker-images/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - docker-images - Apache BookKeeper :: Tests :: Docker Images - - all-released-versions-image - all-versions-image - current-version-image - - diff --git a/tests/integration-tests-base-groovy/pom.xml b/tests/integration-tests-base-groovy/pom.xml deleted file mode 100644 index ccaa5ea4308..00000000000 --- a/tests/integration-tests-base-groovy/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration-tests-base - 4.15.0-SNAPSHOT - ../integration-tests-base - - - org.apache.bookkeeper.tests - integration-tests-base-groovy - pom - - Apache BookKeeper :: Tests :: Base module for Arquillian based integration tests using groovy - - - 3.6.0-03 - 3.0.2-02 - - - - - - maven-compiler-plugin - - - ${javac.target} - ${javac.target} - groovy-eclipse-compiler - - - - org.codehaus.groovy - groovy-eclipse-compiler - ${groovy-eclipse-compiler.version} - - - org.codehaus.groovy - groovy-eclipse-batch - ${groovy-eclipse-batch.version} - - - - - org.codehaus.groovy - groovy-eclipse-compiler - ${groovy-eclipse-compiler.version} - true - - - org.codehaus.gmaven - groovy-maven-plugin - 2.0 - - - org.codehaus.groovy - groovy-all - ${groovy.version} - pom - - - - - org.apache.maven.plugins - maven-surefire-plugin - - 2.8.1 - - -Xmx4G -Djava.net.preferIPv4Stack=true - 1 - false - - - System.out - true - - - - - - - - - org.codehaus.groovy - groovy-all - pom - - - - - bintray - - false - - groovy-bintray-plugins - https://dl.bintray.com/groovy/maven - - - diff --git a/tests/integration-tests-base/pom.xml b/tests/integration-tests-base/pom.xml deleted file mode 100644 index b5a0ce218d7..00000000000 --- a/tests/integration-tests-base/pom.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-base - pom - - Apache BookKeeper :: Tests :: Base module for Arquillian based integration tests - - - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - test - - - - junit - junit - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/tests/integration-tests-topologies/pom.xml b/tests/integration-tests-topologies/pom.xml deleted file mode 100644 index b2797b09635..00000000000 --- a/tests/integration-tests-topologies/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-topologies - jar - - Apache BookKeeper :: Tests :: Common topologies for Docker based integration tests - - - - org.testcontainers - testcontainers - - - junit - junit - compile - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - - - - diff --git a/tests/integration-tests-utils/pom.xml b/tests/integration-tests-utils/pom.xml deleted file mode 100644 index 6c43bbc1d41..00000000000 --- a/tests/integration-tests-utils/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-utils - jar - - Apache BookKeeper :: Tests :: Utility module for Arquillian based integration tests - - - - org.apache.commons - commons-compress - - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-impl-maven - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-api - - - - org.apache.zookeeper - zookeeper - - - - org.arquillian.cube - arquillian-cube-docker - - - com.github.docker-java - * - - - - - - org.testcontainers - testcontainers - - - - org.codehaus.groovy - groovy-all - pom - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - 2.8.1 - - 1 - false - - - - - - diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index 56960061def..94cdedfab96 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -47,10 +47,6 @@ public class BookKeeperClusterUtils { private static final List OLD_CLIENT_VERSIONS_WITH_CURRENT_LEDGER_METADATA_FORMAT = Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3"); - private static final List OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME = - Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3", "4.3-yahoo"); - - private static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterUtils.class); public static boolean hasVersionLatestMetadataFormat(String version) { @@ -101,7 +97,7 @@ public static boolean metadataFormatIfNeeded(DockerClient docker, String version @Cleanup ZooKeeper zk = BookKeeperClusterUtils.zookeeperClient(docker); if (zk.exists("/ledgers", false) == null) { - String bookkeeper = "/opt/bookkeeper/" + version + "/bin/" + computeBinFilenameByVersion(version); + String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; runOnAnyBookie(docker, bookkeeper, "shell", "metaformat", "-nonInteractive"); return true; } else { @@ -131,7 +127,7 @@ public static String createDlogNamespaceIfNeeded(DockerClient docker, } public static void formatAllBookies(DockerClient docker, String version) throws Exception { - String bookkeeper = "/opt/bookkeeper/" + version + "/bin/" + computeBinFilenameByVersion(version); + String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; BookKeeperClusterUtils.runOnAllBookies(docker, bookkeeper, "shell", "bookieformat", "-nonInteractive"); } @@ -259,11 +255,4 @@ public static boolean waitAllBookieUp(DockerClient docker) { .map((b) -> waitBookieUp(docker, b, 10, TimeUnit.SECONDS)) .reduce(true, BookKeeperClusterUtils::allTrue); } - - private static String computeBinFilenameByVersion(String version) { - if (OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME.contains(version)) { - return "bookkeeper"; - } - return "bookkeeper_gradle"; - } } diff --git a/tests/integration/cluster/pom.xml b/tests/integration/cluster/pom.xml deleted file mode 100644 index 604aeab3e1f..00000000000 --- a/tests/integration/cluster/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - cluster - jar - Apache BookKeeper :: Tests :: Integration :: Cluster test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml deleted file mode 100644 index 1d7535f59bd..00000000000 --- a/tests/integration/pom.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - integration - Apache BookKeeper :: Tests :: Integration - - smoke - standalone - cluster - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/tests/integration/smoke/pom.xml b/tests/integration/smoke/pom.xml deleted file mode 100644 index 35cc17511b8..00000000000 --- a/tests/integration/smoke/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - smoke - jar - Apache BookKeeper :: Tests :: Integration :: Smoke test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/integration/standalone/pom.xml b/tests/integration/standalone/pom.xml deleted file mode 100644 index 7a9cb5a4272..00000000000 --- a/tests/integration/standalone/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - standalone - jar - Apache BookKeeper :: Tests :: Integration :: Standalone test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/pom.xml b/tests/pom.xml deleted file mode 100644 index ae4f4b8ab3c..00000000000 --- a/tests/pom.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - tests-parent - Apache BookKeeper :: Tests - - - 2.5.8 - - - - shaded - docker-images - integration-tests-base - integration-tests-base-groovy - integration-tests-utils - integration-tests-topologies - backward-compat - integration - scripts - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/tests/scripts/pom.xml b/tests/scripts/pom.xml deleted file mode 100644 index 4bbc9e80021..00000000000 --- a/tests/scripts/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - scripts - jar - Apache BookKeeper :: Tests :: Bash Scripts Test - - - - - - - - com.googlecode.maven-download-plugin - download-maven-plugin - - - install-shunit2 - integration-test - - wget - - - https://github.com/kward/shunit2/archive/v2.1.7.zip - true - ${project.basedir}/target/lib - ${skipTests} - - - - - - org.codehaus.mojo - exec-maven-plugin - - - bash-tests - integration-test - - exec - - - ${skipTests} - ${project.basedir}/src/test/bash - ${project.basedir}/src/test/bash/bk_test.sh - - - - - - - diff --git a/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh b/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh index c20165f6050..9a1f7f3917e 100644 --- a/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh +++ b/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh @@ -27,7 +27,7 @@ # testDefaultVariables() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh assertEquals "BINDIR is not set correctly" "${BK_BINDIR}" "${BINDIR}" assertEquals "BK_HOME is not set correctly" "${BK_HOMEDIR}" "${BK_HOME}" assertEquals "DEFAULT_LOG_CONF is not set correctly" "${BK_CONFDIR}/log4j.properties" "${DEFAULT_LOG_CONF}" @@ -48,7 +48,7 @@ testDefaultVariables() { } testFindModuleJarAt() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh MODULE="test-module" @@ -113,7 +113,7 @@ testFindModuleJar() { echo "" > ${BK_HOME}/conf/bkenv.sh echo "" > ${BK_HOME}/conf/bk_cli_env.sh - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh MODULE="test-module" MODULE_PATH="testmodule" @@ -159,7 +159,7 @@ testLoadEnvfiles() { echo "CLI_MAX_HEAP_MEMORY=2048M" > ${BK_HOME}/conf/bk_cli_env.sh # load the common_gradle.sh - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh assertEquals "NETTY_LEAK_DETECTION_LEVEL is not set correctly" "enabled" "${NETTY_LEAK_DETECTION_LEVEL}" assertEquals "BOOKIE_MAX_HEAP_MEMORY is not set correctly" "2048M" "${BOOKIE_MAX_HEAP_MEMORY}" @@ -172,7 +172,7 @@ testLoadEnvfiles() { } testBuildBookieJVMOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh TEST_LOG_DIR=${BK_TMPDIR}/logdir TEST_GC_LOG_FILENAME="test-gc.log" @@ -187,7 +187,7 @@ testBuildBookieJVMOpts() { } testBuildCLIJVMOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh TEST_LOG_DIR=${BK_TMPDIR}/logdir TEST_GC_LOG_FILENAME="test-gc.log" @@ -202,7 +202,7 @@ testBuildCLIJVMOpts() { } testBuildNettyOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh ACTUAL_NETTY_OPTS=$(build_netty_opts) EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled \ @@ -213,7 +213,7 @@ testBuildNettyOpts() { } testBuildBookieOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh ACTUAL_OPTS=$(build_bookie_opts) EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true" diff --git a/tests/shaded/bookkeeper-server-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-shaded-test/pom.xml deleted file mode 100644 index cc548bc2c6f..00000000000 --- a/tests/shaded/bookkeeper-server-shaded-test/pom.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-shaded-test - Apache BookKeeper :: Tests :: bookkeeper-server-shaded test - - - org.apache.bookkeeper - bookkeeper-server-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - - diff --git a/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml deleted file mode 100644 index 2a8cb5490c3..00000000000 --- a/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-tests-shaded-test - Apache BookKeeper :: Tests :: bookkeeper-server-tests-shaded test - - - org.apache.bookkeeper - bookkeeper-server-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - org.apache.bookkeeper - bookkeeper-server-tests-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/tests/shaded/distributedlog-core-shaded-test/pom.xml b/tests/shaded/distributedlog-core-shaded-test/pom.xml deleted file mode 100644 index 7b2206c4ccf..00000000000 --- a/tests/shaded/distributedlog-core-shaded-test/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - distributedlog-core-shaded-test - Apache BookKeeper :: Tests :: distributedlog-core-shaded test - - - org.apache.distributedlog - distributedlog-core-shaded - ${project.version} - test - - - - org.apache.bookkeeper - bookkeeper-server - - - org.apache.bookkeeper.http - bookkeeper-http - - - org.apache.bookkeeper - circe-checksum - - - org.apache.distributedlog - distributedlog-core - - - org.apache.distributedlog - distributedlog-common - - - org.apache.distributedlog - distributedlog-protocol - - - org.apache.zookeeper - zookeeper - - - org.apache.thrift - libthrift - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/tests/shaded/pom.xml b/tests/shaded/pom.xml deleted file mode 100644 index f86a357115b..00000000000 --- a/tests/shaded/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - Apache BookKeeper :: Tests :: Test Shaded Jars - - bookkeeper-server-shaded-test - bookkeeper-server-tests-shaded-test - distributedlog-core-shaded-test - - diff --git a/tools/all/build.gradle b/tools/all/build.gradle index 7593b6a2c6e..670f2c41a72 100644 --- a/tools/all/build.gradle +++ b/tools/all/build.gradle @@ -36,5 +36,6 @@ dependencies { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-tools' } diff --git a/tools/all/pom.xml b/tools/all/pom.xml deleted file mode 100644 index ac15e6bb12a..00000000000 --- a/tools/all/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools - Apache BookKeeper :: Tools - - - org.apache.bookkeeper - bookkeeper-tools-ledger - ${project.version} - - - org.apache.bookkeeper - stream-storage-cli - ${project.version} - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - append-ledger-commands - generate-resources - - run - - - - - - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - append-stream-commands - generate-resources - - run - - - - - - - - - - - - - - - diff --git a/tools/framework/pom.xml b/tools/framework/pom.xml deleted file mode 100644 index 9f5e86b1ef6..00000000000 --- a/tools/framework/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-framework - Apache BookKeeper :: Tools :: Framework - - - com.beust - jcommander - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - - - org.apache.bookkeeper - buildtools - ${project.parent.version} - test - - - diff --git a/tools/ledger/pom.xml b/tools/ledger/pom.xml deleted file mode 100644 index 8becea1f281..00000000000 --- a/tools/ledger/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-ledger - Apache BookKeeper :: Tools :: Ledger - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - - - org.apache.logging.log4j - log4j-1.2-api - runtime - - - org.apache.logging.log4j - log4j-core - runtime - - - org.apache.logging.log4j - log4j-slf4j-impl - runtime - - - org.apache.bookkeeper - buildtools - ${project.parent.version} - test - - - org.apache.bookkeeper - bookkeeper-server - test-jar - ${project.parent.version} - test - - - diff --git a/tools/perf/build.gradle b/tools/perf/build.gradle index 1223e2000d0..80df1221ae9 100644 --- a/tools/perf/build.gradle +++ b/tools/perf/build.gradle @@ -67,5 +67,6 @@ dependencies { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-tools-perf' } diff --git a/tools/perf/pom.xml b/tools/perf/pom.xml deleted file mode 100644 index a0c39e39687..00000000000 --- a/tools/perf/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper-tools-parent - 4.15.0-SNAPSHOT - - bookkeeper-perf - Apache BookKeeper :: Tools :: Perf - - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - org.hdrhistogram - HdrHistogram - - - - diff --git a/tools/pom.xml b/tools/pom.xml deleted file mode 100644 index 50010001879..00000000000 --- a/tools/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-parent - Apache BookKeeper :: Tools :: Parent - pom - - framework - ledger - stream - perf - all - - diff --git a/tools/stream/pom.xml b/tools/stream/pom.xml deleted file mode 100644 index ba9eea73590..00000000000 --- a/tools/stream/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper-tools-parent - 4.15.0-SNAPSHOT - - stream-storage-cli - Apache BookKeeper :: Tools :: Stream - - - - org.apache.bookkeeper - stream-storage-java-client - ${project.version} - - - org.apache.bookkeeper - stream-storage-service-impl - ${project.version} - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - From 4df57a88bf6199e8b693f48b4d8b39459869cbe3 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 27 Jan 2022 05:20:34 -0800 Subject: [PATCH 43/79] Upgraded snakeyaml, CVE-2017-18640 (#3014) --- dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.gradle b/dependencies.gradle index 075b3bb7091..3c334b3c9f5 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -75,7 +75,7 @@ depVersions = [ rocksDb: "6.22.1.1", rxjava: "3.0.1", slf4j: "1.7.32", - snakeyaml: "1.19", + snakeyaml: "1.30", spotbugsAnnotations: "3.1.8", protocGenGrpcJava: "1.12.0", shrinkwrap:"3.1.4", From 9ce0651a777c4822811860170348a2616b69ba7a Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 27 Jan 2022 05:20:53 -0800 Subject: [PATCH 44/79] Upgrading protobuf, CVE-2021-22569 (#3013) --- dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.gradle b/dependencies.gradle index 3c334b3c9f5..3b955a96609 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -70,7 +70,7 @@ depVersions = [ nettyTcnativeBoringSsl: "2.0.46.Final", powermock: "2.0.2", prometheus: "0.8.1", - protobuf: "3.14.0", + protobuf: "3.16.1", reflections: "0.9.11", rocksDb: "6.22.1.1", rxjava: "3.0.1", From 145d7fefd0b4418c4b222c24f2291ae027cd5507 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 27 Jan 2022 05:24:44 -0800 Subject: [PATCH 45/79] Added OWASP dependency-check (#3002) * Added OWASP dependency-check * Suppress ETCD-related misdetections --- build.gradle | 43 +++++++++++++ gradle.properties | 1 + settings.gradle | 1 + src/owasp-dependency-check-suppressions.xml | 71 +++++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 src/owasp-dependency-check-suppressions.xml diff --git a/build.gradle b/build.gradle index b92a70cbf41..a3f4416c156 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,7 @@ plugins { id 'checkstyle' id 'org.nosphere.apache.rat' id 'com.github.spotbugs' + id 'org.owasp.dependencycheck' } subprojects { @@ -58,6 +59,13 @@ releaseParams { // ReleaseExtension } } +def skipDepCheck = [] +allprojects { + if (it.path.startsWith(':tests')) { + skipDepCheck << it.path + } +} + allprojects { apply from: "$rootDir/dependencies.gradle" if (it.path != ':circe-checksum:src:main:circe' @@ -70,6 +78,41 @@ allprojects { apply plugin: 'org.nosphere.apache.rat' apply plugin: "checkstyle" apply plugin: 'com.github.spotbugs' + + if (!it.path.startsWith(':tests')) { + apply plugin: 'org.owasp.dependencycheck' + + dependencyCheck { + // see https://jeremylong.github.io/DependencyCheck/dependency-check-gradle/configuration.html + // for configuration properties + suppressionFile = "$rootDir/src/owasp-dependency-check-suppressions.xml" + skipProjects = skipDepCheck + skipConfigurations = ["checkstyle", "spotbugs"] + analyzers { + msbuildEnabled = false + rubygemsEnabled = false + pyDistributionEnabled = false + pyPackageEnabled = false + nuspecEnabled = false + nugetconfEnabled = false + assemblyEnabled = false + cmakeEnabled = false + composerEnabled = false + cpanEnabled = false + nodeEnabled = false + cocoapodsEnabled = false + swiftEnabled = false + swiftPackageResolvedEnabled = false + bundleAuditEnabled = false + golangDepEnabled = false + golangModEnabled = false + + nodeAudit.enabled = false + retirejs.enabled = false + } + } + } + checkstyle { toolVersion "${checkStyleVersion}" configFile file("$rootDir/buildtools/src/main/resources/bookkeeper/checkstyle.xml") diff --git a/gradle.properties b/gradle.properties index b8be5831081..bba0e1dfa0a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -27,3 +27,4 @@ checkStyleVersion=6.19 spotbugsPlugin=4.7.0 testLogger=2.0.0 testRetry=1.0.0 +owaspPlugin=6.5.3 diff --git a/settings.gradle b/settings.gradle index 3d192cfd755..036b10413e0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,6 +25,7 @@ pluginManagement { id "com.github.spotbugs" version "${spotbugsPlugin}" id "com.adarshr.test-logger" version "${testLogger}" id "org.gradle.test-retry" version "${testRetry}" + id "org.owasp.dependencycheck" version "${owaspPlugin}" } } diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml new file mode 100644 index 00000000000..c65b18b1b2e --- /dev/null +++ b/src/owasp-dependency-check-suppressions.xml @@ -0,0 +1,71 @@ + + + + + + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15106 + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15112 + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15113 + + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15106 + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15112 + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15113 + + + + From 0c650c141f39543bd310c45150c0f0a3303bacf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 27 Jan 2022 16:08:59 +0100 Subject: [PATCH 46/79] [build] do not run spotbugsTest by default (#3017) --- .github/workflows/pr-validation.yml | 2 +- build.gradle | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index f96a401dca3..816315b3707 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -47,6 +47,6 @@ jobs: with: java-version: 1.8 - name: Validate pull request - run: ./gradlew build -x spotbugsTest -x signDistTar -x test + run: ./gradlew build -x signDistTar -x test - name: Check license files run: dev/check-all-licenses diff --git a/build.gradle b/build.gradle index a3f4416c156..86495b445fa 100644 --- a/build.gradle +++ b/build.gradle @@ -130,6 +130,7 @@ allprojects { toolVersion = '3.1.8' excludeFilter = file("$rootDir/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml") reportLevel = 'high' + spotbugsTest.enabled = false } From 921c24d8ef8e8c869f1eacba4a34667ef039bea0 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 2 Feb 2022 09:12:38 -0800 Subject: [PATCH 47/79] Forcing the same version of netty across the projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation While experimenting with OWASP dependency checker I noticed that we have 3 versions of netty mixed in: 4.1.72 (current one, expected) plus 4.1.63 and 4.1.50 (brought with ZK and some other dependencies). ### Changes Made gradle enforce the same version of netty in subprojects. Reviewers: Nicolò Boschi , Enrico Olivelli This closes #3008 from dlg99/gradle-netty --- build.gradle | 3 +++ dependencies.gradle | 1 + 2 files changed, 4 insertions(+) diff --git a/build.gradle b/build.gradle index 86495b445fa..03df876f96a 100644 --- a/build.gradle +++ b/build.gradle @@ -280,6 +280,9 @@ allprojects { showStandardStreams = true } } + dependencies { + implementation(enforcedPlatform(depLibs.nettyBom)) + } tasks.register('writeClasspath') { doLast { buildDir.mkdirs() diff --git a/dependencies.gradle b/dependencies.gradle index 3b955a96609..ff760320805 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -176,6 +176,7 @@ depLibs = [ metricsJvm: "io.dropwizard.metrics:metrics-jvm:${depVersions.dropwizard}", metricsGraphite: "io.dropwizard.metrics:metrics-graphite:${depVersions.dropwizard}", mockito: "org.mockito:mockito-core:${depVersions.mockito}", + nettyBom: "io.netty:netty-bom:${depVersions.netty}", nettyBuffer: "io.netty:netty-buffer:${depVersions.netty}", nettyCommon: "io.netty:netty-common:${depVersions.netty}", nettyHandler: "io.netty:netty-handler:${depVersions.netty}", From ec823fe9b963490f1dc82c8c831c10abca274b4c Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 2 Feb 2022 09:14:26 -0800 Subject: [PATCH 48/79] Forced the same version of guava (plus upgraded it); fixed deprecations etc. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation Older versions of guava (w/CVEs) used in some subprojects ### Changes Forced the same version of guava; fixed deprecation problems (murmur3_32) and compilation problems (checkArgument). checkArgument cannot be statically imported because there are now overrides of it; checkstyle was not very cooperative so I had to remove the import altogether. Then updated the guava version to match one in Pulsar. Reviewers: Enrico Olivelli , Nicolò Boschi This closes #3010 from dlg99/gradle-guava --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 8 ++++---- bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt | 8 ++++---- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 8 ++++---- .../main/java/org/apache/bookkeeper/metastore/Value.java | 2 +- build.gradle | 5 +++++ .../bookkeeper/common/util/affinity/impl/NativeUtils.java | 4 +--- .../common/util/affinity/impl/ProcessorsInfo.java | 6 ++---- dependencies.gradle | 3 ++- .../apache/bookkeeper/common/router/HashRouterTest.java | 2 +- 9 files changed, 24 insertions(+), 22 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 2721b5b2886..2799377f88f 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -312,7 +312,7 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ @@ -667,10 +667,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 5d5e822d4f2..0183ae58fae 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -289,7 +289,7 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ @@ -594,10 +594,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 23f1c9a265b..23a197ccd64 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -310,7 +310,7 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ @@ -659,10 +659,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java index 43354c344e5..9a3b43e12d6 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java @@ -92,7 +92,7 @@ public Value project(Set fields) { @Override public int hashCode() { - HashFunction hf = Hashing.murmur3_32(); + HashFunction hf = Hashing.murmur3_32_fixed(); Hasher hc = hf.newHasher(); for (String key : fields.keySet()) { hc.putString(key, Charset.defaultCharset()); diff --git a/build.gradle b/build.gradle index 03df876f96a..a06c27fa8e6 100644 --- a/build.gradle +++ b/build.gradle @@ -289,6 +289,11 @@ allprojects { new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" } } + + dependencies { + implementation(enforcedPlatform(depLibs.guavaBom)) + } + } repositories { diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java index 67c8679ab12..218e9f1af5e 100644 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java +++ b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java @@ -20,8 +20,6 @@ */ package org.apache.bookkeeper.common.util.affinity.impl; -import static com.google.common.base.Preconditions.checkArgument; - import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.File; @@ -50,7 +48,7 @@ public class NativeUtils { value = "OBL_UNSATISFIED_OBLIGATION", justification = "work around for java 9: https://github.com/spotbugs/spotbugs/issues/493") public static void loadLibraryFromJar(String path) throws Exception { - checkArgument(path.startsWith("/"), "absolute path must start with /"); + com.google.common.base.Preconditions.checkArgument(path.startsWith("/"), "absolute path must start with /"); String[] parts = path.split("/"); String filename = (parts.length > 0) ? parts[parts.length - 1] : null; diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java index 8a812a84d1a..a1b3963d2c7 100644 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java +++ b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java @@ -20,8 +20,6 @@ */ package org.apache.bookkeeper.common.util.affinity.impl; -import static com.google.common.base.Preconditions.checkArgument; - import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -80,8 +78,8 @@ static ProcessorsInfo parseCpuInfo(String cpuInfoString) { } } - checkArgument(cpuId >= 0); - checkArgument(coreId >= 0); + com.google.common.base.Preconditions.checkArgument(cpuId >= 0); + com.google.common.base.Preconditions.checkArgument(coreId >= 0); pi.cpus.put(cpuId, coreId); } diff --git a/dependencies.gradle b/dependencies.gradle index ff760320805..920288b3b02 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -45,7 +45,7 @@ depVersions = [ gradleTooling: "4.0.1", grpc: "1.42.1", groovy: "2.5.8", - guava: "30.0-jre", + guava: "31.0.1-jre", hamcrest: "1.3", hdrhistogram: "2.1.10", httpclient: "4.5.13", @@ -139,6 +139,7 @@ depLibs = [ exclude group: 'com.google.guava', module: 'guava' }, groovy: "org.codehaus.groovy:groovy-all:${depVersions.groovy}", + guavaBom: "com.google.guava:guava-bom:${depVersions.guava}", guava: dependencies.create("com.google.guava:guava:${depVersions.guava}"){ exclude group: 'com.google.code.findbugs', module: 'jsr305' }, diff --git a/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java b/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java index a36b29123b4..4f46cb05b1a 100644 --- a/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java +++ b/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java @@ -40,7 +40,7 @@ public void testByteBufHashRouter() { int hash32 = Murmur3.hash32( key, key.readerIndex(), key.readableBytes(), (int) AbstractHashRouter.HASH_SEED); int bytesHash32 = Murmur3.hash32(keyBytes, 0, keyBytes.length, (int) AbstractHashRouter.HASH_SEED); - int guavaHash32 = Hashing.murmur3_32((int) AbstractHashRouter.HASH_SEED) + int guavaHash32 = Hashing.murmur3_32_fixed((int) AbstractHashRouter.HASH_SEED) .newHasher() .putString("foo", UTF_8) .hash() From 3cc9d470b942fe6470992eebace793a429be0956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 2 Feb 2022 18:17:07 +0100 Subject: [PATCH 49/79] Upgrade Netty to 4.1.73.Final ### Motivation Changelog: https://netty.io/news/2022/01/12/4-1-73-Final.html The main reason to upgrade is because of an [intensive I/O disk scheduled task](https://github.com/netty/netty/pull/11943) introduced in 4.1.72.Final which is synchronous and can cause EventLoop to blocked very often. ### Changes * Upgrade Netty from 4.1.72.Final to 4.1.73.Final * [Netty 4.1.73.Final depends on netty-tc-native 2.0.46](https://github.com/netty/netty/blob/b5219aeb4ee62f15d5dfb2b9c29d0c694aca05be/pom.xml#L545) as Netty 4.1.72.Final, so no need to upgrade Reviewers: Andrey Yegorov This closes #3020 from nicoloboschi/upgrade-netty-4.1.73 --- bookkeeper-dist/all/build.gradle | 2 +- bookkeeper-dist/bkctl/build.gradle | 2 +- bookkeeper-dist/server/build.gradle | 2 +- bookkeeper-dist/src/assemble/bin-all.xml | 2 +- bookkeeper-dist/src/assemble/bin-server.xml | 2 +- bookkeeper-dist/src/assemble/bkctl.xml | 2 +- .../src/main/resources/LICENSE-all.bin.txt | 144 +++++++++--------- .../src/main/resources/LICENSE-bkctl.bin.txt | 140 ++++++++--------- .../src/main/resources/LICENSE-server.bin.txt | 144 +++++++++--------- .../src/main/resources/NOTICE-all.bin.txt | 30 ++-- .../src/main/resources/NOTICE-bkctl.bin.txt | 26 ++-- .../src/main/resources/NOTICE-server.bin.txt | 30 ++-- .../LICENSE.aalto-xml.txt | 0 .../LICENSE.base64.txt | 0 .../LICENSE.bouncycastle.txt | 0 .../LICENSE.caliper.txt | 0 .../LICENSE.commons-lang.txt | 0 .../LICENSE.commons-logging.txt | 0 .../LICENSE.compress-lzf.txt | 0 .../LICENSE.dnsinfo.txt | 0 .../LICENSE.harmony.txt | 0 .../LICENSE.hpack.txt | 0 .../LICENSE.hyper-hpack.txt | 0 .../LICENSE.jboss-marshalling.txt | 0 .../LICENSE.jbzip2.txt | 0 .../LICENSE.jctools.txt | 0 .../LICENSE.jfastlz.txt | 0 .../LICENSE.jsr166y.txt | 0 .../LICENSE.jzlib.txt | 0 .../LICENSE.libdivsufsort.txt | 0 .../LICENSE.log4j.txt | 0 .../LICENSE.lz4.txt | 0 .../LICENSE.lzma-java.txt | 0 .../LICENSE.mvn-wrapper.txt | 0 .../LICENSE.nghttp2-hpack.txt | 0 .../LICENSE.protobuf.txt | 0 .../LICENSE.slf4j.txt | 0 .../LICENSE.snappy.txt | 0 .../LICENSE.webbit.txt | 0 .../NOTICE.harmony.txt | 0 dependencies.gradle | 2 +- 41 files changed, 264 insertions(+), 264 deletions(-) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.aalto-xml.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.base64.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.bouncycastle.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.caliper.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.commons-lang.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.commons-logging.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.compress-lzf.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.dnsinfo.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.harmony.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.hpack.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.hyper-hpack.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jboss-marshalling.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jbzip2.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jctools.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jfastlz.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jsr166y.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.jzlib.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.libdivsufsort.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.log4j.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.lz4.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.lzma-java.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.mvn-wrapper.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.nghttp2-hpack.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.protobuf.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.slf4j.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.snappy.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/LICENSE.webbit.txt (100%) rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.72.Final => netty-4.1.73.Final}/NOTICE.harmony.txt (100%) diff --git a/bookkeeper-dist/all/build.gradle b/bookkeeper-dist/all/build.gradle index 68f1043eff8..3657337a0a9 100644 --- a/bookkeeper-dist/all/build.gradle +++ b/bookkeeper-dist/all/build.gradle @@ -60,7 +60,7 @@ def depLicences = [ "javax.servlet-api-4.0.0/CDDL+GPL-1.1", "bouncycastle-1.0.2.1/LICENSE.html", "jsr-305/LICENSE", - "netty-4.1.72.Final/*", + "netty-4.1.73.Final/*", "paranamer-2.8/LICENSE.txt", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/bkctl/build.gradle b/bookkeeper-dist/bkctl/build.gradle index 2495b7b0148..34acdf98a26 100644 --- a/bookkeeper-dist/bkctl/build.gradle +++ b/bookkeeper-dist/bkctl/build.gradle @@ -49,7 +49,7 @@ releaseArtifacts { def depLicences = [ "checker-qual-3.5.0/LICENSE", "google-auth-library-credentials-0.20.0/LICENSE", - "netty-4.1.72.Final/*", + "netty-4.1.73.Final/*", "bouncycastle-1.0.2.1/LICENSE.html", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/server/build.gradle b/bookkeeper-dist/server/build.gradle index ba2d50da268..5c3c3fb97d6 100644 --- a/bookkeeper-dist/server/build.gradle +++ b/bookkeeper-dist/server/build.gradle @@ -53,7 +53,7 @@ def depLicences = [ "checker-qual-3.5.0/LICENSE", "google-auth-library-credentials-0.20.0/LICENSE", "javax.servlet-api-4.0.0/CDDL+GPL-1.1", - "netty-4.1.72.Final/*", + "netty-4.1.73.Final/*", "bouncycastle-1.0.2.1/LICENSE.html", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/src/assemble/bin-all.xml b/bookkeeper-dist/src/assemble/bin-all.xml index 4ec9dea8221..3d700895169 100644 --- a/bookkeeper-dist/src/assemble/bin-all.xml +++ b/bookkeeper-dist/src/assemble/bin-all.xml @@ -59,7 +59,7 @@ javax.servlet-api-4.0.0/CDDL+GPL-1.1 bouncycastle-1.0.2.1/LICENSE.html jsr-305/LICENSE - netty-4.1.72.Final/* + netty-4.1.73.Final/* paranamer-2.8/LICENSE.txt protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/assemble/bin-server.xml b/bookkeeper-dist/src/assemble/bin-server.xml index cfb176d3d41..439491e5e9b 100644 --- a/bookkeeper-dist/src/assemble/bin-server.xml +++ b/bookkeeper-dist/src/assemble/bin-server.xml @@ -52,7 +52,7 @@ checker-qual-3.5.0/LICENSE google-auth-library-credentials-0.20.0/LICENSE javax.servlet-api-4.0.0/CDDL+GPL-1.1 - netty-4.1.72.Final/* + netty-4.1.73.Final/* bouncycastle-1.0.2.1/LICENSE.html protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/assemble/bkctl.xml b/bookkeeper-dist/src/assemble/bkctl.xml index 855f410d962..18e3904c2cc 100644 --- a/bookkeeper-dist/src/assemble/bkctl.xml +++ b/bookkeeper-dist/src/assemble/bkctl.xml @@ -66,7 +66,7 @@ checker-qual-3.5.0/LICENSE google-auth-library-credentials-0.20.0/LICENSE - netty-4.1.72.Final/* + netty-4.1.73.Final/* bouncycastle-1.0.2.1/LICENSE.html protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 2799377f88f..adfb859d517 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -217,23 +217,23 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar [11] +- lib/io.netty-netty-common-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.73.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-transport-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar [11] - lib/io.prometheus-simpleclient-0.8.1.jar [12] - lib/io.prometheus-simpleclient_common-0.8.1.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.8.1.jar [12] @@ -319,7 +319,7 @@ Apache Software License, Version 2. [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.73.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.8.1 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/3.9.8 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/3.9.8 @@ -359,229 +359,229 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.73.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.73.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.73.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.73.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.73.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.73.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.73.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.73.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 0183ae58fae..9a55d42c0f4 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -217,21 +217,21 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar [11] +- lib/io.netty-netty-common-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.73.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-transport-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar [11] - lib/org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar [16] - lib/org.apache.logging.log4j-log4j-api-2.17.1.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.17.1.jar [16] @@ -296,7 +296,7 @@ Apache Software License, Version 2. [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.73.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.17.1 [17] Source available at https://github.com/java-native-access/jna/tree/3.2.7 [18] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-collections.git;a=tag;h=a3a5ad @@ -328,229 +328,229 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.73.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.73.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.73.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.73.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.73.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.73.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.73.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.73.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 23a197ccd64..a00cd92b9a9 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -217,23 +217,23 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar [11] +- lib/io.netty-netty-common-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-4.1.73.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.73.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.73.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-transport-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar [11] - lib/io.prometheus-simpleclient-0.8.1.jar [12] - lib/io.prometheus-simpleclient_common-0.8.1.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.8.1.jar [12] @@ -317,7 +317,7 @@ Apache Software License, Version 2. [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.73.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.8.1 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/3.9.8 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/3.9.8 @@ -357,229 +357,229 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.73.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.73.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.73.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.73.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.73.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.73.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.73.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.73.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.73.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.73.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.73.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.73.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index 0bff137ce70..f145e9e067b 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -22,23 +22,23 @@ LongAdder), which was released with the following comments: http://creativecommons.org/publicdomain/zero/1.0/ ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.73.Final.jar +- lib/io.netty-netty-codec-4.1.73.Final.jar +- lib/io.netty-netty-codec-dns-4.1.73.Final.jar +- lib/io.netty-netty-codec-http-4.1.73.Final.jar +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar +- lib/io.netty-netty-common-4.1.73.Final.jar +- lib/io.netty-netty-handler-4.1.73.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar +- lib/io.netty-netty-resolver-4.1.73.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.73.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-transport-4.1.73.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 8ee2bd437e0..bffec61c02b 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -5,21 +5,21 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.73.Final.jar +- lib/io.netty-netty-codec-4.1.73.Final.jar +- lib/io.netty-netty-codec-http-4.1.73.Final.jar +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar +- lib/io.netty-netty-common-4.1.73.Final.jar +- lib/io.netty-netty-handler-4.1.73.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar +- lib/io.netty-netty-resolver-4.1.73.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-transport-4.1.73.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index a991cd6ba09..f9dbca9ddf7 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -5,23 +5,23 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.73.Final.jar +- lib/io.netty-netty-codec-4.1.73.Final.jar +- lib/io.netty-netty-codec-dns-4.1.73.Final.jar +- lib/io.netty-netty-codec-http-4.1.73.Final.jar +- lib/io.netty-netty-codec-http2-4.1.73.Final.jar +- lib/io.netty-netty-codec-socks-4.1.73.Final.jar +- lib/io.netty-netty-common-4.1.73.Final.jar +- lib/io.netty-netty-handler-4.1.73.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.73.Final.jar +- lib/io.netty-netty-resolver-4.1.73.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.73.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar - lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-transport-4.1.73.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.73.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.73.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.aalto-xml.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.base64.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.base64.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.base64.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.base64.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.bouncycastle.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.caliper.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.caliper.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.caliper.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.caliper.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-lang.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.commons-lang.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-lang.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.commons-lang.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-logging.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.commons-logging.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-logging.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.commons-logging.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.compress-lzf.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.dnsinfo.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.harmony.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.harmony.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.harmony.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.harmony.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.hyper-hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jboss-marshalling.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jbzip2.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jbzip2.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jbzip2.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jbzip2.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jctools.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jctools.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jctools.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jctools.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jfastlz.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jfastlz.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jfastlz.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jfastlz.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jsr166y.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jsr166y.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jsr166y.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jsr166y.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jzlib.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jzlib.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jzlib.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.jzlib.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.libdivsufsort.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.log4j.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.log4j.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.log4j.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.log4j.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lz4.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.lz4.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lz4.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.lz4.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lzma-java.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.lzma-java.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lzma-java.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.lzma-java.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.mvn-wrapper.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.nghttp2-hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.protobuf.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.protobuf.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.protobuf.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.protobuf.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.slf4j.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.slf4j.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.slf4j.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.slf4j.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.snappy.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.snappy.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.snappy.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.snappy.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.webbit.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.webbit.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.webbit.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/LICENSE.webbit.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/NOTICE.harmony.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/NOTICE.harmony.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/NOTICE.harmony.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.73.Final/NOTICE.harmony.txt diff --git a/dependencies.gradle b/dependencies.gradle index 920288b3b02..b822b63af62 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -66,7 +66,7 @@ depVersions = [ lombok: "1.18.20", lz4: "1.3.0", mockito: "3.0.0", - netty: "4.1.72.Final", + netty: "4.1.73.Final", nettyTcnativeBoringSsl: "2.0.46.Final", powermock: "2.0.2", prometheus: "0.8.1", From 7044e0a30a773281ffb6403c08eb19db0139cecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 3 Feb 2022 08:39:33 +0100 Subject: [PATCH 50/79] [website] Remove Maven references and remove dead links from README (#3018) --- README.md | 16 +++---- site/community/contributing.md | 15 ------- .../latest/getting-started/installation.md | 45 +++---------------- 3 files changed, 12 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 53876ee9e3b..29ed1e435f5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ logo -[![Coverage Status](https://coveralls.io/repos/github/apache/bookkeeper/badge.svg?branch=master)](https://coveralls.io/github/apache/bookkeeper?branch=master) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.bookkeeper/bookkeeper/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.bookkeeper/bookkeeper) # Apache BookKeeper @@ -29,7 +28,7 @@ Please visit the [Documentation](https://bookkeeper.apache.org/docs/latest/overv ### Report a Bug -For filing bugs, suggesting improvements, or requesting new features, help us out by [opening a Github issue](https://github.com/apache/bookkeeper/issues) or [opening an Apache jira](https://issues.apache.org/jira/browse/BOOKKEEPER). +For filing bugs, suggesting improvements, or requesting new features, help us out by [opening a Github issue](https://github.com/apache/bookkeeper/issues). ### Need Help? @@ -45,15 +44,10 @@ We feel that a welcoming open community is important and welcome contributions. ### Contributing Code -1. See [Developer Setup](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Developer+Setup) to get your local environment setup. +1. See our [installation guide](https://bookkeeper.apache.org/docs/latest/getting-started/installation/) to get your local environment setup. -2. Take a look at our open issues: [JIRA Issues](https://issues.apache.org/jira/browse/BOOKKEEPER) [Github Issues](https://github.com/apache/bookkeeper/issues). +2. Take a look at our open issues: [Github Issues](https://github.com/apache/bookkeeper/issues). -3. Review our [coding style](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Coding+Guide) and follow our [pull requests](https://github.com/apache/bookkeeper/pulls) to learn about our conventions. - -4. Make your changes according to our [contribution guide](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Contributing+to+BookKeeper). - -### Improving Website and Documentation - -1. See [Building the website and documentation](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Building+the+website+and+documentation) on how to build the website and documentation. +3. Review our [coding style](https://bookkeeper.apache.org/community/coding_guide/) and follow our [pull requests](https://github.com/apache/bookkeeper/pulls) to learn more about our conventions. +4. Make your changes according to our [contributing guide](https://bookkeeper.apache.org/community/contributing/) diff --git a/site/community/contributing.md b/site/community/contributing.md index 7bf513b3c6a..b56dcb742cd 100644 --- a/site/community/contributing.md +++ b/site/community/contributing.md @@ -102,20 +102,6 @@ Depending on your preferred development environment, you may need to prepare it ##### IntelliJ -###### Enable Annotation Processing - -To configure annotation processing in IntelliJ: - -1. Open Annotation Processors Settings dialog box by going to Settings -> Build, Execution, Deployment -> Compiler -> Annotation Processors. -1. Select the following buttons: - 1. "Enable annotation processing" - 1. "Obtain processors from project classpath" - 1. "Store generated sources relative to: Module content root" -1. Set the generated source directories to be equal to the Maven directories: - 1. Set "Production sources directory:" to "target/generated-sources/annotations". - 1. Set "Test sources directory:" to "target/generated-test-sources/test-annotations". -1. Click "OK". - ###### Checkstyle IntelliJ supports checkstyle within the IDE using the Checkstyle-IDEA plugin. @@ -153,7 +139,6 @@ Start Eclipse with a fresh workspace in a separate directory from your checkout. File -> Import... - -> Existing Maven Projects -> Browse to the directory you cloned into and select "bookkeeper" -> make sure all bookkeeper projects are selected -> Finalize diff --git a/site/docs/latest/getting-started/installation.md b/site/docs/latest/getting-started/installation.md index 9f856ec1d21..dcc4a82a24e 100644 --- a/site/docs/latest/getting-started/installation.md +++ b/site/docs/latest/getting-started/installation.md @@ -12,64 +12,33 @@ You can install BookKeeper either by [downloading](#download) a [GZipped](http:/ * [Unix environment](http://www.opengroup.org/unix) * [Java Development Kit 1.8](http://www.oracle.com/technetwork/java/javase/downloads/index.html) or later -* [Maven 3.0](https://maven.apache.org/install.html) or later ## Download -You can download Apache BookKeeper releases from one of many [Apache mirrors](http://www.apache.org/dyn/closer.cgi/bookkeeper). Here's an example for the [apache.claz.org](http://apache.claz.org/bookkeeper) mirror: - -```shell -$ curl -O {{ download_url }} -$ tar xvf bookkeeper-{{ site.latest_release }}-src.tar.gz -$ cd bookkeeper-{{ site.latest_release }} -``` +You can download Apache BookKeeper releases from one of many [Apache mirrors](https://dlcdn.apache.org/bookkeeper/). ## Clone -To build BookKeeper from source, clone the repository, either from the [GitHub mirror]({{ site.github_repo }}) or from the [Apache repository](http://git.apache.org/bookkeeper.git/): +To build BookKeeper from source, clone the repository, either from the [GitHub mirror]({{ site.github_repo }}): ```shell -# From the GitHub mirror $ git clone {{ site.github_repo}} - -# From Apache directly -$ git clone git://git.apache.org/bookkeeper.git/ ``` -## Build using Maven +## Build using Gradle -Once you have the BookKeeper on your local machine, either by [downloading](#download) or [cloning](#clone) it, you can then build BookKeeper from source using Maven: +Once you have the BookKeeper on your local machine, either by [downloading](#download) or [cloning](#clone) it, you can then build BookKeeper from source using Gradle: ```shell -$ mvn package +$ ./gradlew build -x signDistTar -x test ``` -Since 4.8.0, bookkeeper introduces `table service`. If you would like to build and tryout table service, you can build it with `stream` profile. +To run all the tests: ```shell -$ mvn package -Dstream +$ ./gradlew test -x signDistTar ``` -> You can skip tests by adding the `-DskipTests` flag when running `mvn package`. - -### Useful Maven commands - -Some other useful Maven commands beyond `mvn package`: - -Command | Action -:-------|:------ -`mvn clean` | Removes build artifacts -`mvn compile` | Compiles JAR files from Java sources -`mvn compile spotbugs:spotbugs` | Compile using the Maven [SpotBugs](https://github.com/spotbugs/spotbugs-maven-plugin) plugin -`mvn install` | Install the BookKeeper JAR locally in your local Maven cache (usually in the `~/.m2` directory) -`mvn deploy` | Deploy the BookKeeper JAR to the Maven repo (if you have the proper credentials) -`mvn verify` | Performs a wide variety of verification and validation tasks -`mvn apache-rat:check` | Run Maven using the [Apache Rat](http://creadur.apache.org/rat/apache-rat-plugin/) plugin -`mvn compile javadoc:aggregate` | Build Javadocs locally -`mvn -am -pl bookkeeper-dist/server package` | Build a server distribution using the Maven [Assembly](http://maven.apache.org/plugins/maven-assembly-plugin/) plugin - -> You can enable `table service` by adding the `-Dstream` flag when running above commands. - ## Package directory The BookKeeper project contains several subfolders that you should be aware of: From 8e0ad18206ecc8c58730180a59edc37ec84b68ae Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 3 Feb 2022 08:24:30 -0800 Subject: [PATCH 51/79] CI workflow to check dependencies with OWASP (#3021) --- .github/workflows/owasp-dep-check.yml | 59 +++++++++++++++++++++++++++ build.gradle | 1 + 2 files changed, 60 insertions(+) create mode 100644 .github/workflows/owasp-dep-check.yml diff --git a/.github/workflows/owasp-dep-check.yml b/.github/workflows/owasp-dep-check.yml new file mode 100644 index 00000000000..a3fbdffe471 --- /dev/null +++ b/.github/workflows/owasp-dep-check.yml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: OWASP Dependency Check + +on: + push: + pull_request: + branches: + - master + - branch-* + paths-ignore: + - 'site/**' + workflow_dispatch: + + +jobs: + check: + + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: run "clean build dependencyCheckAggregate" to trigger dependency check + run: ./gradlew clean build -x signDistTar -x test dependencyCheckAggregate + + - name: Upload report + uses: actions/upload-artifact@v2 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: dependency report + path: build/reports/dependency-check-report.html diff --git a/build.gradle b/build.gradle index a06c27fa8e6..c53719dfaec 100644 --- a/build.gradle +++ b/build.gradle @@ -88,6 +88,7 @@ allprojects { suppressionFile = "$rootDir/src/owasp-dependency-check-suppressions.xml" skipProjects = skipDepCheck skipConfigurations = ["checkstyle", "spotbugs"] + failBuildOnCVSS = 7 analyzers { msbuildEnabled = false rubygemsEnabled = false From ed19227599a35c499fdd67b79cbd0a7e48bda610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Sat, 5 Feb 2022 00:10:37 +0100 Subject: [PATCH 52/79] [BACKWARD TESTS] add BookKeeper 4.14.4 ### Motivation BK 4.14.4 has been released and we should test it in the upgrade tests Note: for the sake of test performance, we test the upgrades only for the latest releases of each minor release ### Changes * Replaced BK 4.14.3 with 4.14.4 Reviewers: Enrico Olivelli , Andrey Yegorov This closes #2997 from nicoloboschi/tests/add-bk-4144-backward-compat --- .../bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy | 4 ++-- tests/docker-images/all-released-versions-image/Dockerfile | 2 +- .../tests/integration/utils/BookKeeperClusterUtils.java | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy index d885e778ff8..e7e9a3170a3 100644 --- a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy +++ b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy @@ -143,11 +143,11 @@ class TestCompatUpgrade { @Test public void test_006_4130to4143() throws Exception { - testUpgrade("4.13.0", "4.14.3") + testUpgrade("4.13.0", "4.14.4") } @Test public void test_007_4143toCurrentMaster() throws Exception { - testUpgrade("4.14.3", BookKeeperClusterUtils.CURRENT_VERSION) + testUpgrade("4.14.4", BookKeeperClusterUtils.CURRENT_VERSION) } } diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index 9c0301fa205..af2d9651de8 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -38,7 +38,7 @@ RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.10.0/bookke RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.11.1/bookkeeper-server-4.11.1-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookkeeper-server-4.12.1-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.3/bookkeeper-server-4.14.3-bin.tar.gz{,.sha512,.asc} +RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.4/bookkeeper-server-4.14.4-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/pulsar-1.21.0-incubating/apache-pulsar-1.21.0-incubating-bin.tar.gz{,.asc} RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index 94cdedfab96..a29331131fe 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -43,9 +43,10 @@ public class BookKeeperClusterUtils { public static final String CURRENT_VERSION = System.getProperty("currentVersion"); public static final List OLD_CLIENT_VERSIONS = - Arrays.asList("4.8.2", "4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3"); + Arrays.asList("4.8.2", "4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.4"); private static final List OLD_CLIENT_VERSIONS_WITH_CURRENT_LEDGER_METADATA_FORMAT = - Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3"); + Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.4"); + private static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterUtils.class); From 10457052a41b5a086be754b6238947d8e764711e Mon Sep 17 00:00:00 2001 From: mauricebarnum Date: Fri, 4 Feb 2022 16:28:41 -0800 Subject: [PATCH 53/79] Gradle 6.9.2 (#3022) * Remove annoying println * Update gradle to 6.9.2 This includes `Mitigations for log4j vulnerability in Gradle builds` https://github.com/gradle/gradle/issues/19328 Full release notes https://docs.gradle.org/6.9.2/release-notes.html --- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 259 ++++++++++++++--------- microbenchmarks/build.gradle | 3 +- 3 files changed, 156 insertions(+), 108 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 047e8487800..cb38b87d24e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -21,6 +21,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 86c97c2b584..1b6c787337f 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,101 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/microbenchmarks/build.gradle b/microbenchmarks/build.gradle index 526c170753e..a58eea03b75 100644 --- a/microbenchmarks/build.gradle +++ b/microbenchmarks/build.gradle @@ -41,7 +41,6 @@ jar { manifest { attributes 'Main-Class': 'org.openjdk.jmh.Main' } - println configurations.runtimeClasspath.collect().size() from { configurations.compileClasspath.collect {it.isDirectory() ? it : zipTree(it) } } @@ -49,4 +48,4 @@ jar { exclude 'META-INF/*.SF' exclude 'META-INF/*.DSA' archiveBaseName = 'microbenchmarks' -} \ No newline at end of file +} From 52c4f9c01d465135396ff78044de32c911c826e5 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Sun, 6 Feb 2022 01:36:50 +0800 Subject: [PATCH 54/79] Bump gradle to 7.3.3, compat java 17 (#2924) --- build.gradle | 40 +++++++++++++++++-- gradle/wrapper/gradle-wrapper.properties | 2 +- .../backward-compat/bc-non-fips/build.gradle | 9 ++++- tests/integration/cluster/build.gradle | 1 + tests/integration/smoke/build.gradle | 1 + 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index c53719dfaec..6cba809a34f 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,18 @@ releaseArtifacts { fromProject("bookkeeper-dist-all") fromProject("bookkeeper-dist-bkctl") fromProject("bookkeeper-dist-server") + + tasks.withType(Jar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Tar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Zip) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } } releaseParams { // ReleaseExtension @@ -78,7 +90,7 @@ allprojects { apply plugin: 'org.nosphere.apache.rat' apply plugin: "checkstyle" apply plugin: 'com.github.spotbugs' - + if (!it.path.startsWith(':tests')) { apply plugin: 'org.owasp.dependencycheck' @@ -136,12 +148,24 @@ allprojects { + tasks.withType(Jar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Tar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Zip) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + task testJar(type: Jar, dependsOn: testClasses) { - classifier = 'tests' + archiveClassifier = 'tests' from sourceSets.test.output } - // TODO- Seperate rat plugins and other plugins configuration in seperate files + // TODO- Separate rat plugins and other plugins configuration in separate files rat { excludes = [ '**/.*/**', @@ -186,7 +210,9 @@ allprojects { } configurations { - testArtifacts.extendsFrom testRuntime + testArtifacts { + extendsFrom testRuntimeOnly + } } artifacts { @@ -195,6 +221,12 @@ allprojects { plugins.withType(DistributionPlugin) { distTar { + dependsOn(jar) + if (it.path.contains('bookkeeper-dist-all')) { + dependsOn(':bookkeeper-benchmark:jar') + dependsOn(':bookkeeper-common:jar') + dependsOn(':bookkeeper-common-allocator:jar') + } archiveClassifier = "bin" compression = Compression.GZIP archiveExtension = 'tar.gz' diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index cb38b87d24e..f91c4502a57 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -21,6 +21,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/tests/backward-compat/bc-non-fips/build.gradle b/tests/backward-compat/bc-non-fips/build.gradle index b87df1bc510..e321659662f 100644 --- a/tests/backward-compat/bc-non-fips/build.gradle +++ b/tests/backward-compat/bc-non-fips/build.gradle @@ -23,9 +23,14 @@ plugins { } dependencies { - + testImplementation project(':bookkeeper-common') + testImplementation(project(':bookkeeper-server')) { + exclude group: "org.bouncycastle", module: "bc-fips" + } testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') - testImplementation project(path: ':bookkeeper-server', configuration: 'testArtifacts') + testImplementation(project(path: ':bookkeeper-server', configuration: 'testArtifacts')) { + exclude group: "org.bouncycastle", module: "bc-fips" + } testImplementation depLibs.junit testImplementation depLibs.slf4j testImplementation depLibs.bcpkixJdk15on diff --git a/tests/integration/cluster/build.gradle b/tests/integration/cluster/build.gradle index 4cf0dac30d0..71a01ac3881 100644 --- a/tests/integration/cluster/build.gradle +++ b/tests/integration/cluster/build.gradle @@ -23,6 +23,7 @@ plugins { dependencies { implementation depLibs.grpc + testImplementation project(':bookkeeper-common') testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') testImplementation project(':stream:api') testImplementation project(':stream:storage:api') diff --git a/tests/integration/smoke/build.gradle b/tests/integration/smoke/build.gradle index 02ad13b25fb..b154f34a841 100644 --- a/tests/integration/smoke/build.gradle +++ b/tests/integration/smoke/build.gradle @@ -22,6 +22,7 @@ plugins { } dependencies { + testImplementation project(':bookkeeper-common') testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') testImplementation project(':bookkeeper-server') testImplementation project(':tests:integration-tests-utils') From 74f3d4b506fb8a69bd85150bf96d1613dc6e2e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 8 Feb 2022 14:17:10 +0100 Subject: [PATCH 55/79] [tests] remove backward compatibility test against yahoo-version (apache Pulsar 1.21) (#3028) --- .github/workflows/backward-compat-tests.yml | 10 +- .github/workflows/bookie-tests.yml | 5 +- .github/workflows/client-tests.yml | 5 +- .../workflows/compatibility-check-java11.yml | 5 +- .../workflows/compatibility-check-java8.yml | 5 +- .github/workflows/integration-tests.yml | 10 +- .github/workflows/pr-validation.yml | 6 +- .github/workflows/remaining-tests.yml | 6 +- .github/workflows/replication-tests.yml | 4 +- .github/workflows/stream-tests.yml | 19 +- .github/workflows/tls-tests.yml | 5 +- build.gradle | 2 +- gradle.properties | 2 +- settings.gradle | 1 - .../TestCompatUpgradeYahooCustom.groovy | 220 ------------------ .../src/test/resources/arquillian.xml | 28 --- .../all-released-versions-image/Dockerfile | 5 +- .../scripts/install-pulsar-tarball.sh | 41 ---- 18 files changed, 56 insertions(+), 323 deletions(-) delete mode 100644 tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy delete mode 100644 tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml delete mode 100644 tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh diff --git a/.github/workflows/backward-compat-tests.yml b/.github/workflows/backward-compat-tests.yml index 03f4e66e2da..8efa06f68cc 100644 --- a/.github/workflows/backward-compat-tests.yml +++ b/.github/workflows/backward-compat-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,10 +49,10 @@ jobs: with: java-version: 1.8 - name: Build - run: ./gradlew stream:server:build -x test + run: ./gradlew stream:server:build -x test ${GRADLE_ARGS} - name: Test current server with old clients - run: ./gradlew :tests:backward-compat:current-server-old-clients:test || (tail -n +1 tests/backward-compat/current-server-old-clients/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/current-server-old-clients/build/container-logs/**/* && exit 1) + run: ./gradlew :tests:backward-compat:current-server-old-clients:test ${GRADLE_ARGS} || (tail -n +1 tests/backward-compat/current-server-old-clients/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/current-server-old-clients/build/container-logs/**/* && exit 1) - name: Test progressive upgrade - run: ./gradlew :tests:backward-compat:upgrade:test || (tail -n +1 tests/backward-compat/upgrade/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/upgrade/build/container-logs/**/* && exit 1) + run: ./gradlew :tests:backward-compat:upgrade:test ${GRADLE_ARGS} || (tail -n +1 tests/backward-compat/upgrade/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/upgrade/build/container-logs/**/* && exit 1) - name: Other tests - run: ./gradlew :tests:backward-compat:test -x tests:backward-compat:upgrade:test -x :tests:backward-compat:current-server-old-clients:test + run: ./gradlew :tests:backward-compat:test -x tests:backward-compat:upgrade:test -x :tests:backward-compat:current-server-old-clients:test ${GRADLE_ARGS} diff --git a/.github/workflows/bookie-tests.yml b/.github/workflows/bookie-tests.yml index 34a70bfa85b..8eec299c5a0 100644 --- a/.github/workflows/bookie-tests.yml +++ b/.github/workflows/bookie-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -47,4 +50,4 @@ jobs: java-version: 1.8 - name: Run bookie test - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" -Dtestlogger.theme=plain \ No newline at end of file + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" ${GRADLE_ARGS} \ No newline at end of file diff --git a/.github/workflows/client-tests.yml b/.github/workflows/client-tests.yml index f8c2ada72a7..398278fc937 100644 --- a/.github/workflows/client-tests.yml +++ b/.github/workflows/client-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -46,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run client tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" ${GRADLE_ARGS} diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index e83ca17cbf5..7d93c50b90b 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: @@ -47,4 +50,4 @@ jobs: java-version: 1.11 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS} diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml index 82094fddf95..629fed6dac0 100644 --- a/.github/workflows/compatibility-check-java8.yml +++ b/.github/workflows/compatibility-check-java8.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: @@ -47,4 +50,4 @@ jobs: java-version: 1.8 - name: Build with gradle run: | - ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" + ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 229baeb9762..59c5128c5bc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -50,10 +50,10 @@ jobs: java-version: 1.8 - name: Build tar - run: ./gradlew stream:server:build -x test + run: ./gradlew stream:server:build -x test ${GRADLE_ARGS} - name: run cluster integration test - run: ./gradlew :tests:integration:cluster:test -Dtestlogger.theme=plain + run: ./gradlew :tests:integration:cluster:test ${GRADLE_ARGS} - name: run smoke test - run: ./gradlew tests:integration:smoke:test -Dtestlogger.theme=plain + run: ./gradlew tests:integration:smoke:test ${GRADLE_ARGS} - name: run standalone test - run: ./gradlew tests:integration:standalone:test -Dtestlogger.theme=plain + run: ./gradlew tests:integration:standalone:test ${GRADLE_ARGS} diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 816315b3707..0f4cba2c6b0 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: check: @@ -47,6 +50,7 @@ jobs: with: java-version: 1.8 - name: Validate pull request - run: ./gradlew build -x signDistTar -x test + run: ./gradlew build -x signDistTar -x test ${GRADLE_ARGS} + - name: Check license files run: dev/check-all-licenses diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml index 835d39ae3d9..a3f53f44297 100644 --- a/.github/workflows/remaining-tests.yml +++ b/.github/workflows/remaining-tests.yml @@ -29,6 +29,10 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + + jobs: test: @@ -46,4 +50,4 @@ jobs: with: java-version: 1.8 - name: Run remaining tests - run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain \ No newline at end of file + run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" ${GRADLE_ARGS} \ No newline at end of file diff --git a/.github/workflows/replication-tests.yml b/.github/workflows/replication-tests.yml index b1c0aeaeb7f..9664d8b822c 100644 --- a/.github/workflows/replication-tests.yml +++ b/.github/workflows/replication-tests.yml @@ -30,7 +30,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true jobs: test: @@ -49,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run replication tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" ${GRADLE_ARGS} diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml index 1cdb6461766..612b47bf1a8 100644 --- a/.github/workflows/stream-tests.yml +++ b/.github/workflows/stream-tests.yml @@ -28,6 +28,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -45,19 +48,19 @@ jobs: with: java-version: 1.8 - name: Run stream:distributedlog:core tests - run: ./gradlew stream:distributedlog:core:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:core:test ${GRADLE_ARGS} - name: Run stream:distributedlog:common tests - run: ./gradlew stream:distributedlog:common:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:common:test ${GRADLE_ARGS} - name: Run stream:distributedlog:protocol tests - run: ./gradlew stream:distributedlog:protocol:test -Dtestlogger.theme=plain + run: ./gradlew stream:distributedlog:protocol:test ${GRADLE_ARGS} - name: Run stream:proto tests - run: ./gradlew stream:proto:test -Dtestlogger.theme=plain + run: ./gradlew stream:proto:test ${GRADLE_ARGS} - name: Run stream:serve tests - run: ./gradlew stream:server:test -Dtestlogger.theme=plain + run: ./gradlew stream:server:test ${GRADLE_ARGS} - name: Run stream:statelib tests - run: ./gradlew stream:statelib:test -Dtestlogger.theme=plain + run: ./gradlew stream:statelib:test ${GRADLE_ARGS} - name: Run stream:storage:api:test tests - run: ./gradlew stream:storage:api:test -Dtestlogger.theme=plain + run: ./gradlew stream:storage:api:test ${GRADLE_ARGS} - name: Run stream:storage:impl tests - run: ./gradlew stream:storage:impl:test -Dtestlogger.theme=plain + run: ./gradlew stream:storage:impl:test ${GRADLE_ARGS} diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml index 3aeab4ac269..48f7f74d4d2 100644 --- a/.github/workflows/tls-tests.yml +++ b/.github/workflows/tls-tests.yml @@ -29,6 +29,9 @@ on: - 'site/**' workflow_dispatch: +env: + GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true + jobs: test: @@ -46,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run tls tests - run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" ${GRADLE_ARGS} diff --git a/build.gradle b/build.gradle index 6cba809a34f..ad5a9e11715 100644 --- a/build.gradle +++ b/build.gradle @@ -310,7 +310,7 @@ allprojects { systemProperty "currentVersion", project.rootProject.version testLogging { outputs.upToDateWhen {false} - showStandardStreams = true + showStandardStreams = !Boolean.getBoolean("testHideStandardOut") } } dependencies { diff --git a/gradle.properties b/gradle.properties index bba0e1dfa0a..58ef9bbd17f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -25,6 +25,6 @@ shadowPluginVersion=6.1.0 licenseGradlePluginVersion=0.15.0 checkStyleVersion=6.19 spotbugsPlugin=4.7.0 -testLogger=2.0.0 +testLogger=3.1.0 testRetry=1.0.0 owaspPlugin=6.5.3 diff --git a/settings.gradle b/settings.gradle index 036b10413e0..0e2455a9607 100644 --- a/settings.gradle +++ b/settings.gradle @@ -91,7 +91,6 @@ include(':bookkeeper-benchmark', 'tests:backward-compat:recovery-no-password', 'tests:backward-compat:upgrade', 'tests:backward-compat:upgrade-direct', - 'tests:backward-compat:yahoo-custom-version', 'tests:integration-tests-base', 'tests:integration-tests-topologies', 'tests:integration-tests-utils', diff --git a/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy b/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy deleted file mode 100644 index f1d024a61dc..00000000000 --- a/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy +++ /dev/null @@ -1,220 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.bookkeeper.tests.backwardcompat - -import static java.nio.charset.StandardCharsets.UTF_8 - -import com.github.dockerjava.api.DockerClient - -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutionException -import java.util.concurrent.TimeUnit -import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils -import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader - -import org.jboss.arquillian.junit.Arquillian -import org.jboss.arquillian.test.api.ArquillianResource - -import org.junit.Assert -import org.junit.Test -import org.junit.runner.RunWith - -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -@RunWith(Arquillian.class) -class TestCompatUpgradeYahooCustom { - private static final Logger LOG = LoggerFactory.getLogger(TestCompatUpgradeYahooCustom.class) - private static byte[] PASSWD = "foobar".getBytes() - - def yahooRepo = "https://raw.githubusercontent.com/yahoo/bookkeeper/mvn-repo" - def yahooArtifact = "org.apache.bookkeeper:bookkeeper-server:4.3.1.85-yahoo" - - @ArquillianResource - DockerClient docker - - int numEntries = 1 - - def yahooConfiguredBookKeeper(def classLoader, def zookeeper) { - // bookkeeper client configured in same way as pulsar configures it - def bkConf = classLoader.newInstance("org.apache.bookkeeper.conf.ClientConfiguration") - - bkConf.setThrottleValue(0) - bkConf.setAddEntryTimeout(30) - bkConf.setReadEntryTimeout(30) - bkConf.setSpeculativeReadTimeout(0) - bkConf.setNumChannelsPerBookie(16) - bkConf.setUseV2WireProtocol(true) - // we can't specify the class name, because it would try to use the thread - // context classloader to load, which doesn't have the required classes loaded. - bkConf.setLedgerManagerType("hierarchical") - - bkConf.enableBookieHealthCheck() - bkConf.setBookieHealthCheckInterval(60, TimeUnit.SECONDS) - bkConf.setBookieErrorThresholdPerInterval(5) - bkConf.setBookieQuarantineTime(1800, TimeUnit.SECONDS) - - bkConf.setZkServers(zookeeper) - return classLoader.newInstance("org.apache.bookkeeper.client.BookKeeper", bkConf) - } - - def addEntry(def classLoader, def ledger, def entryData) { - def promise = new CompletableFuture() - def buffer = classLoader.callStaticMethod("io.netty.buffer.Unpooled", "copiedBuffer", - [entryData.getBytes(UTF_8)]) - def callback = classLoader.createCallback( - "org.apache.bookkeeper.client.AsyncCallback\$AddCallback", - { rc, _ledger, entryId, ctx -> - if (rc != 0) { - promise.completeExceptionally( - classLoader.callStaticMethod("org.apache.bookkeeper.client.BKException", - "create", [rc])) - } else { - promise.complete(entryId) - } - }) - ledger.asyncAddEntry(buffer, callback, null) - return promise - } - - def assertCantWrite(def cl, def ledger) throws Exception { - try { - addEntry(cl, ledger, "Shouldn't write").get() - fail("Shouldn't be able to write to ledger") - } catch (ExecutionException e) { - // correct behaviour - // TODO add more - } - } - def createAndWrite(def cl, def bk) throws Exception { - def ledger = bk.createLedger(3, 2, cl.digestType("CRC32"), PASSWD) - LOG.info("Created ledger ${ledger.getId()}") - for (int i = 0; i < numEntries; i++) { - addEntry(cl, ledger, "foobar" + i).get() - } - return ledger - } - - def openAndVerifyEntries(def cl, def bk, def ledgerId) throws Exception { - LOG.info("Opening ledger $ledgerId") - def ledger = bk.openLedger(ledgerId, cl.digestType("CRC32"), PASSWD) - def entries = ledger.readEntries(0, ledger.getLastAddConfirmed()) - int j = 0 - while (entries.hasMoreElements()) { - def e = entries.nextElement() - Assert.assertEquals(new String(e.getEntry()), "foobar"+ j) - j++ - } - ledger.close() - } - - def List exerciseClients(List toVerify) { - String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker) - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - - def ledgers = [] - def yahooCL = MavenClassLoader.forArtifact(yahooRepo, yahooArtifact) - def yahooBK = yahooConfiguredBookKeeper(yahooCL, zookeeper) - def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion) - def currentBK = yahooConfiguredBookKeeper(currentCL, zookeeper) - try { - // verify we can read ledgers from previous run - for (Long id : toVerify) { - LOG.info("Verifying $id with yahoo client") - openAndVerifyEntries(yahooCL, yahooBK, id) - LOG.info("Verifying $id with current client") - openAndVerifyEntries(currentCL, currentBK, id) - } - // yahoo client and create open and read - def ledger0 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger0.getId()) - ledger0.close() - openAndVerifyEntries(yahooCL, yahooBK, ledger0.getId()) - - // yahoo client can fence on yahoo bookies - def ledger1 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger1.getId()) - openAndVerifyEntries(yahooCL, yahooBK, ledger1.getId()) - assertCantWrite(yahooCL, ledger1) - - // current client can create open and read - def ledger2 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger2.getId()) - ledger2.close() - openAndVerifyEntries(currentCL, currentBK, ledger2.getId()) - - // current client can fence on yahoo bookies - def ledger3 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger3.getId()) - openAndVerifyEntries(currentCL, currentBK, ledger3.getId()) - assertCantWrite(currentCL, ledger3) - - // current client can fence a bookie created by yahoo client - def ledger4 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger4.getId()) - openAndVerifyEntries(currentCL, currentBK, ledger4.getId()) - assertCantWrite(yahooCL, ledger4) - - // Since METADATA_VERSION is upgraded and it is using binary format, the older - // clients which are expecting text format would fail to read ledger metadata. - def ledger5 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger5.getId()) - try { - openAndVerifyEntries(yahooCL, yahooBK, ledger5.getId()) - } catch (Exception exc) { - Assert.assertEquals(exc.getClass().getName(), - "org.apache.bookkeeper.client.BKException\$ZKException") - } - } finally { - currentBK.close() - currentCL.close() - yahooBK.close() - yahooCL.close() - } - return ledgers - } - - @Test - public void testUpgradeYahooCustom() throws Exception { - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - String yahooVersion = "4.3-yahoo" - BookKeeperClusterUtils.metadataFormatIfNeeded(docker, yahooVersion) - - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, yahooVersion)) - def preUpgradeLedgers = exerciseClients([]) - Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)) - BookKeeperClusterUtils.runOnAllBookies( - docker, "cp", "/opt/bookkeeper/${yahooVersion}/conf/bookkeeper.conf", - "/opt/bookkeeper/${currentVersion}/conf/bk_server.conf") - BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, - "logSizeLimit", "1073741824") - BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, - "statsProviderClass", - "org.apache.bookkeeper.stats.NullStatsProvider") - - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion)) - // Since METADATA_VERSION is upgraded and it is using binary format, the older - // clients which are expecting text format would fail to read ledger metadata. - try { - exerciseClients(preUpgradeLedgers) - } catch (Exception exc) { - Assert.assertEquals(exc.getClass().getName(), - "org.apache.bookkeeper.client.BKException\$ZKException") - } - } -} diff --git a/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml b/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml deleted file mode 100644 index f914ff2c258..00000000000 --- a/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - CUBE - cube-definitions/3-node-all-version-unstarted.yaml - - - diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index af2d9651de8..aee3227ecd1 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -40,17 +40,14 @@ RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookke RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.4/bookkeeper-server-4.14.4-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/pulsar-1.21.0-incubating/apache-pulsar-1.21.0-incubating-bin.tar.gz{,.asc} RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS RUN wget -nv http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552 -O KEYS.old -RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/KEYS -O KEYS.pulsar RUN mkdir -p /etc/supervisord/conf.d && mkdir -p /var/run/supervisor && mkdir -p /var/log/bookkeeper ADD conf/supervisord.conf /etc/supervisord.conf ADD scripts/install-all-tarballs.sh /install-all-tarballs.sh ADD scripts/install-tarball.sh /install-tarball.sh -ADD scripts/install-pulsar-tarball.sh /install-pulsar-tarball.sh -RUN bash /install-all-tarballs.sh && bash /install-pulsar-tarball.sh && rm -rf /tarballs +RUN bash /install-all-tarballs.sh && rm -rf /tarballs WORKDIR / ADD scripts/update-conf-and-boot.sh /update-conf-and-boot.sh diff --git a/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh b/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh deleted file mode 100644 index 68fdd0437aa..00000000000 --- a/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -set -e - -gpg --import KEYS.pulsar - -TARBALL=$(ls apache-pulsar-*-incubating-bin.tar.gz | tail -1) -gpg --verify $TARBALL.asc - -tar -zxf $TARBALL -rm $TARBALL -VERSION=4.3-yahoo - -mv apache-pulsar-*-incubating /opt/bookkeeper/$VERSION - -cat > /etc/supervisord/conf.d/bookkeeper-$VERSION.conf < Date: Wed, 9 Feb 2022 09:54:54 +0800 Subject: [PATCH 56/79] Support specifying bookie http port as a command argument (#2769) ### Motivation I was trying to start multiple bookies locally and found it's a bit inconvenient to specify different http ports for different bookies. ### Changes Add a command-line argument `httpport` to the bookie command to support specifying bookie http port from the command line. --- .../src/main/java/org/apache/bookkeeper/server/Main.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java index 3c7d7fe7cfd..ddc8613acd2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java @@ -116,6 +116,7 @@ public class Main { BK_OPTS.addOption("z", "zkserver", true, "Zookeeper Server"); BK_OPTS.addOption("m", "zkledgerpath", true, "Zookeeper ledgers root path"); BK_OPTS.addOption("p", "bookieport", true, "bookie port exported"); + BK_OPTS.addOption("hp", "httpport", true, "bookie http port exported"); BK_OPTS.addOption("j", "journal", true, "bookie journal directory"); Option indexDirs = new Option ("i", "indexdirs", true, "bookie index directories"); indexDirs.setArgs(10); @@ -212,6 +213,13 @@ private static ServerConfiguration parseArgs(String[] args) conf.setBookiePort(Integer.parseInt(sPort)); } + if (cmdLine.hasOption("httpport")) { + String sPort = cmdLine.getOptionValue("httpport"); + log.info("Get cmdline http port: {}", sPort); + Integer iPort = Integer.parseInt(sPort); + conf.setHttpServerPort(iPort.intValue()); + } + if (cmdLine.hasOption('j')) { String sJournalDir = cmdLine.getOptionValue('j'); log.info("Get cmdline journal dir: {}", sJournalDir); From 7907a590e3094b75bb550796bae68859347944f2 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Thu, 10 Feb 2022 03:24:21 +0800 Subject: [PATCH 57/79] Fix performance issue to avoid unnessary loop. (#3030) --- .../main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java index c047977c383..be55ca18f3f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java @@ -95,6 +95,7 @@ public void onProtocolUpgrade() { for (String allowedRole : allowedRoles) { if (certRole[0].equals(allowedRole)) { authorized = true; + break; } } if (authorized) { From f4eb77d8e05028c0c77c2dc0cc43beb3313bc677 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Wed, 9 Feb 2022 13:13:44 -0800 Subject: [PATCH 58/79] Upgrade RocksDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Descriptions of the changes in this PR: Dependency change ### Motivation I encountered https://github.com/apache/bookkeeper/issues/3024 and noticed that newer version of RocksDB includes multiple fixes for concurrency issues with various side-effects and fixes for a few crashes. I upgraded, ran `org.apache.bookkeeper.bookie.BookieJournalTest` test in a loop and didn't repro the crash so far. It is hard to say 100% if it is fixed given it was not happening all the time. ### Changes Upgraded RocksDB Master Issue: #3024 Reviewers: Enrico Olivelli , Nicolò Boschi This closes #3026 from dlg99/rocksdb-upgrade --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 4 ++-- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 4 ++-- dependencies.gradle | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index adfb859d517..03ee387e733 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -260,7 +260,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.43.v20210629.jar [22] -- lib/org.rocksdb-rocksdbjni-6.22.1.1.jar [23] +- lib/org.rocksdb-rocksdbjni-6.27.3.jar [23] - lib/com.beust-jcommander-1.78.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -586,7 +586,7 @@ This private header is also used by Apple's open source * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-6.22.1.1.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-6.27.3.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index a00cd92b9a9..df8b99e57c6 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -260,7 +260,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.43.v20210629.jar [22] -- lib/org.rocksdb-rocksdbjni-6.22.1.1.jar [23] +- lib/org.rocksdb-rocksdbjni-6.27.3.jar [23] - lib/com.beust-jcommander-1.78.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -584,7 +584,7 @@ This private header is also used by Apple's open source * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-6.22.1.1.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-6.27.3.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/dependencies.gradle b/dependencies.gradle index b822b63af62..e1c297052ea 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -72,7 +72,7 @@ depVersions = [ prometheus: "0.8.1", protobuf: "3.16.1", reflections: "0.9.11", - rocksDb: "6.22.1.1", + rocksDb: "6.27.3", rxjava: "3.0.1", slf4j: "1.7.32", snakeyaml: "1.30", From b623fd46b4bd2b251896a387bf66d984363f3979 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Fri, 11 Feb 2022 11:10:28 +0800 Subject: [PATCH 59/79] Skip update entryLogMetaMap if not modified (#2964) ### Motivation After we support RocksDB backend entryMetaMap, we should avoid updating the entryMetaMap if unnecessary. In `doGcEntryLogs` method, it iterate through the entryLogMetaMap and update the meta if ledgerNotExists. We should check whether the meta has been modified in `removeIfLedgerNotExists`. If not modified, we can avoid update the entryLogMetaMap. ### Modification 1. Add a flag to represent whether the meta has been modified in `removeIfLedgerNotExists` method. If not, skip update the entryLogMetaMap. --- .../bookie/GarbageCollectorThread.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index bf00566f1b3..8e46a630d4c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -45,6 +45,7 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.util.MathUtils; import org.apache.bookkeeper.util.SafeRunnable; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -438,9 +439,7 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { // Loop through all of the entry logs and remove the non-active ledgers. entryLogMetaMap.forEach((entryLogId, meta) -> { try { - removeIfLedgerNotExists(meta); - // update entryMetadta to persistent-map - entryLogMetaMap.put(meta.getEntryLogId(), meta); + boolean modified = removeIfLedgerNotExists(meta); if (meta.isEmpty()) { // This means the entry log is not associated with any active // ledgers anymore. @@ -448,6 +447,9 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { LOG.info("Deleting entryLogId {} as it has no active ledgers!", entryLogId); removeEntryLog(entryLogId); gcStats.getReclaimedSpaceViaDeletes().add(meta.getTotalSize()); + } else if (modified) { + // update entryLogMetaMap only when the meta modified. + entryLogMetaMap.put(meta.getEntryLogId(), meta); } } catch (EntryLogMetadataMapException e) { // Ignore and continue because ledger will not be cleaned up @@ -462,16 +464,23 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { this.numActiveEntryLogs = entryLogMetaMap.size(); } - private void removeIfLedgerNotExists(EntryLogMetadata meta) throws EntryLogMetadataMapException { + private boolean removeIfLedgerNotExists(EntryLogMetadata meta) throws EntryLogMetadataMapException { + MutableBoolean modified = new MutableBoolean(false); meta.removeLedgerIf((entryLogLedger) -> { // Remove the entry log ledger from the set if it isn't active. try { - return !ledgerStorage.ledgerExists(entryLogLedger); + boolean exist = ledgerStorage.ledgerExists(entryLogLedger); + if (!exist) { + modified.setTrue(); + } + return !exist; } catch (IOException e) { LOG.error("Error reading from ledger storage", e); return false; } }); + + return modified.getValue(); } /** From 874f038a6a01d42e2c8effb30773912b85a4e6ec Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Fri, 11 Feb 2022 11:11:03 +0800 Subject: [PATCH 60/79] Add rack name invalid check (#2980) ### Motivation When we set region or rack placement policy, but the region or rack name set to `/` or empty string, it will throw the following exception on handling bookies join. ``` java.lang.StringIndexOutOfBoundsException: String index out of range: -1 at java.lang.String.substring(String.java:1841) ~[?:?] at org.apache.bookkeeper.net.NetworkTopologyImpl$InnerNode.getNextAncestorName(NetworkTopologyImpl.java:144) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.net.NetworkTopologyImpl$InnerNode.add(NetworkTopologyImpl.java:180) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.net.NetworkTopologyImpl.add(NetworkTopologyImpl.java:425) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy.handleBookiesThatJoined(TopologyAwareEnsemblePlacementPolicy.java:717) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.handleBookiesThatJoined(RackawareEnsemblePlacementPolicyImpl.java:80) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy.handleBookiesThatJoined(RackawareEnsemblePlacementPolicy.java:249) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy.onClusterChanged(TopologyAwareEnsemblePlacementPolicy.java:663) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.onClusterChanged(RackawareEnsemblePlacementPolicyImpl.java:80) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy.onClusterChanged(RackawareEnsemblePlacementPolicy.java:92) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.BookieWatcherImpl.processWritableBookiesChanged(BookieWatcherImpl.java:197) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.client.BookieWatcherImpl.lambda$initialBlockingBookieRead$1(BookieWatcherImpl.java:233) ~[io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.discover.ZKRegistrationClient$WatchTask.accept(ZKRegistrationClient.java:147) [io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at org.apache.bookkeeper.discover.ZKRegistrationClient$WatchTask.accept(ZKRegistrationClient.java:70) [io.streamnative-bookkeeper-server-4.14.3.1.jar:4.14.3.1] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) [?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) [?:?] at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:478) [?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.72.Final.jar:4.1.72.Final] at java.lang.Thread.run(Thread.java:829) [?:?] ``` The root cause is that the node networkLocation is empty string and then use `substring(1)` operation, which will lead to `StringIndexOutOfBoundsException` ### Modification 1. Add `n.getNetworkLocation()` is empty check on `isAncestor` method to make the exception more clear. --- .../bookkeeper/net/NetworkTopologyImpl.java | 6 ++-- .../net/NetworkTopologyImplTest.java | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java index 123f13ca8a3..19b2c0497f7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java @@ -17,6 +17,7 @@ */ package org.apache.bookkeeper.net; +import com.google.common.base.Strings; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -119,9 +120,10 @@ boolean isRack() { * @return true if this node is an ancestor of n */ boolean isAncestor(Node n) { - return getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) + return !Strings.isNullOrEmpty(n.getNetworkLocation()) + && (getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) || (n.getNetworkLocation() + NodeBase.PATH_SEPARATOR_STR).startsWith(getPath(this) - + NodeBase.PATH_SEPARATOR_STR); + + NodeBase.PATH_SEPARATOR_STR)); } /** diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java index a6cc8c330a3..6122e51f0cd 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java @@ -17,7 +17,9 @@ */ package org.apache.bookkeeper.net; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.Set; import org.junit.Test; @@ -97,4 +99,31 @@ public void getLeavesShouldReturnLeavesThatAreNotInExcludedScope() { assertTrue(leavesExcludingRack2Scope.contains(bookieRack0ScopeNode)); assertTrue(leavesExcludingRack2Scope.contains(bookieRack2ScopeNode)); } + + @Test + public void testInvalidRackName() { + NetworkTopologyImpl networkTopology = new NetworkTopologyImpl(); + String rack0Scope = ""; + BookieId bookieIdScopeRack0 = BookieId.parse("bookieIdScopeRack0"); + BookieNode bookieRack0ScopeNode = new BookieNode(bookieIdScopeRack0, rack0Scope); + + String rack1Scope = "/"; + BookieId bookieIdScopeRack1 = BookieId.parse("bookieIdScopeRack1"); + BookieNode bookieRack1ScopeNode = new BookieNode(bookieIdScopeRack1, rack1Scope); + + try { + networkTopology.add(bookieRack0ScopeNode); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("bookieIdScopeRack0, which is located at , is not a decendent of /", e.getMessage()); + } + + try { + networkTopology.add(bookieRack1ScopeNode); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("bookieIdScopeRack1, which is located at , is not a decendent of /", e.getMessage()); + } + + } } \ No newline at end of file From 2a63101120c0280d7cfb83b96972d035f7fbbc52 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Fri, 11 Feb 2022 11:12:27 +0800 Subject: [PATCH 61/79] Support multi ledger directories for rocksdb backend entryMetadataMap (#2965) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we use RocksDB backend entryMetadataMap for multi ledger directories configured, the bookie start up failed, and throw the following exception. ``` 12:24:28.530 [main] ERROR org.apache.pulsar.PulsarStandaloneStarter - Failed to start pulsar service. java.io.IOException: Error open RocksDB database at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:202) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:89) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.lambda$static$0(KeyValueStorageRocksDB.java:62) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.PersistentEntryLogMetadataMap.(PersistentEntryLogMetadataMap.java:87) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.GarbageCollectorThread.createEntryLogMetadataMap(GarbageCollectorThread.java:265) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.GarbageCollectorThread.(GarbageCollectorThread.java:154) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.GarbageCollectorThread.(GarbageCollectorThread.java:133) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage.(SingleDirectoryDbLedgerStorage.java:182) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.newSingleDirectoryDbLedgerStorage(DbLedgerStorage.java:190) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.initialize(DbLedgerStorage.java:150) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.bookkeeper.bookie.BookieResources.createLedgerStorage(BookieResources.java:110) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] at org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.buildBookie(LocalBookkeeperEnsemble.java:328) ~[org.apache.pulsar-pulsar-zookeeper-utils-2.8.1.jar:2.8.1] at org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.runBookies(LocalBookkeeperEnsemble.java:391) ~[org.apache.pulsar-pulsar-zookeeper-utils-2.8.1.jar:2.8.1] at org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.startStandalone(LocalBookkeeperEnsemble.java:521) ~[org.apache.pulsar-pulsar-zookeeper-utils-2.8.1.jar:2.8.1] at org.apache.pulsar.PulsarStandalone.start(PulsarStandalone.java:264) ~[org.apache.pulsar-pulsar-broker-2.8.1.jar:2.8.1] at org.apache.pulsar.PulsarStandaloneStarter.main(PulsarStandaloneStarter.java:121) [org.apache.pulsar-pulsar-broker-2.8.1.jar:2.8.1] Caused by: org.rocksdb.RocksDBException: lock hold by current process, acquire time 1640492668 acquiring thread 123145515651072: data/standalone/bookkeeper00/entrylogIndexCache/metadata-cache/LOCK: No locks available at org.rocksdb.RocksDB.open(Native Method) ~[org.rocksdb-rocksdbjni-6.10.2.jar:?] at org.rocksdb.RocksDB.open(RocksDB.java:239) ~[org.rocksdb-rocksdbjni-6.10.2.jar:?] at org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.(KeyValueStorageRocksDB.java:199) ~[org.apache.bookkeeper-bookkeeper-server-4.15.0-SNAPSHOT.jar:4.15.0-SNAPSHOT] ... 15 more ``` The reason is multi garbageCollectionThread will open the same RocksDB and own the LOCK, and then throw the above exception. 1. Change the default GcEntryLogMetadataCachePath from `getLedgerDirNames()[0] + "/" + ENTRYLOG_INDEX_CACHE` to `null`. If it is `null`, it will use each ledger's directory. 2. Remove the internal directory `entrylogIndexCache`. The data structure looks like: ```    └── current    ├── lastMark    ├── ledgers    │   ├── 000003.log    │   ├── CURRENT    │   ├── IDENTITY    │   ├── LOCK    │   ├── LOG    │   ├── MANIFEST-000001    │   └── OPTIONS-000005    ├── locations    │   ├── 000003.log    │   ├── CURRENT    │   ├── IDENTITY    │   ├── LOCK    │   ├── LOG    │   ├── MANIFEST-000001    │   └── OPTIONS-000005    └── metadata-cache    ├── 000003.log    ├── CURRENT    ├── IDENTITY    ├── LOCK    ├── LOG    ├── MANIFEST-000001    └── OPTIONS-000005 ``` 3. If user configured `GcEntryLogMetadataCachePath` in `bk_server.conf`, it only support one ledger directory configured for `ledgerDirectories`. Otherwise, the best practice is to keep it default. 4. The PR is better to release with #1949 --- .../apache/bookkeeper/bookie/BookieImpl.java | 11 +++++++---- .../bookie/GarbageCollectorThread.java | 17 +++++++++++++---- .../bookie/InterleavedLedgerStorage.java | 3 ++- .../ldb/PersistentEntryLogMetadataMap.java | 7 ++++--- .../ldb/SingleDirectoryDbLedgerStorage.java | 2 +- .../bookkeeper/conf/ServerConfiguration.java | 12 +++++++----- .../bookkeeper/util/BookKeeperConstants.java | 2 +- 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java index f0a002f4583..94b75254fd0 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java @@ -25,6 +25,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_LEDGER_SCOPE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -1192,10 +1193,12 @@ public static boolean format(ServerConfiguration conf, // Clean up metadata directories if they are separate from the // ledger dirs - File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); - if (!cleanDir(metadataDir)) { - LOG.error("Formatting ledger metadata directory {} failed", metadataDir); - return false; + if (!Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath())) { + File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); + if (!cleanDir(metadataDir)) { + LOG.error("Formatting ledger metadata directory {} failed", metadataDir); + return false; + } } LOG.info("Bookie format completed successfully"); return true; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index 8e46a630d4c..123ab14f17d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -21,7 +21,9 @@ package org.apache.bookkeeper.bookie; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; @@ -121,6 +123,7 @@ public class GarbageCollectorThread extends SafeRunnable { final GarbageCleaner garbageCleaner; final ServerConfiguration conf; + final LedgerDirsManager ledgerDirsManager; /** * Create a garbage collector thread. @@ -130,8 +133,10 @@ public class GarbageCollectorThread extends SafeRunnable { * @throws IOException */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, - final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger) throws IOException { - this(conf, ledgerManager, ledgerStorage, statsLogger, + final LedgerDirsManager ledgerDirsManager, + final CompactableLedgerStorage ledgerStorage, + StatsLogger statsLogger) throws IOException { + this(conf, ledgerManager, ledgerDirsManager, ledgerStorage, statsLogger, Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollectorThread"))); } @@ -144,6 +149,7 @@ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerMana */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, + final LedgerDirsManager ledgerDirsManager, final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger, ScheduledExecutorService gcExecutor) @@ -151,6 +157,7 @@ public GarbageCollectorThread(ServerConfiguration conf, this.gcExecutor = gcExecutor; this.conf = conf; + this.ledgerDirsManager = ledgerDirsManager; this.entryLogger = ledgerStorage.getEntryLogger(); this.entryLogMetaMap = createEntryLogMetadataMap(); this.ledgerStorage = ledgerStorage; @@ -261,11 +268,13 @@ public void removeEntryLog(long logToRemove) { private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { if (conf.isGcEntryLogMetadataCacheEnabled()) { - String baseDir = this.conf.getGcEntryLogMetadataCachePath(); + String baseDir = Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath()) + ? this.ledgerDirsManager.getAllLedgerDirs().get(0).getPath() : conf.getGcEntryLogMetadataCachePath(); try { return new PersistentEntryLogMetadataMap(baseDir, conf); } catch (IOException e) { - LOG.error("Failed to initialize persistent-metadata-map , clean up {}", baseDir, e); + LOG.error("Failed to initialize persistent-metadata-map , clean up {}", + baseDir + "/" + METADATA_CACHE, e); throw e; } } else { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java index dcfeed0bf3c..0960c6658ca 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java @@ -193,7 +193,8 @@ public void initializeWithEntryLogger(ServerConfiguration conf, this.entryLogger.addListener(this); ledgerCache = new LedgerCacheImpl(conf, activeLedgers, null == indexDirsManager ? ledgerDirsManager : indexDirsManager, statsLogger); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger.scope("gc")); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, + this, statsLogger.scope("gc")); pageSize = conf.getPageSize(); ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener()); // Expose Stats diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java index 8f1faada69d..95b365970f9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java @@ -20,6 +20,7 @@ */ package org.apache.bookkeeper.bookie.storage.ldb; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import io.netty.util.concurrent.FastThreadLocal; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -77,15 +78,15 @@ protected DataInputStream initialValue() { }; public PersistentEntryLogMetadataMap(String metadataPath, ServerConfiguration conf) throws IOException { - LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath); + LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath + "/" + METADATA_CACHE); File dir = new File(metadataPath); if (!dir.mkdirs() && !dir.exists()) { String err = "Unable to create directory " + dir; LOG.error(err); throw new IOException(err); } - metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, "metadata-cache", - DbConfigType.Default, conf); + metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, METADATA_CACHE, + DbConfigType.Small, conf); } @Override diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 4d2e4adafc4..ce8af36b4e0 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -183,7 +183,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES, TimeUnit.MINUTES); entryLogger = new EntryLogger(conf, ledgerDirsManager, null, statsLogger, allocator); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, this, statsLogger); dbLedgerStorageStats = new DbLedgerStorageStats( ledgerDirStatsLogger, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 545b6ebe9cb..144c415219d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -17,7 +17,6 @@ */ package org.apache.bookkeeper.conf; -import static org.apache.bookkeeper.util.BookKeeperConstants.ENTRYLOG_INDEX_CACHE; import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; import com.google.common.annotations.Beta; @@ -524,17 +523,20 @@ public ServerConfiguration setGcEntryLogMetadataCacheEnabled( * gcPersistentEntrylogMetadataMapEnabled is true. * * @return entrylog metadata-map persistent store dir path.(default: it - * creates a sub-directory under a first available base ledger - * directory with name "entrylogIndexCache"). + * creates a sub-directory under each ledger + * directory with name "metadata-cache". If it set, it only works for one ledger directory + * configured for ledgerDirectories). */ public String getGcEntryLogMetadataCachePath() { - return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, getLedgerDirNames()[0] + "/" + ENTRYLOG_INDEX_CACHE); + return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, null); } /** * Set directory to persist Entrylog metadata if gcPersistentEntrylogMetadataMapEnabled is true. + * If it set, it only works for one ledger directory configured for ledgerDirectories. For multi ledgerDirectory + * configured, keep the default value is the best practice. * - * @param gcPersistentEntrylogMetadataMapPath. + * @param gcEntrylogMetadataCachePath * @return server configuration. */ public ServerConfiguration setGcEntryLogMetadataCachePath(String gcEntrylogMetadataCachePath) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java index 8e10a151d55..d928aac2fc5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java @@ -31,7 +31,7 @@ public class BookKeeperConstants { public static final String BOOKIE_STATUS_FILENAME = "BOOKIE_STATUS"; public static final String PASSWD = "passwd"; public static final String CURRENT_DIR = "current"; - public static final String ENTRYLOG_INDEX_CACHE = "entrylogIndexCache"; + public static final String METADATA_CACHE = "metadata-cache"; public static final String READONLY = "readonly"; // ////////////////////////// From 0c11c738f5eb4435b5ba200706620d7c424d5145 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 10 Feb 2022 23:16:30 -0800 Subject: [PATCH 62/79] [ISSUE 3038] Fixed flaky CompactionTest.testMinorCompactionWithMaxTimeMillis (#3039) --- .../bookkeeper/bookie/CompactionTest.java | 71 ++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java index 1c2ee73f11d..aef181f2482 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java @@ -402,7 +402,7 @@ public void testMinorCompaction() throws Exception { } @Test - public void testMinorCompactionWithMaxTimeMillis() throws Exception { + public void testMinorCompactionWithMaxTimeMillisOk() throws Exception { // prepare data LedgerHandle[] lhs = prepareData(6, false); @@ -419,8 +419,10 @@ public void testMinorCompactionWithMaxTimeMillis() throws Exception { c.setMajorCompactionInterval(240000); // Setup limit on compaction duration. - c.setMinorCompactionMaxTimeMillis(15); - c.setMajorCompactionMaxTimeMillis(15); + // The limit is enough to compact. + c.setMinorCompactionMaxTimeMillis(5000); + c.setMajorCompactionMaxTimeMillis(5000); + return c; }); @@ -477,7 +479,70 @@ public void testMinorCompactionWithMaxTimeMillis() throws Exception { } + @Test + public void testMinorCompactionWithMaxTimeMillisTooShort() throws Exception { + // prepare data + LedgerHandle[] lhs = prepareData(6, false); + + for (LedgerHandle lh : lhs) { + lh.close(); + } + + // disable major compaction + // restart bookies + restartBookies(c-> { + c.setMajorCompactionThreshold(0.0f); + c.setGcWaitTime(60000); + c.setMinorCompactionInterval(120000); + c.setMajorCompactionInterval(240000); + + // Setup limit on compaction duration. + // The limit is not enough to finish the compaction + c.setMinorCompactionMaxTimeMillis(1); + c.setMajorCompactionMaxTimeMillis(1); + + return c; + }); + + getGCThread().enableForceGC(); + getGCThread().triggerGC().get(); + assertTrue( + "ACTIVE_ENTRY_LOG_COUNT should have been updated", + getStatsProvider(0) + .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_COUNT) + .getSample().intValue() > 0); + assertTrue( + "ACTIVE_ENTRY_LOG_SPACE_BYTES should have been updated", + getStatsProvider(0) + .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_SPACE_BYTES) + .getSample().intValue() > 0); + + long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime; + long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime; + assertFalse(getGCThread().enableMajorCompaction); + assertTrue(getGCThread().enableMinorCompaction); + + // remove ledger2 and ledger3 + bkc.deleteLedger(lhs[1].getId()); + bkc.deleteLedger(lhs[2].getId()); + LOG.info("Finished deleting the ledgers contains most entries."); + getGCThread().enableForceGC(); + getGCThread().triggerGC().get(); + + // after garbage collection, major compaction should not be executed + assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime); + assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime); + + // entry logs ([0,1,2].log) should be compacted. + for (File ledgerDirectory : tmpDirs.getDirs()) { + // Compaction of at least one of the files should not finish up + assertTrue("Not found entry log file ([0,1,2].log that should not have been compacted in ledgerDirectory: " + + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2)); + } + + verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed()); + } @Test public void testForceMinorCompaction() throws Exception { From cf6eeeafdcd897d63388fd789ef75fac3e3d92b2 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Mon, 14 Feb 2022 10:39:23 +0800 Subject: [PATCH 63/79] change log level from error to warn when dns resolver initialize failed (#2856) Descriptions of the changes in this PR: ### Motivation When start bookie, it will throws the following error message when dns resolver initialize failed. ``` [main] ERROR org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl - Failed to initialize DNS Resolver org.apache.bookkeeper.net.ScriptBasedMapping, used default subnet resolver : java.lang.RuntimeException: No network topology script is found when using script based DNS resolver. ``` It is confusing for users. ### Modification 1. change the log level from error to warn. --- .../bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 50297d121c8..46c5a10786b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -269,7 +269,7 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, } } catch (RuntimeException re) { if (!conf.getEnforceMinNumRacksPerWriteQuorum()) { - LOG.error("Failed to initialize DNS Resolver {}, used default subnet resolver ", + LOG.warn("Failed to initialize DNS Resolver {}, used default subnet resolver ", dnsResolverName, re); dnsResolver = new DefaultResolver(this::getDefaultRack); dnsResolver.setBookieAddressResolver(bookieAddressResolver); From 19979346dc3af4f7fff5b013e2fa0cb166384aaa Mon Sep 17 00:00:00 2001 From: Jack Vanlightly Date: Mon, 14 Feb 2022 19:56:19 +0100 Subject: [PATCH 64/79] Ensure BookKeeper process receives sigterm in docker container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation Current official docker images do not handle the SIGTERM sent by the docker runtime and so get killed after the timeout. No graceful shutdown occurs. The reason is that the entrypoint does not use `exec` when executing the `bin/bookkeeper` shell script and so the BookKeeper process cannot receive signals from the docker runtime. ### Changes Use `exec` when calling the `bin/bookkeeper` shell script. Reviewers: Nicolò Boschi , Enrico Olivelli , Lari Hotari , Matteo Merli This closes #2857 from Vanlightly/docker-image-handle-sigterm --- docker/scripts/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index 657eb6b918e..753f59a20db 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -41,11 +41,11 @@ function run_command() { chmod -R +x ${BINDIR} chmod -R +x ${SCRIPTS_DIR} echo "This is root, will use user $BK_USER to run command '$@'" - sudo -s -E -u "$BK_USER" /bin/bash "$@" + exec sudo -s -E -u "$BK_USER" /bin/bash -c 'exec "$@"' -- "$@" exit else echo "Run command '$@'" - $@ + exec "$@" fi } From 028cf0e990b1fc2bce83628f1af64a19fb69db2a Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Tue, 15 Feb 2022 03:00:06 +0800 Subject: [PATCH 65/79] make rocksdb format version configurable Fix #2823 RocksDB support several format versions which uses different data structure to implement key-values indexes and have huge different performance. https://rocksdb.org/blog/2019/03/08/format-version-4.html https://github.com/facebook/rocksdb/blob/d52b520d5168de6be5f1494b2035b61ff0958c11/include/rocksdb/table.h#L368-L394 ```C++ // We currently have five versions: // 0 -- This version is currently written out by all RocksDB's versions by // default. Can be read by really old RocksDB's. Doesn't support changing // checksum (default is CRC32). // 1 -- Can be read by RocksDB's versions since 3.0. Supports non-default // checksum, like xxHash. It is written by RocksDB when // BlockBasedTableOptions::checksum is something other than kCRC32c. (version // 0 is silently upconverted) // 2 -- Can be read by RocksDB's versions since 3.10. Changes the way we // encode compressed blocks with LZ4, BZip2 and Zlib compression. If you // don't plan to run RocksDB before version 3.10, you should probably use // this. // 3 -- Can be read by RocksDB's versions since 5.15. Changes the way we // encode the keys in index blocks. If you don't plan to run RocksDB before // version 5.15, you should probably use this. // This option only affects newly written tables. When reading existing // tables, the information about version is read from the footer. // 4 -- Can be read by RocksDB's versions since 5.16. Changes the way we // encode the values in index blocks. If you don't plan to run RocksDB before // version 5.16 and you are using index_block_restart_interval > 1, you should // probably use this as it would reduce the index size. // This option only affects newly written tables. When reading existing // tables, the information about version is read from the footer. // 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned // filters use a generally faster and more accurate Bloom filter // implementation, with a different schema. uint32_t format_version = 5; ``` Different format version requires different rocksDB version and it couldn't roll back once upgrade to new format version In our current RocksDB storage code, we hard code the format_version to 2, which is hard to to upgrade format_version to achieve new RocksDB's high performance. 1. Make the format_version configurable. Reviewers: Matteo Merli , Enrico Olivelli This closes #2824 from hangc0276/chenhang/make_rocksdb_format_version_configurable --- conf/bk_server.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/bk_server.conf b/conf/bk_server.conf index 801976ebbb3..f83a46e852c 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -740,6 +740,7 @@ gcEntryLogMetadataCacheEnabled=false # dbStorage_rocksDB_numFilesInLevel0=4 # dbStorage_rocksDB_maxSizeInLevel1MB=256 # dbStorage_rocksDB_logPath= +# dbStorage_rocksDB_format_version=2 ############################################## Metadata Services ############################################## From d0c9e5ce3c04047c9330a235e6eceee5a3297b96 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Tue, 15 Feb 2022 03:01:22 +0800 Subject: [PATCH 66/79] delete duplicated semicolon As title, delete duplicated semicolon Reviewers: Andrey Yegorov This closes #2810 from gaozhangmin/remove-duplicated-semicolon From 3443cec56e19490e8de700355650d85886db3108 Mon Sep 17 00:00:00 2001 From: Jack Vanlightly Date: Mon, 14 Feb 2022 20:11:38 +0100 Subject: [PATCH 67/79] BP-46: Running without journal proposal Includes the BP-46 design proposal markdown document. Master Issue: #2705 Reviewers: Andrey Yegorov , Enrico Olivelli This closes #2706 from Vanlightly/bp-44 --- site/bps/BP-46-run-without-journal.md | 205 ++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 site/bps/BP-46-run-without-journal.md diff --git a/site/bps/BP-46-run-without-journal.md b/site/bps/BP-46-run-without-journal.md new file mode 100644 index 00000000000..83030356db2 --- /dev/null +++ b/site/bps/BP-46-run-without-journal.md @@ -0,0 +1,205 @@ +--- +title: "BP-46: Running without the journal" +issue: https://github.com/apache/bookkeeper/2705 +state: "Under Discussion" +release: "N/A" +--- + +### Motivation + +The journal allows for fast add operations that provide strong data safety guarantees. An add operation is only acked to a client once written to the journal and an fsync performed. This however means that every entry must be written twice: once to the journal and once to an entry log file. + +This double write increases the cost of ownership as more disks must be provisioned to service requests and makes disk provisioning more complex (separating journal from entry log writes onto separate disks). Running without the journal would halve the disk IO required (ignoring indexes) thereby reducing costs and simplifying provisioning. + +However, running without the journal would introduce data consistency problems as the BookKeeper Replication Protocol requires that all writes are persistent for correctness. Running without the journal introduces the possibility of lost writes. In order to continue to offer strong data safety and support running without the journal, changes to the protocol are required. + +### A note on Response Codes + +The following categories are relevant: + +- Positive: OK +- Explicit Negative: NoSuchEntry/NoSuchLedger +- Unknown: Any other non-success response that is not an explicit negative. + +For correctness explicit negatives must be treated differently than other errors. + +### A note on Quorums + +In order to explain the protocol changes, it is useful to first consider how quorums are used for safety. We have the following relevant quorums: + +- Single bookie (S) +- Ack quorum (AQ) +- Write quorum (WQ) +- Quorum Coverage (QC) where QC = (WQ - AQ) + 1 +- Ensemble Coverage (EC) where EC = (E - AQ) + 1 +- Whole Ensemble + +Quorum Coverage (QC) and Ensemble Coverage (EC) are both defined by the following, only the cohorts differ: + +- A given property is satisfied by at least one bookie from every possible ack quorum within the cohort. +- There exists no ack quorum of bookies that do not satisfy the property within the cohort. + +For QC, the cohort is the writeset of a given entry, and therefore QC is only used when we need guarantees regarding a single entry. For EC, the cohort is the ensemble of bookies of the current fragment. EC is required when we need a guarantee across an entire fragment. + +For example: + +- For fencing, we need to ensure that no AQ of bookies is unfenced before starting the read/write phase of recovery. This is true once EC successful fencing responses have been received. +- For a recovery read, a read is only negative once we know that no AQ of bookies could exist that might have the entry. Doing otherwise could truncate committed entries from a ledger. A read is negative once NoSuchEntry responses reach QC. + +Different protocol actions require different quorums: + +- Add entry: AQ success responses +- Read entry: + - Positive when positive response from a single bookie + - Negative when explicit negative from all bookies + - Unknown: when at least one unknown and no positive from all bookies +- Fencing phase, LAC read (sent to ensemble of current fragment): + - Complete when EC positive responses + - Unknown (cannot make progress) when AQ unknown responses (fencing LAC reads cannot cause an explicit negative as fencing creates the ledger on the bookie if it doesn’t exist) +- Recovery read (sent to writeset of entry): + - Entry recoverable: AQ positive read responses + - Entry Unrecoverable: QC negative read responses + - Unknown (cannot make progress): + - QC unknown responses or + - All responses received, but not enough for either a positive or negative + + +### Impact of Undetected Data Loss on Consistency + +The ledger recovery process assumes that ledger entries are never arbitrarily lost. In the event of the loss of an entry, the recovery process can: +- allow the original client to keep writing entries to a ledger that has just been fenced and closed, thus losing those entries +- allow the recovery client to truncate the ledger too soon, closing it with a last entry id lower than that of previously acknowledged entries - thus losing data. + +The following scenarios assume existing behaviour but simply skipping the writing of entries and fencing ops to the journal. + +### Scenario 1 - Lost Fenced Status Allows Writes After Ledger Close + +1. 3 bookies, B1, B2 & B3 +2. 2 clients, C1 & C2 +3. 1 ledger, L1, with e3:w3:a2 configuration. +4. C1 writes entry E1 to L1. The write hits all three bookies. +5. C1 hangs for an indeterminate length of time. +6. C2 sees that C1 is unresponsive, and assumes it has failed. C2 tries to recover the ledger L1. +7. L1 sends a fencing message to all bookies in the ensemble. +8. The fencing message succeeds in arriving at B1 & B2 and is acknowledged by both. The message to B3 is lost. +9. C2 sees that at least one bookie in each possible ack quorum has acknowledged the fencing message (EC threshold reached), so continues with the read/write phase of recovery, finding that E1 is the last entry of the ledger, and committing the endpoint of the ledger in the ZK. +10. B2 crashes and boots again with all unflushed operations lost. +11. C1 wakes up and writes entry E2 to all bookies. B2 & B3 acknowledge positively, so C1 considers E2 as persisted. B1 rejects the message as the ledger is fenced, but since ack quorum is 2, B2 & B3 are enough to consider the entry written. + +### Scenario 2 - Recovery Truncates Previously Acknowledged Entries + +1. C1 adds E0 to B1, B2, B3 +2. B1 and B3 confirms. C1 confirms the write to its client. +3. C2 starts recovery +4. B2 fails to respond. C1 tries to change ensemble but gets a metadata version conflict. +5. B1 crashes and restarts, has lost E0 (undetected) +6. C2 fences the ledger on B1, B2, B3 +7. C2 sends Read E0 to B1, B2, B3 +8. B1 responds with NoSuchEntry +9. B2 responds with NoSuchEntry +10. QC negative response threshold reached. C2 closes the ledger as empty. Losing E0. + +The problem is that without the journal (and syncing to entry log files before acknowledgement) a bookie can: +- lose the fenced status of a previously existing ledger +- respond with an explicit negative even though it had previously seen an entry. + +Undetected data loss could occur when running without the journal. Bookie crashes and loses most recent entries and fence statuses that had not yet been written and synced to disk. + +### A note on cookies + +Cookies play an essential part in the bookkeeper replication protocol, but their purpose is often unclear. + +When a bookie boots for the first time, it generates a cookie. The cookie encapsulates the identity of the bookie and should be considered immutable. This identity contains the advertised address of the bookie, the disks used for the journal, index, and ledger storage, and a unique ID. The bookie writes the cookie to ZK and each of the disks in use. On all subsequent boots, if the cookie is missing from any of these places, the bookie fails to boot. + +The absence of a disk's cookie implies that the rest of the disk's data is also missing. Cookie validation is performed on boot-up and prevents the boot from succeeding if the validation fails, thus preventing the bookie starting with undetected data loss. + +This proposal improves the cookie mechanism by automating the resolution of a cookie validation error which currently requires human intervention to resolve. This automated feature will be configurable (enabled or disabled) and additionally a CLI command will be made available so an admin can manually run the operation (for when this feature is disabled - likely to be the default). + +### Proposed Changes + +The proposed changes involve: +- A new config that controls whether add operations go into the journal +- Detecting possible data loss on boot +- Prevent explicit negative responses when data loss may have occurred, instead reply with unknown code, until data is repaired. +- Repair data loss +- Auto fix cookies (with new config to enable or disable the feature) +- CLI command for admin to run fix cookie logic in the case that auto fix is disabled + +In these proposed changes, when running "without" the journal, the journal still exists, but add entry operations skip the addition to the journal. The boot-up sequence still replays the journal. + +Add operations can be configured to be written to the journal or not based on the config `journalWriteData`. When set to `false`, add operations are not added to the journal. + +### Detecting Data Loss On Boot + +The new mechanism for data loss detection is checking for an unclean shutdown (aka a crash or abrupt termination of the bookie). When an unclean shutdown is detected further measures are taken to prevent data inconsistency. + +The unclean shutdown detection will consist of setting a bit in the index on start-up and clearing it on shutdown. On subsequent start-up, the value will be checked and if it remains set, it knows that the prior shutdown was not clean. + +Cookie validation will continue to be used to detect booting with one or more missing or empty disks (that once existed and contained a cookie). + +### Protection Mechanism + +Once possible data loss has been detected the following protection mechanism is carried out during the boot: + +- Fencing: Ledger metadata for all ledgers of the cluster are obtained and all those ledgers are fenced on this bookie. This prevents data loss scenario 1. +- Limbo: All open ledgers are placed in the limbo status. Limbo ledgers can serve read requests, but never respond with an explicit negative, all explicit negatives are converted to unknowns (with the use of a new code EUNKNOWN). +- Recovery: All open ledgers are opened and recovered. +- Repair: Each ledger is scanned and any missing entries are sourced from peers. +- Limbo ledgers that have been repaired have their limbo status cleared. + +### The Full Boot-Up Sequence + +This mechanism of limbo ledgers and self-repair needs to work hand-in hand with the cookie validation check. Combining everything together: + +On boot: +1. Check for unclean shutdown and validate cookies +2. Fetch the metadata for all ledgers in the cluster from ZK where the bookie is a member of its ensemble. +3. Phase one: + - If the cookie check fails or unclean shutdown is detected: + - For each non-closed ledger, mark the ledger as fenced and in-limbo in the index. + - Update the cookie if it was a cookie failure +4. Phase two + - For each ledger + 1. If the ledger is in-limbo, open and recover the ledger. + 2. Check that all entries assigned to this bookie exist in the index. + 3. For any entries that are missing, copy from another bookie. + 4. Clear limbo status if set + +When booting a bookie with empty disks, only phase one needs to be complete before the bookie makes itself available for client requests. + +In phase one, if the cookie check fails, we mark all non-closed ledgers as “fenced”. This prevents any future writes to these ledgers on this bookie. This solves the problem of an empty bookie disk allowing writes to closed ledgers (Scenario 1). + +Given that the algorithm solves both the issues that cookies are designed to solve, we can now allow the bookie to update its cookie without operator intervention. + +### Formal Verification of Proposed Changes + +The use of the limbo status and fencing of all ledgers on boot-up when detecting an unclean shutdown has been modelled in TLA+. It does not model the whole boot-up sequence but a simplified version with only fencing and limbo status. + +The specification models the lifetime of a single ledger and includes a single bookie crashing, losing all data. The specification allows the testing of: + +- enabling/disabling the fencing +- enabling/disabling the limbo status. + +When running without limbo status, the model checker finds the counterexample of scenario 2. When running without fencing of all ledgers, the model checker finds the counterexample of scenario 1. When running with both enabled, the model checker finds no invariant violation. + +The specification can be found here: https://github.com/Vanlightly/bookkeeper-tlaplus + +### Public Interfaces + +- Return codes. Addition of a new return code: `EUNKNOWN` which is returned when a read hits an in-limbo ledger and that ledger not contain the requested entry id. +- Bookie ledger metadata format (LedgerData). Addition of the limbo status. + +### Compatibility, Deprecation, and Migration Plan + +- Because we only skip the journal for add operations, there is no impact on existing deployments. When a bookie is booted with the new version, and `journalWriteData` is set to false, the journal is still replayed on boot-up causing no risk of data loss in the transition. + +### Test Plan + +- There is confidence in the design due to the modelling in TLA+ but this model does not include the full boot sequence. +- The implementation will require aggressive chaos testing to ensure correctness. + +### Rejected Alternatives + +Entry Log Per Ledger (ELPL) without the journal. From our performance testing of ELPL, performance degrades significantly with a large number of active ledgers and syncing to disk multiple times a second (which is required to offer low latency writes). + +In the future this design could be extended to offer ledger level configuration of journal use. The scope of this BP is limited to cluster level. \ No newline at end of file From 10049f5654f2e03d84b6c0bf90b1d39dcd7a13ff Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Tue, 15 Feb 2022 03:17:28 +0800 Subject: [PATCH 68/79] Replication stat num-under-replicated-ledgers changed as with the process of replication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation Now ReplicationStats numUnderReplicatedLedger registers when `publishSuspectedLedgersAsync`, but its value doesn't decrease as with the ledger replicated successfully, We cannot know the progress of replication from the stat. Changes registers a notifyUnderReplicationLedgerChanged when auditor starts. numUnderReplicatedLedger value will decrease when the ledger path under replicate deleted. Reviewers: Nicolò Boschi , Enrico Olivelli , Andrey Yegorov This closes #2805 from gaozhangmin/replication-stats-num-under-replicated-ledgers --- .../meta/LedgerUnderreplicationManager.java | 9 ++++ .../meta/NullMetadataBookieDriver.java | 2 + .../meta/ZkLedgerUnderreplicationManager.java | 23 +++++++++ .../bookkeeper/replication/Auditor.java | 47 +++++++++++++++++++ .../replication/ReplicationStats.java | 2 + 5 files changed, 83 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java index ac468d31563..c24f9208023 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java @@ -236,6 +236,15 @@ boolean initializeLostBookieRecoveryDelay(int lostBookieRecoveryDelay) */ long getReplicasCheckCTime() throws ReplicationException.UnavailableException; + /** + * Receive notification asynchronously when the num of under-replicated ledgers Changed. + * + * @param cb + * @throws ReplicationException.UnavailableException + */ + void notifyUnderReplicationLedgerChanged(GenericCallback cb) + throws ReplicationException.UnavailableException; + /** * Receive notification asynchronously when the lostBookieRecoveryDelay value is Changed. * diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java index 3a4247f6c61..4a1e6e419fa 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java @@ -394,5 +394,7 @@ public String getReplicationWorkerIdRereplicatingLedger(long ledgerId) throws ReplicationException.UnavailableException { throw new ReplicationException.UnavailableException("null"); } + @Override + public void notifyUnderReplicationLedgerChanged(GenericCallback cb) {} } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java index b340e0721cb..9dcc81c1622 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java @@ -59,6 +59,7 @@ import org.apache.bookkeeper.util.BookKeeperConstants; import org.apache.bookkeeper.util.SubTreeCache; import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.AddWatchMode; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.Code; @@ -866,6 +867,28 @@ public int getLostBookieRecoveryDelay() throws UnavailableException { } } + @Override + public void notifyUnderReplicationLedgerChanged(GenericCallback cb) throws UnavailableException { + LOG.debug("notifyUnderReplicationLedgerChanged()"); + Watcher w = new Watcher() { + @Override + public void process(WatchedEvent e) { + if (e.getType() == Event.EventType.NodeDeleted && idExtractionPattern.matcher(e.getPath()).find()) { + cb.operationComplete(0, null); + } + } + }; + try { + zkc.addWatch(urLedgerPath, w, AddWatchMode.PERSISTENT_RECURSIVE); + } catch (KeeperException ke) { + LOG.error("Error while checking the state of underReplicated ledgers", ke); + throw new ReplicationException.UnavailableException("Error contacting zookeeper", ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new ReplicationException.UnavailableException("Interrupted while contacting zookeeper", ie); + } + } + @Override public void notifyLostBookieRecoveryDelayChanged(GenericCallback cb) throws UnavailableException { LOG.debug("notifyLostBookieRecoveryDelayChanged()"); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java index cd85145c1b3..c3b69dd4beb 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java @@ -34,8 +34,10 @@ import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY; +import static org.apache.bookkeeper.replication.ReplicationStats.NUM_REPLICATED_LEDGERS; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS; +import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS_GUAGE; import static org.apache.bookkeeper.replication.ReplicationStats.PLACEMENT_POLICY_CHECK_TIME; import static org.apache.bookkeeper.replication.ReplicationStats.REPLICAS_CHECK_TIME; import static org.apache.bookkeeper.replication.ReplicationStats.UNDER_REPLICATED_LEDGERS_TOTAL_SIZE; @@ -45,6 +47,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.collect.HashMultiset; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Multiset; import com.google.common.collect.Sets; @@ -162,6 +165,7 @@ public class Auditor implements AutoCloseable { private final AtomicInteger numLedgersFoundHavingLessThanAQReplicasOfAnEntry; private final AtomicInteger numLedgersHavingLessThanWQReplicasOfAnEntryGuageValue; private final AtomicInteger numLedgersFoundHavingLessThanWQReplicasOfAnEntry; + private final AtomicInteger underReplicatedLedgersGuageValue; private final long underreplicatedLedgerRecoveryGracePeriod; private final int zkOpTimeoutMs; private final Semaphore openLedgerNoRecoverySemaphore; @@ -234,6 +238,11 @@ public class Auditor implements AutoCloseable { help = "the number of delayed-bookie-audits cancelled" ) private final Counter numDelayedBookieAuditsCancelled; + @StatsDoc( + name = NUM_REPLICATED_LEDGERS, + help = "the number of replicated ledgers" + ) + private final Counter numReplicatedLedgers; @StatsDoc( name = NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY, help = "Gauge for number of ledgers not adhering to placement policy found in placement policy check" @@ -266,6 +275,11 @@ public class Auditor implements AutoCloseable { + ", this doesn't include ledgers counted towards numLedgersHavingLessThanAQReplicasOfAnEntry" ) private final Gauge numLedgersHavingLessThanWQReplicasOfAnEntry; + @StatsDoc( + name = NUM_UNDER_REPLICATED_LEDGERS_GUAGE, + help = "Gauge for num of underreplicated ledgers" + ) + private final Gauge numUnderReplicatedLedgers; static BookKeeper createBookKeeperClient(ServerConfiguration conf) throws InterruptedException, IOException { return createBookKeeperClient(conf, NullStatsLogger.INSTANCE); @@ -363,6 +377,7 @@ public Auditor(final String bookieIdentifier, this.openLedgerNoRecoverySemaphoreWaitTimeoutMSec = conf.getAuditorAcquireConcurrentOpenLedgerOperationsTimeoutMSec(); + this.underReplicatedLedgersGuageValue = new AtomicInteger(0); numUnderReplicatedLedger = this.statsLogger.getOpStatsLogger(ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS); underReplicatedLedgerTotalSize = this.statsLogger.getOpStatsLogger(UNDER_REPLICATED_LEDGERS_TOTAL_SIZE); uRLPublishTimeForLostBookies = this.statsLogger @@ -379,6 +394,7 @@ public Auditor(final String bookieIdentifier, numBookieAuditsDelayed = this.statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); numDelayedBookieAuditsCancelled = this.statsLogger .getCounter(ReplicationStats.NUM_DELAYED_BOOKIE_AUDITS_DELAYES_CANCELLED); + numReplicatedLedgers = this.statsLogger.getCounter(NUM_REPLICATED_LEDGERS); numLedgersNotAdheringToPlacementPolicy = new Gauge() { @Override public Integer getDefaultValue() { @@ -459,6 +475,18 @@ public Integer getSample() { }; this.statsLogger.registerGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY, numLedgersHavingLessThanWQReplicasOfAnEntry); + numUnderReplicatedLedgers = new Gauge() { + @Override + public Integer getDefaultValue() { + return 0; + } + + @Override + public Integer getSample() { + return underReplicatedLedgersGuageValue.get(); + } + }; + this.statsLogger.registerGauge(NUM_UNDER_REPLICATED_LEDGERS_GUAGE, numUnderReplicatedLedgers); this.bkc = bkc; this.ownBkc = ownBkc; @@ -703,6 +731,15 @@ public void start() { submitShutdownTask(); } + try { + this.ledgerUnderreplicationManager.notifyUnderReplicationLedgerChanged( + new UnderReplicatedLedgersChangedCb()); + } catch (UnavailableException ue) { + LOG.error("Exception while registering for under-replicated ledgers change notification, so exiting", + ue); + submitShutdownTask(); + } + scheduleBookieCheckTask(); scheduleCheckAllLedgersTask(); schedulePlacementPolicyCheckTask(); @@ -1010,6 +1047,16 @@ public void run() { }), initialDelay, interval, TimeUnit.SECONDS); } + private class UnderReplicatedLedgersChangedCb implements GenericCallback { + @Override + public void operationComplete(int rc, Void result) { + Iterator underreplicatedLedgersInfo = ledgerUnderreplicationManager + .listLedgersToRereplicate(null); + underReplicatedLedgersGuageValue.set(Iterators.size(underreplicatedLedgersInfo)); + numReplicatedLedgers.inc(); + } + } + private class LostBookieRecoveryDelayChangedCb implements GenericCallback { @Override public void operationComplete(int rc, Void result) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java index d04fcf04279..74b76b23b22 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java @@ -64,4 +64,6 @@ public interface ReplicationStats { String REPLICATE_EXCEPTION = "exceptions"; String NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER = "NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER"; String NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION = "NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION"; + String NUM_UNDER_REPLICATED_LEDGERS_GUAGE = "NUM_UNDER_REPLICATED_LEDGERS_GUAGE"; + String NUM_REPLICATED_LEDGERS = "NUM_REPLICATED_LEDGERS"; } From c7d3c6950bd4ed7ff1c9e20fbb69c41a68eb40c2 Mon Sep 17 00:00:00 2001 From: shustsud <51769018+shustsud@users.noreply.github.com> Date: Tue, 15 Feb 2022 04:20:49 +0900 Subject: [PATCH 69/79] Explicit error message if an exception other than BKNoSuchLedgerExistsOnMetadataServerException occurs in over-replicated ledger GC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation - Even if an exception other than BKNoSuchLedgerExistsOnMetadataServerException occurs of readLedgerMetadata in over-replicated ledger GC, nothing will be output to the log. (https://github.com/apache/bookkeeper/pull/2844#discussion_r735219876) ### Changes - If an exception other than BKNoSuchLedgerExistsOnMetadataServerException occurs in readLedgerMetadata, output information to the log. Reviewers: Andrey Yegorov , Nicolò Boschi This closes #2873 from shustsud/improved_error_handling --- .../bookkeeper/bookie/ScanAndCompareGarbageCollector.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java index 93bf47c4e03..b3c724b231e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java @@ -243,6 +243,10 @@ private Set removeOverReplicatedledgers(Set bkActiveledgers, final G continue; } } catch (Throwable t) { + if (!(t.getCause() instanceof BKException.BKNoSuchLedgerExistsOnMetadataServerException)) { + LOG.warn("Failed to get metadata for ledger {}. {}: {}", + ledgerId, t.getClass().getName(), t.getMessage()); + } latch.countDown(); continue; } @@ -268,6 +272,10 @@ private Set removeOverReplicatedledgers(Set bkActiveledgers, final G overReplicatedLedgers.add(ledgerId); garbageCleaner.clean(ledgerId); } + } else if (!(exception + instanceof BKException.BKNoSuchLedgerExistsOnMetadataServerException)) { + LOG.warn("Failed to get metadata for ledger {}. {}: {}", + ledgerId, exception.getClass().getName(), exception.getMessage()); } } finally { semaphore.release(); From e1e46cc19ae7d3f73d5f5e1ab64280e444926283 Mon Sep 17 00:00:00 2001 From: Eric Shen Date: Tue, 15 Feb 2022 03:32:55 +0800 Subject: [PATCH 70/79] fix(cli): incorrect description for autodiscovery Signed-off-by: Eric Shen Descriptions of the changes in this PR: ### Motivation The description of `bin/bookkeeper autorecovery` is wrong, it won't start in daemon. ### Changes * Changed the description in bookkeeper shell * Update the doc Reviewers: Yong Zhang This closes #2910 from ericsyh/fix-bk-cli --- bin/bookkeeper | 2 +- site/_data/cli/bookkeeper.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bookkeeper b/bin/bookkeeper index 47cb42de270..ac409684ff5 100755 --- a/bin/bookkeeper +++ b/bin/bookkeeper @@ -62,7 +62,7 @@ where command is one of: [service commands] bookie Run a bookie server - autorecovery Run AutoRecovery service daemon + autorecovery Run AutoRecovery service zookeeper Run zookeeper server [development commands] diff --git a/site/_data/cli/bookkeeper.yaml b/site/_data/cli/bookkeeper.yaml index cc4b2eca5d5..fd780f151f3 100644 --- a/site/_data/cli/bookkeeper.yaml +++ b/site/_data/cli/bookkeeper.yaml @@ -7,7 +7,7 @@ commands: description: Starts up an ensemble of N bookies in a single JVM process. Typically used for local experimentation and development. argument: N - name: autorecovery - description: Runs the autorecovery service daemon. + description: Runs the autorecovery service. - name: upgrade description: Upgrades the bookie's filesystem. options: From cee0627f9d1a4bd6c792f32fc418acdf7e471937 Mon Sep 17 00:00:00 2001 From: Qiang Zhao <74767115+mattisonchao@users.noreply.github.com> Date: Tue, 15 Feb 2022 03:36:06 +0800 Subject: [PATCH 71/79] Add flaky-test template to track many flaky-test. ### Motivation I found many flaky-test like #3031 #3034 #3033. Because many flaky tests are actually production code issues so I think it's a good way to add flaky-test template to track them ### Changes - Add flaky-test template. Reviewers: Andrey Yegorov This closes #3035 from mattisonchao/template_flaky_test --- .github/ISSUE_TEMPLATE/flaky_test.md | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/flaky_test.md diff --git a/.github/ISSUE_TEMPLATE/flaky_test.md b/.github/ISSUE_TEMPLATE/flaky_test.md new file mode 100644 index 00000000000..fc8159776c7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/flaky_test.md @@ -0,0 +1,31 @@ +--- +name: Flaky test +about: Report a flaky test failure +title: 'Flaky-test: test_class.test_method' +labels: ["area/tests", "flaky-tests"] +assignees: '' +--- + +test_class.test_method is flaky. It fails sporadically. + +``` +[relevant parts of the exception stacktrace here] +``` + +